==================================================================================================== ========================================= Pod describe ========================================= ==================================================================================================== Name: lodemon-654bd894f6-w9k2n Namespace: xlou Priority: 0 Service Account: k8s-svc-acct-lodemon Node: gke-xlou-cdm-frontend-18a44ad7-wxs9/10.142.0.11 Start Time: Tue, 17 Dec 2024 17:50:16 +0000 Labels: app=lodemon app.kubernetes.io/name=lodemon pod-template-hash=654bd894f6 skaffold.dev/run-id=4bc6ecde-64f8-4bd8-85e8-7ba86ea128d4 Annotations: Status: Running IP: 10.106.40.171 IPs: IP: 10.106.40.171 Controlled By: ReplicaSet/lodemon-654bd894f6 Containers: lodemon: Container ID: containerd://1510f7da5bf79f2aa7921e28253fc1b2271817bd1f8e307962d227e32d8938d2 Image: gcr.io/engineeringpit/lodestar-images/lodestarbox:master Image ID: gcr.io/engineeringpit/lodestar-images/lodestarbox@sha256:9c42d6522c86b889e0ca15cd976cb8091dfcdb966a6690404435e9f3644081e0 Port: 8080/TCP Host Port: 0/TCP Command: python3 Args: /lodestar/scripts/lodemon_run.py -W default State: Running Started: Tue, 17 Dec 2024 17:50:17 +0000 Ready: True Restart Count: 0 Limits: cpu: 2 memory: 2Gi Requests: cpu: 1 memory: 1Gi Liveness: exec [cat /tmp/lodemon_alive] delay=20s timeout=1s period=10s #success=1 #failure=3 Readiness: exec [cat /tmp/lodemon_alive] delay=20s timeout=1s period=10s #success=1 #failure=3 Environment: SKAFFOLD_PROFILE: idc ds-only Mounts: /lodestar/config/config.yaml from config (rw,path="config.yaml") /lodestar/config/deployments_to_monitor.yaml from deployments-config (rw,path="deployments_to_monitor.yaml") /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-zwkwr (ro) Conditions: Type Status PodReadyToStartContainers True Initialized True Ready True ContainersReady True PodScheduled True Volumes: config: Type: ConfigMap (a volume populated by a ConfigMap) Name: lodemon-config Optional: false deployments-config: Type: ConfigMap (a volume populated by a ConfigMap) Name: lodemon-deployments Optional: false kube-api-access-zwkwr: Type: Projected (a volume that contains injected data from multiple sources) TokenExpirationSeconds: 3607 ConfigMapName: kube-root-ca.crt ConfigMapOptional: DownwardAPI: true QoS Class: Burstable Node-Selectors: Tolerations: node.kubernetes.io/not-ready:NoExecute op=Exists for 300s node.kubernetes.io/unreachable:NoExecute op=Exists for 300s Events: ==================================================================================================== =========================================== Pod logs =========================================== ==================================================================================================== * Serving Flask app 'lodemon_run' * Debug mode: off WARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead. * Running on all addresses (0.0.0.0) * Running on http://127.0.0.1:8080 * Running on http://10.106.40.171:8080 Press CTRL+C to quit 17:50:52 INFO 17:50:52 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 17:50:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:52 INFO [loop_until]: OK (rc = 0) 17:50:52 DEBUG --- stdout --- 17:50:52 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 17:50:52 DEBUG --- stderr --- 17:50:52 DEBUG 17:50:52 INFO 17:50:52 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 17:50:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:52 INFO [loop_until]: OK (rc = 0) 17:50:52 DEBUG --- stdout --- 17:50:52 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 17:50:52 DEBUG --- stderr --- 17:50:52 DEBUG 17:50:52 INFO 17:50:52 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 17:50:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:52 INFO [loop_until]: OK (rc = 0) 17:50:52 DEBUG --- stdout --- 17:50:52 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 17:50:52 DEBUG --- stderr --- 17:50:52 DEBUG 17:50:52 INFO 17:50:52 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 17:50:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:52 INFO [loop_until]: OK (rc = 0) 17:50:52 DEBUG --- stdout --- 17:50:52 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 17:50:52 DEBUG --- stderr --- 17:50:52 DEBUG 17:50:52 INFO 17:50:52 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 17:50:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:52 INFO [loop_until]: OK (rc = 0) 17:50:52 DEBUG --- stdout --- 17:50:52 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 17:50:52 DEBUG --- stderr --- 17:50:52 DEBUG 17:50:52 INFO 17:50:52 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 17:50:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:52 INFO [loop_until]: OK (rc = 0) 17:50:52 DEBUG --- stdout --- 17:50:52 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 17:50:52 DEBUG --- stderr --- 17:50:52 DEBUG 17:50:52 INFO 17:50:52 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 17:50:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:53 INFO [loop_until]: OK (rc = 0) 17:50:53 DEBUG --- stdout --- 17:50:53 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 17:50:53 DEBUG --- stderr --- 17:50:53 DEBUG 17:50:53 INFO 17:50:53 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 17:50:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:53 INFO [loop_until]: OK (rc = 0) 17:50:53 DEBUG --- stdout --- 17:50:53 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 17:50:53 DEBUG --- stderr --- 17:50:53 DEBUG 17:50:53 INFO 17:50:53 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 17:50:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:53 INFO [loop_until]: OK (rc = 0) 17:50:53 DEBUG --- stdout --- 17:50:53 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 17:50:53 DEBUG --- stderr --- 17:50:53 DEBUG 17:50:53 INFO 17:50:53 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 17:50:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:53 INFO [loop_until]: OK (rc = 0) 17:50:53 DEBUG --- stdout --- 17:50:53 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 17:50:53 DEBUG --- stderr --- 17:50:53 DEBUG 17:50:53 INFO 17:50:53 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 17:50:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:53 INFO [loop_until]: OK (rc = 0) 17:50:53 DEBUG --- stdout --- 17:50:53 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 17:50:53 DEBUG --- stderr --- 17:50:53 DEBUG 17:50:53 INFO 17:50:53 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 17:50:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:53 INFO [loop_until]: OK (rc = 0) 17:50:53 DEBUG --- stdout --- 17:50:53 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 17:50:53 DEBUG --- stderr --- 17:50:53 DEBUG 17:50:53 INFO 17:50:53 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 17:50:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:53 INFO [loop_until]: OK (rc = 0) 17:50:53 DEBUG --- stdout --- 17:50:53 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 17:50:53 DEBUG --- stderr --- 17:50:53 DEBUG 17:50:54 INFO 17:50:54 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 17:50:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:54 INFO [loop_until]: OK (rc = 0) 17:50:54 DEBUG --- stdout --- 17:50:54 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 17:50:54 DEBUG --- stderr --- 17:50:54 DEBUG 17:50:54 INFO 17:50:54 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 17:50:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:54 INFO [loop_until]: OK (rc = 0) 17:50:54 DEBUG --- stdout --- 17:50:54 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 17:50:54 DEBUG --- stderr --- 17:50:54 DEBUG 17:50:54 INFO 17:50:54 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 17:50:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:54 INFO [loop_until]: OK (rc = 0) 17:50:54 DEBUG --- stdout --- 17:50:54 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 17:50:54 DEBUG --- stderr --- 17:50:54 DEBUG 17:50:54 INFO 17:50:54 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 17:50:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:54 INFO [loop_until]: OK (rc = 0) 17:50:54 DEBUG --- stdout --- 17:50:54 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 17:50:54 DEBUG --- stderr --- 17:50:54 DEBUG 17:50:54 INFO 17:50:54 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 17:50:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:54 INFO [loop_until]: OK (rc = 0) 17:50:54 DEBUG --- stdout --- 17:50:54 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 17:50:54 DEBUG --- stderr --- 17:50:54 DEBUG 17:50:54 INFO 17:50:54 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 17:50:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:54 INFO [loop_until]: OK (rc = 0) 17:50:54 DEBUG --- stdout --- 17:50:54 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 17:50:54 DEBUG --- stderr --- 17:50:54 DEBUG 17:50:54 INFO 17:50:54 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 17:50:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:55 INFO [loop_until]: OK (rc = 0) 17:50:55 DEBUG --- stdout --- 17:50:55 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 17:50:55 DEBUG --- stderr --- 17:50:55 DEBUG 17:50:55 INFO 17:50:55 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 17:50:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:55 INFO [loop_until]: OK (rc = 0) 17:50:55 DEBUG --- stdout --- 17:50:55 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 17:50:55 DEBUG --- stderr --- 17:50:55 DEBUG 17:50:55 INFO 17:50:55 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 17:50:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:55 INFO [loop_until]: OK (rc = 0) 17:50:55 DEBUG --- stdout --- 17:50:55 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 17:50:55 DEBUG --- stderr --- 17:50:55 DEBUG 17:50:55 INFO 17:50:55 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 17:50:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:55 INFO [loop_until]: OK (rc = 0) 17:50:55 DEBUG --- stdout --- 17:50:55 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 17:50:55 DEBUG --- stderr --- 17:50:55 DEBUG 17:50:55 INFO 17:50:55 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 17:50:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:55 INFO [loop_until]: OK (rc = 0) 17:50:55 DEBUG --- stdout --- 17:50:55 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 17:50:55 DEBUG --- stderr --- 17:50:55 DEBUG 17:50:55 INFO 17:50:55 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 17:50:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:55 INFO [loop_until]: OK (rc = 0) 17:50:55 DEBUG --- stdout --- 17:50:55 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 17:50:55 DEBUG --- stderr --- 17:50:55 DEBUG 17:50:55 INFO 17:50:55 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 17:50:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:55 INFO [loop_until]: OK (rc = 0) 17:50:55 DEBUG --- stdout --- 17:50:55 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 17:50:55 DEBUG --- stderr --- 17:50:55 DEBUG 17:50:56 INFO 17:50:56 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 17:50:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:56 INFO [loop_until]: OK (rc = 0) 17:50:56 DEBUG --- stdout --- 17:50:56 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 17:50:56 DEBUG --- stderr --- 17:50:56 DEBUG 17:50:56 INFO 17:50:56 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 17:50:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:56 INFO [loop_until]: OK (rc = 0) 17:50:56 DEBUG --- stdout --- 17:50:56 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 17:50:56 DEBUG --- stderr --- 17:50:56 DEBUG 17:50:56 INFO 17:50:56 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 17:50:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:56 INFO [loop_until]: OK (rc = 0) 17:50:56 DEBUG --- stdout --- 17:50:56 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 17:50:56 DEBUG --- stderr --- 17:50:56 DEBUG 17:50:56 INFO 17:50:56 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 17:50:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:56 INFO [loop_until]: OK (rc = 0) 17:50:56 DEBUG --- stdout --- 17:50:56 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 17:50:56 DEBUG --- stderr --- 17:50:56 DEBUG 17:50:56 INFO 17:50:56 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 17:50:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:56 INFO [loop_until]: OK (rc = 0) 17:50:56 DEBUG --- stdout --- 17:50:56 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 17:50:56 DEBUG --- stderr --- 17:50:56 DEBUG 17:50:56 INFO 17:50:56 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 17:50:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:56 INFO [loop_until]: OK (rc = 0) 17:50:56 DEBUG --- stdout --- 17:50:56 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 17:50:56 DEBUG --- stderr --- 17:50:56 DEBUG 17:50:56 INFO 17:50:56 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 17:50:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:57 INFO [loop_until]: OK (rc = 0) 17:50:57 DEBUG --- stdout --- 17:50:57 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 17:50:57 DEBUG --- stderr --- 17:50:57 DEBUG 17:50:57 INFO 17:50:57 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 17:50:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:57 INFO [loop_until]: OK (rc = 0) 17:50:57 DEBUG --- stdout --- 17:50:57 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 17:50:57 DEBUG --- stderr --- 17:50:57 DEBUG 17:50:57 INFO 17:50:57 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 17:50:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:57 INFO [loop_until]: OK (rc = 0) 17:50:57 DEBUG --- stdout --- 17:50:57 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 17:50:57 DEBUG --- stderr --- 17:50:57 DEBUG 17:50:57 INFO 17:50:57 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 17:50:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:57 INFO [loop_until]: OK (rc = 0) 17:50:57 DEBUG --- stdout --- 17:50:57 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 17:50:57 DEBUG --- stderr --- 17:50:57 DEBUG 17:50:57 INFO 17:50:57 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 17:50:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:57 INFO [loop_until]: OK (rc = 0) 17:50:57 DEBUG --- stdout --- 17:50:57 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 17:50:57 DEBUG --- stderr --- 17:50:57 DEBUG 17:50:57 INFO 17:50:57 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 17:50:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:57 INFO [loop_until]: OK (rc = 0) 17:50:57 DEBUG --- stdout --- 17:50:57 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 17:50:57 DEBUG --- stderr --- 17:50:57 DEBUG 17:50:57 INFO Initializing monitoring instance threads 17:50:57 DEBUG Monitoring instance thread list: [, , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 17:50:57 INFO Starting instance threads 17:50:57 INFO 17:50:57 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 17:50:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:57 INFO Thread started 17:50:57 INFO 17:50:57 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 17:50:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:57 INFO Thread started 17:50:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457857" 17:50:57 INFO Thread started 17:50:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734457857" 17:50:57 INFO Thread started 17:50:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457857" 17:50:57 INFO Thread started 17:50:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457857" 17:50:57 INFO Thread started 17:50:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734457857" 17:50:57 INFO Thread started 17:50:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734457857" 17:50:57 INFO Thread started 17:50:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734457857" 17:50:57 INFO Thread started 17:50:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734457857" 17:50:57 INFO Thread started 17:50:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734457857" 17:50:57 INFO [http_cmd]: http status code OK 17:50:57 INFO Thread started 17:50:57 DEBUG --- status code --- 17:50:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457857" 17:50:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:57 INFO Thread started 17:50:57 INFO [http_cmd]: http status code OK 17:50:57 INFO [http_cmd]: http status code OK 17:50:57 INFO [http_cmd]: http status code OK 17:50:57 DEBUG --- http response --- 17:50:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457857" 17:50:57 DEBUG --- status code --- 17:50:57 INFO Thread started 17:50:57 DEBUG --- status code --- 17:50:57 DEBUG --- status code --- 17:50:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 17:50:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457857" 17:50:57 INFO Thread started 17:50:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:57 INFO [http_cmd]: http status code OK 17:50:57 DEBUG --- http response --- 17:50:57 DEBUG ----- output ----- 17:50:57 DEBUG --- http response --- 17:50:57 DEBUG --- http response --- 17:50:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457857" 17:50:57 DEBUG --- status code --- 17:50:57 INFO Thread started 17:50:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 17:50:57 DEBUG { 17:50:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:50:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 17:50:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:57 INFO [http_cmd]: http status code OK 17:50:57 DEBUG ----- output ----- 17:50:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734457857" 17:50:57 DEBUG "status": "success", 17:50:57 INFO Thread started 17:50:57 DEBUG ----- output ----- 17:50:57 DEBUG ----- output ----- 17:50:57 DEBUG --- http response --- 17:50:57 DEBUG --- status code --- 17:50:57 DEBUG { 17:50:57 DEBUG "data": { 17:50:57 INFO [http_cmd]: http status code OK 17:50:57 DEBUG { 17:50:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734457857" 17:50:57 DEBUG { 17:50:57 INFO Thread started 17:50:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 17:50:57 DEBUG "status": "success", 17:50:57 DEBUG "resultType": "vector", 17:50:57 DEBUG --- status code --- 17:50:57 DEBUG "status": "success", 17:50:57 DEBUG "status": "success", 17:50:57 DEBUG --- http response --- 17:50:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734457857" 17:50:57 DEBUG ----- output ----- 17:50:57 INFO Thread started 17:50:57 DEBUG "data": { 17:50:57 DEBUG "result": [ 17:50:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:57 DEBUG "data": { 17:50:57 DEBUG "data": { 17:50:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:50:57 DEBUG ----- output ----- 17:50:57 DEBUG "resultType": "vector", 17:50:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734457857" 17:50:57 INFO Thread started 17:50:57 DEBUG { 17:50:57 DEBUG --- http response --- 17:50:57 DEBUG "resultType": "vector", 17:50:57 DEBUG "resultType": "vector", 17:50:57 DEBUG { 17:50:57 DEBUG { 17:50:57 DEBUG "result": [ 17:50:57 DEBUG "metric": { 17:50:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734457857" 17:50:57 INFO [http_cmd]: http status code OK 17:50:57 INFO Thread started 17:50:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:50:57 DEBUG "result": [ 17:50:57 INFO [http_cmd]: http status code OK 17:50:57 INFO [http_cmd]: http status code OK 17:50:57 INFO [http_cmd]: http status code OK 17:50:57 DEBUG "result": [ 17:50:57 INFO [http_cmd]: http status code OK 17:50:57 INFO [http_cmd]: http status code OK 17:50:57 DEBUG "status": "success", 17:50:57 INFO [http_cmd]: http status code OK 17:50:57 INFO [http_cmd]: http status code OK 17:50:57 INFO [http_cmd]: http status code OK 17:50:57 DEBUG "status": "success", 17:50:57 INFO [http_cmd]: http status code OK 17:50:57 DEBUG { 17:50:57 DEBUG "pod": "ds-cts-0" 17:50:57 DEBUG --- status code --- 17:50:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734457857" 17:50:57 INFO Thread started 17:50:57 DEBUG { 17:50:57 DEBUG --- status code --- 17:50:57 DEBUG --- status code --- 17:50:57 DEBUG --- status code --- 17:50:57 DEBUG { 17:50:57 DEBUG --- status code --- 17:50:57 DEBUG --- status code --- 17:50:57 DEBUG "data": { 17:50:57 DEBUG --- status code --- 17:50:57 DEBUG --- status code --- 17:50:57 DEBUG --- status code --- 17:50:57 DEBUG "data": { 17:50:57 DEBUG --- status code --- 17:50:57 DEBUG "metric": { 17:50:57 DEBUG }, 17:50:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734457857" 17:50:57 DEBUG "metric": { 17:50:57 INFO Thread started 17:50:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:57 DEBUG "metric": { 17:50:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:57 DEBUG --- http response --- 17:50:57 DEBUG "resultType": "vector", 17:50:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:57 DEBUG "resultType": "vector", 17:50:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:57 DEBUG "pod": "ds-cts-0" 17:50:57 DEBUG "value": [ 17:50:57 DEBUG --- http response --- 17:50:57 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 17:50:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734457857" 17:50:57 DEBUG --- http response --- 17:50:57 INFO Thread started 17:50:57 DEBUG --- http response --- 17:50:57 DEBUG --- http response --- 17:50:57 DEBUG "pod": "ds-cts-0" 17:50:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:50:57 DEBUG "result": [ 17:50:57 DEBUG --- http response --- 17:50:57 DEBUG --- http response --- 17:50:57 DEBUG --- http response --- 17:50:57 DEBUG "result": [ 17:50:57 DEBUG --- http response --- 17:50:57 DEBUG }, 17:50:57 DEBUG 17:50:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:50:57 DEBUG }, 17:50:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:50:57 INFO [http_cmd]: http status code OK 17:50:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457857" 17:50:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:50:57 INFO Thread started 17:50:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:50:57 DEBUG }, 17:50:57 DEBUG --- http response --- 17:50:57 DEBUG ----- output ----- 17:50:57 DEBUG { 17:50:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:50:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:50:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:50:57 DEBUG { 17:50:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:50:57 DEBUG "value": [ 17:50:57 DEBUG 17:50:57 INFO [http_cmd]: http status code OK 17:50:57 INFO [http_cmd]: http status code OK 17:50:57 DEBUG "value": [ 17:50:57 DEBUG --- status code --- 17:50:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457857" 17:50:57 INFO Thread started 17:50:57 DEBUG "value": [ 17:50:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:50:57 DEBUG { 17:50:57 DEBUG "metric": { 17:50:57 DEBUG ----- output ----- 17:50:57 DEBUG "metric": { 17:50:57 INFO [loop_until]: OK (rc = 0) 17:50:57 INFO [http_cmd]: http status code OK 17:50:57 DEBUG ----- output ----- 17:50:57 DEBUG 17:50:57 DEBUG 17:50:57 DEBUG --- status code --- 17:50:57 DEBUG --- status code --- 17:50:57 DEBUG 17:50:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457857" 17:50:57 DEBUG 17:50:57 INFO Thread started 17:50:57 DEBUG "status": "success", 17:50:57 DEBUG "pod": "ds-cts-0" 17:50:57 DEBUG { 17:50:57 DEBUG "instance": "10.142.0.11:9100" 17:50:57 DEBUG --- stdout --- 17:50:57 DEBUG --- status code --- 17:50:57 DEBUG { 17:50:57 DEBUG 17:50:57 DEBUG [...] 17:50:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:57 DEBUG 17:50:57 DEBUG --- http response --- 17:50:57 DEBUG 17:50:57 INFO [http_cmd]: http status code OK 17:50:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457857" 17:50:57 DEBUG "data": { 17:50:57 INFO Thread started 17:50:57 INFO [loop_until]: OK (rc = 0) 17:50:57 DEBUG }, 17:50:57 DEBUG "status": "success", 17:50:57 DEBUG }, 17:50:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:57 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 9m 388Mi ds-idrepo-0 10m 558Mi rcs-5d696ff5b-7ttvr 10m 292Mi rcs-5d696ff5b-wwlbr 4m 319Mi 17:50:57 DEBUG "status": "success", 17:50:57 DEBUG 17:50:57 DEBUG 17:50:57 DEBUG --- http response --- 17:50:57 DEBUG --- http response --- 17:50:57 DEBUG 17:50:57 DEBUG 17:50:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:50:57 DEBUG --- status code --- 17:50:57 DEBUG "resultType": "vector", 17:50:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457857" 17:50:57 DEBUG --- stdout --- 17:50:57 INFO Thread started 17:50:57 DEBUG "value": [ 17:50:57 DEBUG "data": { 17:50:57 DEBUG "value": [ 17:50:57 DEBUG --- http response --- 17:50:57 DEBUG --- stderr --- 17:50:57 DEBUG 17:50:57 DEBUG [...] 17:50:57 DEBUG 17:50:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:50:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:50:57 DEBUG [...] 17:50:57 INFO [http_cmd]: http status code OK 17:50:57 DEBUG [...] 17:50:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:57 DEBUG "result": [ 17:50:57 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 109m 0% 2262Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 179m 0% 2549Mi 8% 17:50:57 DEBUG 17:50:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734457857" 17:50:57 DEBUG "resultType": "vector", 17:50:57 INFO Thread started 17:50:57 DEBUG 17:50:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:50:57 DEBUG "data": { 17:50:57 DEBUG 17:50:57 DEBUG 17:50:57 DEBUG 17:50:57 INFO [http_cmd]: http status code OK 17:50:57 DEBUG --- status code --- 17:50:57 DEBUG 17:50:57 DEBUG --- http response --- 17:50:57 DEBUG { 17:50:57 INFO [http_cmd]: http status code OK 17:50:57 DEBUG --- stderr --- 17:50:57 DEBUG 17:50:57 DEBUG "result": [ 17:50:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457857" 17:50:57 DEBUG 17:50:57 INFO Thread started 17:50:57 DEBUG "resultType": "vector", 17:50:57 DEBUG 17:50:57 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 17:50:57 DEBUG 17:50:57 DEBUG --- status code --- 17:50:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:57 DEBUG 17:50:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:50:57 DEBUG "metric": { 17:50:57 DEBUG --- status code --- 17:50:57 DEBUG 17:50:57 DEBUG 17:50:57 DEBUG { 17:50:57 DEBUG 17:50:57 DEBUG "result": [ 17:50:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734457857" 17:50:57 DEBUG 17:50:57 INFO Thread started 17:50:57 DEBUG }, 17:50:57 DEBUG 17:50:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:57 DEBUG --- http response --- 17:50:57 DEBUG 17:50:57 DEBUG "instance": "10.142.0.11:9100" 17:50:57 INFO [http_cmd]: http status code OK 17:50:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:57 DEBUG [...] 17:50:57 DEBUG "metric": { 17:50:57 DEBUG [...] 17:50:57 DEBUG { 17:50:57 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 17:50:57 DEBUG "value": [ 17:50:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734457857" 17:50:57 INFO Thread started 17:50:57 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 17:50:57 DEBUG --- http response --- 17:50:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:50:57 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 17:50:57 DEBUG }, 17:50:57 DEBUG --- status code --- 17:50:57 DEBUG --- http response --- 17:50:57 DEBUG 17:50:57 DEBUG "instance": "10.142.0.11:9100" 17:50:57 INFO [http_cmd]: http status code OK 17:50:57 DEBUG 17:50:57 DEBUG "metric": { 17:50:57 DEBUG }, 17:50:57 DEBUG 1734457857, 17:50:57 DEBUG }, 17:50:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734457857" 17:50:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:50:57 INFO Thread started 17:50:57 DEBUG }, 17:50:57 DEBUG "value": [ 17:50:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:50:57 DEBUG 17:50:57 DEBUG }, 17:50:57 DEBUG --- status code --- 17:50:57 DEBUG 17:50:57 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 17:50:57 DEBUG "value": [ 17:50:57 DEBUG "0.007766553947930008" 17:50:57 INFO [http_cmd]: http status code OK 17:50:57 DEBUG "value": [ 17:50:57 INFO [http_cmd]: http status code OK 17:50:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734457857" 17:50:57 DEBUG "value": [ 17:50:57 INFO Thread started 17:50:57 DEBUG 17:50:57 DEBUG --- http response --- 17:50:57 DEBUG 17:50:57 DEBUG "value": [ 17:50:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:57 DEBUG 17:50:57 DEBUG }, 17:50:57 DEBUG 1734457857, 17:50:57 DEBUG ] 17:50:57 DEBUG --- status code --- 17:50:57 DEBUG 1734457857, 17:50:57 DEBUG --- status code --- 17:50:57 DEBUG 1734457857, 17:50:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457857" 17:50:57 DEBUG 17:50:57 INFO Thread started 17:50:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:50:57 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 17:50:57 DEBUG 17:50:57 DEBUG --- http response --- 17:50:57 DEBUG "instance": "10.142.0.51:9100" 17:50:57 DEBUG "value": [ 17:50:57 INFO [http_cmd]: http status code OK 17:50:57 DEBUG "671043584" 17:50:57 DEBUG } 17:50:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:57 DEBUG "2.7928213553461476" 17:50:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:57 DEBUG "524.4558258642766" 17:50:57 DEBUG 17:50:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457857" 17:50:57 INFO Thread started 17:50:57 DEBUG }, 17:50:57 DEBUG 17:50:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:50:57 DEBUG }, 17:50:57 ERROR [http_cmd]: ERROR 17:50:57 DEBUG 17:50:57 DEBUG --- status code --- 17:50:57 DEBUG ] 17:50:57 DEBUG ] 17:50:57 DEBUG --- http response --- 17:50:57 DEBUG ] 17:50:57 DEBUG --- http response --- 17:50:57 DEBUG ] 17:50:57 DEBUG [...] 17:50:57 DEBUG "value": [ 17:50:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734457857" 17:50:57 DEBUG 17:50:57 INFO Thread started 17:50:57 DEBUG "value": [ 17:50:57 ERROR --- status code --- 17:50:57 DEBUG 17:50:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:57 DEBUG } 17:50:57 INFO [http_cmd]: http status code OK 17:50:57 DEBUG } 17:50:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:50:57 DEBUG } 17:50:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:50:57 DEBUG } 17:50:57 DEBUG 17:50:57 DEBUG 1734457857, 17:50:57 DEBUG [...] 17:50:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734457857" 17:50:57 DEBUG 1734457857, 17:50:57 INFO Thread started 17:50:57 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:50:57 DEBUG 17:50:57 DEBUG --- http response --- 17:50:57 DEBUG ] 17:50:57 DEBUG --- status code --- 17:50:57 DEBUG } 17:50:57 DEBUG ] 17:50:57 INFO [http_cmd]: http status code OK 17:50:57 DEBUG ] 17:50:57 DEBUG 17:50:57 DEBUG "0" 17:50:57 DEBUG 17:50:57 DEBUG "80688.6" 17:50:57 ERROR --- http response --- 17:50:57 DEBUG [...] 17:50:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734457857" 17:50:57 INFO Thread started 17:50:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:50:57 DEBUG } 17:50:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:57 DEBUG } 17:50:57 DEBUG --- status code --- 17:50:57 INFO [http_cmd]: http status code OK 17:50:57 INFO [http_cmd]: http status code OK 17:50:57 DEBUG } 17:50:57 DEBUG 17:50:57 DEBUG ] 17:50:57 DEBUG 17:50:57 DEBUG ] 17:50:57 DEBUG 17:50:57 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:50:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734457857" 17:50:57 DEBUG } 17:50:57 INFO Thread started 17:50:57 DEBUG --- http response --- 17:50:57 DEBUG } 17:50:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:57 DEBUG --- status code --- 17:50:57 DEBUG --- status code --- 17:50:57 DEBUG } 17:50:57 DEBUG "instance": "10.142.0.51:9100" 17:50:57 DEBUG } 17:50:57 DEBUG 17:50:57 DEBUG } 17:50:57 DEBUG 17:50:57 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:50:57 INFO All threads has been started 127.0.0.1 - - [17/Dec/2024 17:50:57] "GET /monitoring/start HTTP/1.1" 200 - 17:50:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:50:57 DEBUG --- http response --- 17:50:57 INFO [http_cmd]: http status code OK 17:50:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:57 DEBUG }, 17:50:57 DEBUG ] 17:50:57 DEBUG "instance": "10.142.0.51:9100" 17:50:57 DEBUG ] 17:50:57 DEBUG 17:50:57 DEBUG ----- output ----- 17:50:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:50:57 DEBUG --- status code --- 17:50:57 DEBUG --- http response --- 17:50:57 DEBUG --- http response --- 17:50:57 DEBUG "value": [ 17:50:57 DEBUG } 17:50:57 INFO [http_cmd]: http status code OK 17:50:57 DEBUG }, 17:50:58 DEBUG } 17:50:58 INFO [http_cmd]: http status code OK 17:50:58 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 17:50:58 DEBUG { 17:50:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:50:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:50:58 DEBUG 1734457857, 17:50:58 DEBUG } 17:50:58 DEBUG --- status code --- 17:50:58 DEBUG "value": [ 17:50:58 DEBUG } 17:50:58 DEBUG --- status code --- 17:50:58 DEBUG }, 17:50:58 DEBUG "status": "success", 17:50:58 DEBUG --- http response --- 17:50:58 DEBUG "0" 17:50:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:58 DEBUG 1734457857, 17:50:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:58 DEBUG "value": [ 17:50:58 DEBUG "data": { 17:50:58 DEBUG ] 17:50:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:50:58 DEBUG --- http response --- 17:50:58 DEBUG "78756.76666666666" 17:50:58 DEBUG --- http response --- 17:50:58 DEBUG 1734457857, 17:50:58 DEBUG "resultType": "vector", 17:50:58 DEBUG } 17:50:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:50:58 DEBUG ] 17:50:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:50:58 DEBUG "2032439296" 17:50:58 DEBUG "result": [ 17:50:58 DEBUG ] 17:50:58 DEBUG } 17:50:58 DEBUG ----- output ----- 17:50:58 DEBUG ] 17:50:58 DEBUG { 17:50:58 DEBUG } 17:50:58 DEBUG ] 17:50:58 DEBUG { 17:50:58 DEBUG } 17:50:58 DEBUG "metric": { 17:50:58 DEBUG } 17:50:58 DEBUG } 17:50:58 DEBUG "status": "success", 17:50:58 DEBUG ] 17:50:58 DEBUG "instance": "10.142.0.11:9100" 17:50:58 DEBUG } 17:50:58 DEBUG "data": { 17:50:58 DEBUG "resultType": "vector", 17:50:58 DEBUG }, 17:50:58 DEBUG "value": [ 17:50:58 DEBUG "result": [ 17:50:58 DEBUG } 17:50:58 DEBUG 17:50:58 DEBUG { 17:50:58 DEBUG } 17:50:58 DEBUG 17:50:58 DEBUG "metric": { 17:50:58 DEBUG 17:50:58 DEBUG "instance": "10.142.0.11:9100" 17:50:58 DEBUG [...] 17:50:58 DEBUG }, 17:50:58 DEBUG 17:50:58 DEBUG "value": [ 17:50:58 DEBUG 17:50:58 DEBUG 17:50:58 DEBUG 17:50:58 DEBUG 17:50:58 DEBUG "instance": "10.142.0.51:9100" 17:50:58 DEBUG 17:50:58 DEBUG }, 17:50:58 DEBUG [...] 17:50:58 DEBUG "value": [ 17:50:58 DEBUG 17:50:58 DEBUG 1734457857, 17:50:58 DEBUG 17:50:58 DEBUG "20206.933333333334" 17:50:58 DEBUG 17:50:58 DEBUG ] 17:50:58 DEBUG "instance": "10.142.0.51:9100" 17:50:58 DEBUG } 17:50:58 DEBUG }, 17:50:58 DEBUG ] 17:50:58 DEBUG "value": [ 17:50:58 DEBUG } 17:50:58 DEBUG 1734457857, 17:50:58 DEBUG } 17:50:58 DEBUG "0.000366666666666049" 17:50:58 DEBUG ] 17:50:58 DEBUG } 17:50:58 DEBUG ] 17:50:58 DEBUG } 17:50:58 DEBUG } 17:51:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457917" 17:51:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457917" 17:51:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734457917" 17:51:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734457917" 17:51:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734457917" 17:51:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734457917" 17:51:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457917" 17:51:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457917" 17:51:57 INFO [http_cmd]: http status code OK 17:51:57 DEBUG --- status code --- 17:51:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:57 DEBUG --- http response --- 17:51:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:51:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734457917" 17:51:57 INFO [http_cmd]: http status code OK 17:51:57 DEBUG --- status code --- 17:51:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:57 INFO [http_cmd]: http status code OK 17:51:57 DEBUG --- http response --- 17:51:57 INFO 17:51:57 INFO [http_cmd]: http status code OK 17:51:57 INFO [http_cmd]: http status code OK 17:51:57 INFO [http_cmd]: http status code OK 17:51:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734457917" 17:51:57 DEBUG --- status code --- 17:51:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734457917" 17:51:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:51:57 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 17:51:57 DEBUG --- status code --- 17:51:57 DEBUG --- status code --- 17:51:57 DEBUG --- status code --- 17:51:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:51:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734457917" 17:51:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:57 INFO [http_cmd]: http status code OK 17:51:57 DEBUG --- http response --- 17:51:57 INFO [http_cmd]: http status code OK 17:51:57 DEBUG --- http response --- 17:51:57 DEBUG --- http response --- 17:51:57 DEBUG --- http response --- 17:51:57 DEBUG --- status code --- 17:51:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:51:57 DEBUG --- status code --- 17:51:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457917" 17:51:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:51:57 INFO 17:51:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:51:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:51:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:57 DEBUG --- http response --- 17:51:57 DEBUG --- http response --- 17:51:57 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 17:51:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:51:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:51:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:51:57 INFO [http_cmd]: http status code OK 17:51:57 DEBUG --- status code --- 17:51:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:57 DEBUG --- http response --- 17:51:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:51:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457917" 17:51:57 INFO [http_cmd]: http status code OK 17:51:57 DEBUG --- status code --- 17:51:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:57 DEBUG --- http response --- 17:51:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:51:57 INFO [http_cmd]: http status code OK 17:51:57 DEBUG --- status code --- 17:51:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:57 DEBUG --- http response --- 17:51:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:51:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457917" 17:51:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457917" 17:51:57 INFO [http_cmd]: http status code OK 17:51:57 DEBUG --- status code --- 17:51:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:57 DEBUG --- http response --- 17:51:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:51:57 INFO [http_cmd]: http status code OK 17:51:57 DEBUG --- status code --- 17:51:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:57 DEBUG --- http response --- 17:51:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:51:57 INFO [http_cmd]: http status code OK 17:51:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457917" 17:51:57 DEBUG --- status code --- 17:51:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:57 DEBUG --- http response --- 17:51:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:51:57 INFO [http_cmd]: http status code OK 17:51:57 DEBUG --- status code --- 17:51:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:57 DEBUG --- http response --- 17:51:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:51:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734457917" 17:51:57 INFO [http_cmd]: http status code OK 17:51:57 INFO [http_cmd]: http status code OK 17:51:57 DEBUG --- status code --- 17:51:57 DEBUG --- status code --- 17:51:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:57 DEBUG --- http response --- 17:51:57 DEBUG --- http response --- 17:51:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:51:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:51:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457917" 17:51:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734457917" 17:51:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457917" 17:51:57 INFO [http_cmd]: http status code OK 17:51:57 DEBUG --- status code --- 17:51:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:57 DEBUG --- http response --- 17:51:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:51:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734457917" 17:51:57 INFO [http_cmd]: http status code OK 17:51:57 DEBUG --- status code --- 17:51:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:57 DEBUG --- http response --- 17:51:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:51:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734457917" 17:51:57 INFO [http_cmd]: http status code OK 17:51:57 DEBUG --- status code --- 17:51:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:57 DEBUG --- http response --- 17:51:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:51:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457917" 17:51:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734457917" 17:51:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734457917" 17:51:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457918" 17:51:58 INFO [http_cmd]: http status code OK 17:51:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734457918" 17:51:58 INFO [http_cmd]: http status code OK 17:51:58 DEBUG --- status code --- 17:51:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457918" 17:51:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457918" 17:51:58 DEBUG --- status code --- 17:51:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734457918" 17:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734457918" 17:51:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734457918" 17:51:58 INFO [http_cmd]: http status code OK 17:51:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734457918" 17:51:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734457918" 17:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:58 INFO [http_cmd]: http status code OK 17:51:58 ERROR [http_cmd]: ERROR 17:51:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734457918" 17:51:58 INFO [http_cmd]: http status code OK 17:51:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734457918" 17:51:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734457918" 17:51:58 DEBUG --- http response --- 17:51:58 INFO [http_cmd]: http status code OK 17:51:58 DEBUG --- status code --- 17:51:58 INFO [http_cmd]: http status code OK 17:51:58 INFO [http_cmd]: http status code OK 17:51:58 DEBUG --- http response --- 17:51:58 DEBUG --- status code --- 17:51:58 ERROR --- status code --- 17:51:58 DEBUG --- status code --- 17:51:58 INFO [http_cmd]: http status code OK 17:51:58 INFO [http_cmd]: http status code OK 17:51:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:51:58 DEBUG --- status code --- 17:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:58 INFO [http_cmd]: http status code OK 17:51:58 INFO [http_cmd]: http status code OK 17:51:58 INFO [http_cmd]: http status code OK 17:51:58 DEBUG --- status code --- 17:51:58 INFO [http_cmd]: http status code OK 17:51:58 DEBUG --- status code --- 17:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 17:51:58 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:58 DEBUG --- status code --- 17:51:58 DEBUG --- status code --- 17:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:58 DEBUG --- http response --- 17:51:58 DEBUG --- status code --- 17:51:58 DEBUG --- status code --- 17:51:58 DEBUG --- status code --- 17:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:58 DEBUG --- status code --- 17:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:58 INFO [http_cmd]: http status code OK 17:51:58 DEBUG --- http response --- 17:51:58 DEBUG ----- output ----- 17:51:58 ERROR --- http response --- 17:51:58 DEBUG --- http response --- 17:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:58 DEBUG --- http response --- 17:51:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 17:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:58 DEBUG --- http response --- 17:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:58 DEBUG --- http response --- 17:51:58 DEBUG --- status code --- 17:51:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:51:58 DEBUG { 17:51:58 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:51:58 INFO [http_cmd]: http status code OK 17:51:58 INFO [http_cmd]: http status code OK 17:51:58 DEBUG --- http response --- 17:51:58 INFO [loop_until]: OK (rc = 0) 17:51:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 17:51:58 DEBUG --- http response --- 17:51:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:51:58 DEBUG ----- output ----- 17:51:58 DEBUG --- http response --- 17:51:58 DEBUG --- http response --- 17:51:58 DEBUG --- http response --- 17:51:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:51:58 DEBUG --- http response --- 17:51:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 17:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:58 DEBUG ----- output ----- 17:51:58 DEBUG "status": "success", 17:51:58 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:51:58 DEBUG --- status code --- 17:51:58 DEBUG --- status code --- 17:51:58 DEBUG --- stdout --- 17:51:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:51:58 DEBUG ----- output ----- 17:51:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:51:58 DEBUG { 17:51:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:51:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:51:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:51:58 DEBUG ----- output ----- 17:51:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:51:58 DEBUG --- http response --- 17:51:58 DEBUG { 17:51:58 DEBUG "data": { 17:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:58 DEBUG ----- output ----- 17:51:58 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 17m 393Mi ds-idrepo-0 79m 566Mi rcs-5d696ff5b-7ttvr 8m 291Mi rcs-5d696ff5b-wwlbr 16m 328Mi 17:51:58 DEBUG { 17:51:58 DEBUG "status": "success", 17:51:58 DEBUG ----- output ----- 17:51:58 DEBUG { 17:51:58 DEBUG ----- output ----- 17:51:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:51:58 DEBUG "status": "success", 17:51:58 DEBUG "resultType": "vector", 17:51:58 DEBUG --- http response --- 17:51:58 DEBUG --- http response --- 17:51:58 DEBUG { 17:51:58 DEBUG --- stderr --- 17:51:58 DEBUG "status": "success", 17:51:58 DEBUG "data": { 17:51:58 DEBUG { 17:51:58 DEBUG "status": "success", 17:51:58 DEBUG { 17:51:58 DEBUG ----- output ----- 17:51:58 DEBUG "data": { 17:51:58 DEBUG "result": [ 17:51:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:51:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:51:58 DEBUG "status": "success", 17:51:58 DEBUG 17:51:58 DEBUG "data": { 17:51:58 DEBUG "resultType": "vector", 17:51:58 DEBUG "status": "success", 17:51:58 DEBUG "data": { 17:51:58 DEBUG "status": "success", 17:51:58 DEBUG { 17:51:58 DEBUG "resultType": "vector", 17:51:58 DEBUG { 17:51:58 DEBUG ----- output ----- 17:51:58 DEBUG ----- output ----- 17:51:58 DEBUG "data": { 17:51:58 DEBUG "resultType": "vector", 17:51:58 DEBUG "result": [ 17:51:58 DEBUG "data": { 17:51:58 DEBUG "resultType": "vector", 17:51:58 DEBUG "data": { 17:51:58 DEBUG "status": "success", 17:51:58 DEBUG "result": [ 17:51:58 DEBUG "metric": { 17:51:58 DEBUG { 17:51:58 DEBUG { 17:51:58 DEBUG "resultType": "vector", 17:51:58 DEBUG "result": [ 17:51:58 DEBUG { 17:51:58 DEBUG "resultType": "vector", 17:51:58 DEBUG "result": [ 17:51:58 INFO [loop_until]: OK (rc = 0) 17:51:58 DEBUG "resultType": "vector", 17:51:58 DEBUG "data": { 17:51:58 DEBUG { 17:51:58 DEBUG "pod": "ds-cts-0" 17:51:58 DEBUG "status": "success", 17:51:58 DEBUG "status": "success", 17:51:58 DEBUG "result": [ 17:51:58 DEBUG { 17:51:58 DEBUG "metric": { 17:51:58 DEBUG "result": [ 17:51:58 DEBUG { 17:51:58 DEBUG --- stdout --- 17:51:58 DEBUG "result": [ 17:51:58 DEBUG "resultType": "vector", 17:51:58 DEBUG "metric": { 17:51:58 DEBUG }, 17:51:58 DEBUG "data": { 17:51:58 DEBUG "data": { 17:51:58 DEBUG { 17:51:58 DEBUG "metric": { 17:51:58 DEBUG "pod": "ds-cts-0" 17:51:58 DEBUG }, 17:51:58 DEBUG "metric": { 17:51:58 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 119m 0% 2262Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 2169m 6% 2572Mi 9% 17:51:58 DEBUG { 17:51:58 DEBUG "result": [ 17:51:58 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 17:51:58 DEBUG "value": [ 17:51:58 DEBUG "resultType": "vector", 17:51:58 DEBUG "resultType": "vector", 17:51:58 DEBUG "metric": { 17:51:58 DEBUG "pod": "ds-cts-0" 17:51:58 DEBUG { 17:51:58 DEBUG "value": [ 17:51:58 DEBUG "pod": "ds-cts-0" 17:51:58 DEBUG --- stderr --- 17:51:58 DEBUG "metric": { 17:51:58 DEBUG { 17:51:58 DEBUG }, 17:51:58 DEBUG 17:51:58 DEBUG "result": [ 17:51:58 DEBUG "result": [ 17:51:58 DEBUG "instance": "10.142.0.11:9100" 17:51:58 DEBUG }, 17:51:58 DEBUG "metric": { 17:51:58 DEBUG 17:51:58 DEBUG }, 17:51:58 DEBUG 17:51:58 DEBUG "instance": "10.142.0.11:9100" 17:51:58 DEBUG "metric": { 17:51:58 DEBUG "value": [ 17:51:58 DEBUG 17:51:58 DEBUG { 17:51:58 DEBUG { 17:51:58 DEBUG }, 17:51:58 DEBUG "value": [ 17:51:58 DEBUG "instance": "10.142.0.11:9100" 17:51:58 DEBUG 17:51:58 DEBUG "value": [ 17:51:58 DEBUG }, 17:51:58 DEBUG "instance": "10.142.0.11:9100" 17:51:58 DEBUG 17:51:58 DEBUG 17:51:58 DEBUG "metric": { 17:51:58 DEBUG "metric": { 17:51:58 DEBUG "value": [ 17:51:58 DEBUG 17:51:58 DEBUG }, 17:51:58 DEBUG 17:51:58 DEBUG 17:51:58 DEBUG "value": [ 17:51:58 DEBUG }, 17:51:58 DEBUG 17:51:58 DEBUG [...] 17:51:58 DEBUG "instance": "10.142.0.11:9100" 17:51:58 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 17:51:58 DEBUG 17:51:58 DEBUG 17:51:58 DEBUG "value": [ 17:51:58 DEBUG [...] 17:51:58 DEBUG 17:51:58 DEBUG 17:51:58 DEBUG "value": [ 17:51:58 DEBUG 17:51:58 DEBUG 17:51:58 DEBUG }, 17:51:58 DEBUG }, 17:51:58 DEBUG 17:51:58 DEBUG 17:51:58 DEBUG 17:51:58 DEBUG 17:51:58 DEBUG 17:51:58 DEBUG 17:51:58 DEBUG 17:51:58 DEBUG [...] 17:51:58 DEBUG 17:51:58 DEBUG "value": [ 17:51:58 DEBUG "value": [ 17:51:58 DEBUG 17:51:58 DEBUG [...] 17:51:58 DEBUG 17:51:58 DEBUG 17:51:58 DEBUG [...] 17:51:58 DEBUG 17:51:58 DEBUG 17:51:58 DEBUG 17:51:58 DEBUG 17:51:58 DEBUG 17:51:58 DEBUG 17:51:58 DEBUG [...] 17:51:58 DEBUG 17:51:58 DEBUG 17:51:58 DEBUG 17:51:58 DEBUG 17:51:58 DEBUG [...] 17:51:58 DEBUG 17:51:58 DEBUG 17:51:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 17:51:58 DEBUG 17:51:58 DEBUG 17:51:58 DEBUG 17:51:58 DEBUG 17:51:58 DEBUG [...] 17:51:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 17:51:58 DEBUG 17:51:58 DEBUG 17:51:58 DEBUG [...] 17:51:58 DEBUG 17:51:58 DEBUG }, 17:51:58 DEBUG 17:51:58 DEBUG 17:51:58 DEBUG 17:51:58 DEBUG 17:51:58 DEBUG 17:51:58 DEBUG }, 17:51:58 DEBUG 17:51:58 DEBUG 17:51:58 DEBUG 17:51:58 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 17:51:58 DEBUG "value": [ 17:51:58 DEBUG [...] 17:51:58 DEBUG [...] 17:51:58 DEBUG 17:51:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 17:51:58 DEBUG 17:51:58 DEBUG "value": [ 17:51:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 17:51:58 DEBUG 17:51:58 DEBUG 17:51:58 DEBUG }, 17:51:58 DEBUG 1734457917, 17:51:58 DEBUG 17:51:58 DEBUG 17:51:58 DEBUG "instance": "10.142.0.51:9100" 17:51:58 DEBUG }, 17:51:58 DEBUG 17:51:58 DEBUG 1734457917, 17:51:58 DEBUG }, 17:51:58 DEBUG "instance": "10.142.0.51:9100" 17:51:58 DEBUG 17:51:58 DEBUG "value": [ 17:51:58 DEBUG "0.016825355148641672" 17:51:58 DEBUG 17:51:58 DEBUG 17:51:58 DEBUG }, 17:51:58 DEBUG "value": [ 17:51:58 DEBUG "instance": "10.142.0.51:9100" 17:51:58 DEBUG "689065984" 17:51:58 DEBUG "value": [ 17:51:58 DEBUG }, 17:51:58 DEBUG "instance": "10.142.0.51:9100" 17:51:58 DEBUG 1734457917, 17:51:58 DEBUG ] 17:51:58 DEBUG 17:51:58 DEBUG 17:51:58 DEBUG "value": [ 17:51:58 DEBUG 1734457917, 17:51:58 DEBUG }, 17:51:58 DEBUG ] 17:51:58 DEBUG 1734457918, 17:51:58 DEBUG "value": [ 17:51:58 DEBUG }, 17:51:58 DEBUG "3.0864150624854423" 17:51:58 DEBUG } 17:51:58 DEBUG "instance": "10.142.0.51:9100" 17:51:58 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 17:51:58 DEBUG 1734457918, 17:51:58 DEBUG "1563.381805695809" 17:51:58 DEBUG "value": [ 17:51:58 DEBUG } 17:51:58 DEBUG "0" 17:51:58 DEBUG 1734457918, 17:51:58 DEBUG "value": [ 17:51:58 DEBUG ] 17:51:58 DEBUG ] 17:51:58 DEBUG }, 17:51:58 DEBUG }, 17:51:58 DEBUG "91560.13333333333" 17:51:58 DEBUG ] 17:51:58 DEBUG 1734457918, 17:51:58 DEBUG ] 17:51:58 DEBUG ] 17:51:58 DEBUG "87774.06666666665" 17:51:58 DEBUG 1734457918, 17:51:58 DEBUG } 17:51:58 DEBUG } 17:51:58 DEBUG "value": [ 17:51:58 DEBUG "value": [ 17:51:58 DEBUG ] 17:51:58 DEBUG } 17:51:58 DEBUG "0" 17:51:58 DEBUG } 17:51:58 DEBUG } 17:51:58 DEBUG ] 17:51:58 DEBUG "0.00033333333333160906" 17:51:58 DEBUG ] 17:51:58 DEBUG } 17:51:58 DEBUG 1734457918, 17:51:58 DEBUG 1734457918, 17:51:58 DEBUG } 17:51:58 DEBUG ] 17:51:58 DEBUG ] 17:51:58 DEBUG } 17:51:58 DEBUG ] 17:51:58 DEBUG } 17:51:58 DEBUG ] 17:51:58 DEBUG } 17:51:58 DEBUG "8055.466666666666" 17:51:58 DEBUG "2153807872" 17:51:58 DEBUG ] 17:51:58 DEBUG } 17:51:58 DEBUG } 17:51:58 DEBUG } 17:51:58 DEBUG ] 17:51:58 DEBUG } 17:51:58 DEBUG } 17:51:58 DEBUG ] 17:51:58 DEBUG ] 17:51:58 DEBUG } 17:51:58 DEBUG } 17:51:58 DEBUG ] 17:51:58 DEBUG } 17:51:58 DEBUG } 17:51:58 DEBUG } 17:51:58 DEBUG } 17:51:58 DEBUG } 17:51:58 DEBUG } 17:51:58 DEBUG } 17:51:58 DEBUG ] 17:51:58 DEBUG ] 17:51:58 DEBUG ] 17:51:58 DEBUG } 17:51:58 DEBUG } 17:51:58 DEBUG } 17:51:58 DEBUG } 17:51:58 DEBUG } 17:51:58 DEBUG } 17:51:58 DEBUG } 17:52:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457977" 17:52:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734457977" 17:52:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734457977" 17:52:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734457977" 17:52:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457977" 17:52:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457977" 17:52:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457977" 17:52:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734457977" 17:52:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734457977" 17:52:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734457977" 17:52:57 INFO [http_cmd]: http status code OK 17:52:57 DEBUG --- status code --- 17:52:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:57 DEBUG --- http response --- 17:52:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:52:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734457977" 17:52:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457977" 17:52:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734457977" 17:52:57 INFO [http_cmd]: http status code OK 17:52:57 DEBUG --- status code --- 17:52:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:57 DEBUG --- http response --- 17:52:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:52:57 INFO [http_cmd]: http status code OK 17:52:57 DEBUG --- status code --- 17:52:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:57 DEBUG --- http response --- 17:52:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:52:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457977" 17:52:57 INFO [http_cmd]: http status code OK 17:52:57 DEBUG --- status code --- 17:52:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:57 DEBUG --- http response --- 17:52:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:52:57 INFO [http_cmd]: http status code OK 17:52:57 INFO [http_cmd]: http status code OK 17:52:57 INFO [http_cmd]: http status code OK 17:52:57 INFO [http_cmd]: http status code OK 17:52:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457977" 17:52:57 INFO [http_cmd]: http status code OK 17:52:57 DEBUG --- status code --- 17:52:57 INFO [http_cmd]: http status code OK 17:52:57 INFO [http_cmd]: http status code OK 17:52:57 INFO [http_cmd]: http status code OK 17:52:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457977" 17:52:57 DEBUG --- status code --- 17:52:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457977" 17:52:57 DEBUG --- status code --- 17:52:57 DEBUG --- status code --- 17:52:57 INFO [http_cmd]: http status code OK 17:52:57 DEBUG --- status code --- 17:52:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:57 DEBUG --- status code --- 17:52:57 DEBUG --- status code --- 17:52:57 DEBUG --- status code --- 17:52:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734457977" 17:52:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:57 DEBUG --- status code --- 17:52:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:57 DEBUG --- http response --- 17:52:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:57 INFO [http_cmd]: http status code OK 17:52:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:57 DEBUG --- http response --- 17:52:57 DEBUG --- http response --- 17:52:57 DEBUG --- http response --- 17:52:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:57 DEBUG --- http response --- 17:52:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:52:57 DEBUG --- http response --- 17:52:57 DEBUG --- status code --- 17:52:57 DEBUG --- http response --- 17:52:57 DEBUG --- http response --- 17:52:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:52:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:52:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:52:57 DEBUG --- http response --- 17:52:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:52:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:52:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:52:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:52:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:52:57 INFO [http_cmd]: http status code OK 17:52:57 DEBUG --- http response --- 17:52:57 INFO [http_cmd]: http status code OK 17:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457978" 17:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734457978" 17:52:58 INFO [http_cmd]: http status code OK 17:52:58 DEBUG --- status code --- 17:52:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:52:58 DEBUG --- status code --- 17:52:58 INFO [http_cmd]: http status code OK 17:52:58 DEBUG --- status code --- 17:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:58 DEBUG --- status code --- 17:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:58 DEBUG --- http response --- 17:52:58 DEBUG --- http response --- 17:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:58 DEBUG --- http response --- 17:52:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:52:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:52:58 DEBUG --- http response --- 17:52:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:52:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:52:58 INFO [http_cmd]: http status code OK 17:52:58 DEBUG --- status code --- 17:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:58 DEBUG --- http response --- 17:52:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:52:58 INFO [http_cmd]: http status code OK 17:52:58 DEBUG --- status code --- 17:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:58 DEBUG --- http response --- 17:52:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734457978" 17:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734457978" 17:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457978" 17:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734457978" 17:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457978" 17:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734457978" 17:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734457978" 17:52:58 INFO [http_cmd]: http status code OK 17:52:58 INFO 17:52:58 DEBUG --- status code --- 17:52:58 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 17:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:52:58 DEBUG --- http response --- 17:52:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:52:58 INFO [http_cmd]: http status code OK 17:52:58 DEBUG --- status code --- 17:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:58 DEBUG --- http response --- 17:52:58 INFO [http_cmd]: http status code OK 17:52:58 ERROR [http_cmd]: ERROR 17:52:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:52:58 DEBUG --- status code --- 17:52:58 ERROR --- status code --- 17:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:58 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:52:58 INFO [http_cmd]: http status code OK 17:52:58 INFO [http_cmd]: http status code OK 17:52:58 INFO 17:52:58 DEBUG --- http response --- 17:52:58 ERROR --- http response --- 17:52:58 DEBUG --- status code --- 17:52:58 DEBUG --- status code --- 17:52:58 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 17:52:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:52:58 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:58 DEBUG --- http response --- 17:52:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:52:58 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:52:58 DEBUG --- http response --- 17:52:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:52:58 INFO [http_cmd]: http status code OK 17:52:58 DEBUG --- status code --- 17:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:58 DEBUG --- http response --- 17:52:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457978" 17:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734457978" 17:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734457978" 17:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734457978" 17:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734457978" 17:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457978" 17:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734457978" 17:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734457978" 17:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734457978" 17:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734457978" 17:52:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734457978" 17:52:58 INFO [http_cmd]: http status code OK 17:52:58 DEBUG --- status code --- 17:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:58 DEBUG --- http response --- 17:52:58 INFO [http_cmd]: http status code OK 17:52:58 DEBUG --- status code --- 17:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:58 DEBUG --- http response --- 17:52:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 17:52:58 DEBUG ----- output ----- 17:52:58 DEBUG { 17:52:58 DEBUG "status": "success", 17:52:58 DEBUG "data": { 17:52:58 DEBUG "resultType": "vector", 17:52:58 DEBUG "result": [ 17:52:58 DEBUG { 17:52:58 DEBUG "metric": { 17:52:58 INFO [http_cmd]: http status code OK 17:52:58 INFO [http_cmd]: http status code OK 17:52:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:52:58 DEBUG "pod": "ds-cts-0" 17:52:58 INFO [http_cmd]: http status code OK 17:52:58 DEBUG --- status code --- 17:52:58 DEBUG --- status code --- 17:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:58 DEBUG --- http response --- 17:52:58 DEBUG --- status code --- 17:52:58 INFO [http_cmd]: http status code OK 17:52:58 INFO [http_cmd]: http status code OK 17:52:58 INFO [http_cmd]: http status code OK 17:52:58 INFO [http_cmd]: http status code OK 17:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:58 DEBUG ----- output ----- 17:52:58 DEBUG }, 17:52:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:58 DEBUG --- status code --- 17:52:58 DEBUG --- status code --- 17:52:58 DEBUG --- status code --- 17:52:58 DEBUG --- status code --- 17:52:58 DEBUG --- http response --- 17:52:58 DEBUG { 17:52:58 INFO [http_cmd]: http status code OK 17:52:58 DEBUG "value": [ 17:52:58 DEBUG ----- output ----- 17:52:58 DEBUG --- http response --- 17:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 17:52:58 DEBUG ----- output ----- 17:52:58 DEBUG { 17:52:58 DEBUG "status": "success", 17:52:58 DEBUG { 17:52:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:52:58 DEBUG --- http response --- 17:52:58 INFO [http_cmd]: http status code OK 17:52:58 DEBUG --- http response --- 17:52:58 DEBUG --- http response --- 17:52:58 DEBUG --- http response --- 17:52:58 DEBUG "status": "success", 17:52:58 DEBUG --- status code --- 17:52:58 DEBUG 17:52:58 DEBUG "data": { 17:52:58 DEBUG "status": "success", 17:52:58 DEBUG ----- output ----- 17:52:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:52:58 DEBUG --- status code --- 17:52:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:52:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 17:52:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 17:52:58 DEBUG "data": { 17:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:58 DEBUG 17:52:58 DEBUG "resultType": "vector", 17:52:58 DEBUG "data": { 17:52:58 DEBUG { 17:52:58 DEBUG ----- output ----- 17:52:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:58 DEBUG ----- output ----- 17:52:58 DEBUG ----- output ----- 17:52:58 DEBUG ----- output ----- 17:52:58 DEBUG "resultType": "vector", 17:52:58 DEBUG --- http response --- 17:52:58 DEBUG 17:52:58 DEBUG "result": [ 17:52:58 DEBUG "resultType": "vector", 17:52:58 DEBUG "status": "success", 17:52:58 DEBUG { 17:52:58 DEBUG --- http response --- 17:52:58 DEBUG { 17:52:58 DEBUG { 17:52:58 DEBUG { 17:52:58 DEBUG "result": [ 17:52:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:52:58 DEBUG [...] 17:52:58 DEBUG { 17:52:58 DEBUG "result": [ 17:52:58 DEBUG "data": { 17:52:58 DEBUG "status": "success", 17:52:58 DEBUG "status": "success", 17:52:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:52:58 DEBUG "status": "success", 17:52:58 DEBUG "status": "success", 17:52:58 DEBUG { 17:52:58 DEBUG ----- output ----- 17:52:58 DEBUG 17:52:58 DEBUG "metric": { 17:52:58 DEBUG { 17:52:58 DEBUG "resultType": "vector", 17:52:58 DEBUG "data": { 17:52:58 DEBUG "data": { 17:52:58 DEBUG ----- output ----- 17:52:58 DEBUG "data": { 17:52:58 DEBUG "data": { 17:52:58 DEBUG "metric": { 17:52:58 DEBUG { 17:52:58 DEBUG 17:52:58 DEBUG "pod": "ds-cts-0" 17:52:58 DEBUG "metric": { 17:52:58 DEBUG "result": [ 17:52:58 DEBUG "resultType": "vector", 17:52:58 DEBUG "resultType": "vector", 17:52:58 DEBUG { 17:52:58 DEBUG "resultType": "vector", 17:52:58 DEBUG "resultType": "vector", 17:52:58 DEBUG "instance": "10.142.0.11:9100" 17:52:58 DEBUG "status": "success", 17:52:58 DEBUG 17:52:58 DEBUG }, 17:52:58 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 17:52:58 DEBUG { 17:52:58 DEBUG "result": [ 17:52:58 DEBUG "result": [ 17:52:58 DEBUG "status": "success", 17:52:58 DEBUG "result": [ 17:52:58 DEBUG "result": [ 17:52:58 DEBUG }, 17:52:58 DEBUG "data": { 17:52:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 17:52:58 DEBUG "value": [ 17:52:58 DEBUG }, 17:52:58 DEBUG "metric": { 17:52:58 DEBUG { 17:52:58 DEBUG { 17:52:58 DEBUG "data": { 17:52:58 DEBUG { 17:52:58 DEBUG { 17:52:58 DEBUG "value": [ 17:52:58 DEBUG "resultType": "vector", 17:52:58 DEBUG }, 17:52:58 DEBUG 17:52:58 DEBUG "value": [ 17:52:58 DEBUG "instance": "10.142.0.11:9100" 17:52:58 DEBUG "metric": { 17:52:58 DEBUG "metric": { 17:52:58 DEBUG "resultType": "vector", 17:52:58 DEBUG "metric": { 17:52:58 DEBUG "metric": { 17:52:58 DEBUG 17:52:58 DEBUG "result": [ 17:52:58 DEBUG "value": [ 17:52:58 DEBUG 17:52:58 DEBUG 17:52:58 DEBUG }, 17:52:58 DEBUG "instance": "10.142.0.11:9100" 17:52:58 DEBUG "instance": "10.142.0.11:9100" 17:52:58 DEBUG "result": [ 17:52:58 DEBUG "pod": "ds-cts-0" 17:52:58 DEBUG "pod": "ds-cts-0" 17:52:58 DEBUG 17:52:58 DEBUG { 17:52:58 DEBUG 1734457978, 17:52:58 DEBUG 17:52:58 DEBUG 17:52:58 DEBUG "value": [ 17:52:58 DEBUG }, 17:52:58 DEBUG }, 17:52:58 DEBUG { 17:52:58 DEBUG }, 17:52:58 DEBUG }, 17:52:58 DEBUG 17:52:58 DEBUG "metric": { 17:52:58 DEBUG "689303552" 17:52:58 DEBUG [...] 17:52:58 DEBUG 17:52:58 DEBUG 17:52:58 DEBUG "value": [ 17:52:58 DEBUG "value": [ 17:52:58 DEBUG 17:52:58 DEBUG "value": [ 17:52:58 DEBUG "value": [ 17:52:58 DEBUG [...] 17:52:58 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 17:52:58 DEBUG ] 17:52:58 DEBUG 17:52:58 DEBUG [...] 17:52:58 DEBUG 17:52:58 DEBUG 17:52:58 DEBUG [...] 17:52:58 DEBUG 17:52:58 DEBUG 17:52:58 DEBUG 17:52:58 DEBUG 17:52:58 DEBUG }, 17:52:58 DEBUG } 17:52:58 DEBUG 17:52:58 DEBUG 17:52:58 DEBUG 17:52:58 DEBUG "metric": { 17:52:58 DEBUG 17:52:58 DEBUG 17:52:58 DEBUG 17:52:58 DEBUG 17:52:58 DEBUG 17:52:58 DEBUG "value": [ 17:52:58 DEBUG ] 17:52:58 DEBUG 17:52:58 DEBUG 17:52:58 DEBUG 17:52:58 DEBUG 17:52:58 DEBUG [...] 17:52:58 DEBUG 17:52:58 DEBUG [...] 17:52:58 DEBUG 17:52:58 DEBUG [...] 17:52:58 DEBUG 17:52:58 DEBUG } 17:52:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 17:52:58 DEBUG 17:52:58 DEBUG 17:52:58 DEBUG 17:52:58 DEBUG 17:52:58 DEBUG 17:52:58 DEBUG } 17:52:58 DEBUG }, 17:52:58 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 17:52:58 DEBUG }, 17:52:58 DEBUG 17:52:58 DEBUG 17:52:58 DEBUG 17:52:58 DEBUG "value": [ 17:52:58 DEBUG [...] 17:52:58 DEBUG 17:52:58 DEBUG 17:52:58 DEBUG 17:52:58 DEBUG "instance": "10.142.0.11:9100" 17:52:58 DEBUG "value": [ 17:52:58 DEBUG "instance": "10.142.0.51:9100" 17:52:58 DEBUG 17:52:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 17:52:58 DEBUG 1734457978, 17:52:58 DEBUG 17:52:58 DEBUG 17:52:58 DEBUG 17:52:58 DEBUG "instance": "10.142.0.51:9100" 17:52:58 DEBUG }, 17:52:58 DEBUG 1734457978, 17:52:58 DEBUG }, 17:52:58 DEBUG "instance": "10.142.0.51:9100" 17:52:58 DEBUG }, 17:52:58 DEBUG "0.0069497118079431775" 17:52:58 DEBUG 17:52:58 DEBUG "instance": "10.142.0.51:9100" 17:52:58 DEBUG 17:52:58 DEBUG }, 17:52:58 DEBUG "value": [ 17:52:58 DEBUG "0.5133526526066697" 17:52:58 DEBUG "value": [ 17:52:58 DEBUG }, 17:52:58 DEBUG "value": [ 17:52:58 DEBUG ] 17:52:58 DEBUG 17:52:58 DEBUG }, 17:52:58 DEBUG [...] 17:52:58 DEBUG 17:52:58 DEBUG 17:52:58 DEBUG 17:52:58 DEBUG ] 17:52:58 DEBUG "value": [ 17:52:58 DEBUG 1734457978, 17:52:58 DEBUG } 17:52:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 17:52:58 DEBUG "value": [ 17:52:58 DEBUG "value": [ 17:52:58 DEBUG 17:52:58 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 17:52:58 DEBUG }, 17:52:58 DEBUG "value": [ 17:52:58 DEBUG 1734457978, 17:52:58 DEBUG "2258681856" 17:52:58 DEBUG ] 17:52:58 DEBUG } 17:52:58 DEBUG }, 17:52:58 DEBUG 1734457978, 17:52:58 DEBUG 17:52:58 DEBUG 1734457978, 17:52:58 DEBUG } 17:52:58 DEBUG 1734457978, 17:52:58 DEBUG "0" 17:52:58 DEBUG ] 17:52:58 DEBUG 1734457978, 17:52:58 DEBUG ] 17:52:58 DEBUG "value": [ 17:52:58 DEBUG "125952.96666666666" 17:52:58 DEBUG 17:52:58 DEBUG "0.00046666666666557904" 17:52:58 DEBUG ] 17:52:58 DEBUG "0" 17:52:58 DEBUG ] 17:52:58 DEBUG } 17:52:58 DEBUG "122490.56666666668" 17:52:58 DEBUG } 17:52:58 DEBUG } 17:52:58 DEBUG ] 17:52:58 DEBUG [...] 17:52:58 DEBUG ] 17:52:58 DEBUG } 17:52:58 DEBUG ] 17:52:58 DEBUG } 17:52:58 DEBUG } 17:52:58 DEBUG ] 17:52:58 DEBUG 1734457978, 17:52:58 DEBUG } 17:52:58 DEBUG 17:52:58 DEBUG } 17:52:58 DEBUG } 17:52:58 DEBUG } 17:52:58 DEBUG ] 17:52:58 DEBUG } 17:52:58 DEBUG "1737.988726548179" 17:52:58 DEBUG ] 17:52:58 DEBUG 17:52:58 DEBUG ] 17:52:58 DEBUG } 17:52:58 DEBUG ] 17:52:58 DEBUG ] 17:52:58 DEBUG ] 17:52:58 DEBUG } 17:52:58 DEBUG 17:52:58 DEBUG } 17:52:58 DEBUG } 17:52:58 DEBUG } 17:52:58 DEBUG } 17:52:58 DEBUG } 17:52:58 DEBUG } 17:52:58 DEBUG "instance": "10.142.0.51:9100" 17:52:58 DEBUG } 17:52:58 DEBUG } 17:52:58 DEBUG ] 17:52:58 DEBUG }, 17:52:58 DEBUG } 17:52:58 DEBUG } 17:52:58 DEBUG "value": [ 17:52:58 INFO [loop_until]: OK (rc = 0) 17:52:58 DEBUG } 17:52:58 DEBUG 1734457978, 17:52:58 DEBUG --- stdout --- 17:52:58 DEBUG "27989.333333333332" 17:52:58 DEBUG ] 17:52:58 DEBUG } 17:52:58 DEBUG ] 17:52:58 DEBUG } 17:52:58 DEBUG } 17:52:58 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 8m 397Mi ds-idrepo-0 17m 571Mi rcs-5d696ff5b-7ttvr 2m 291Mi rcs-5d696ff5b-wwlbr 4m 328Mi 17:52:58 DEBUG --- stderr --- 17:52:58 DEBUG 17:52:58 INFO [loop_until]: OK (rc = 0) 17:52:58 DEBUG --- stdout --- 17:52:58 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 107m 0% 2258Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1152m 3% 2792Mi 9% 17:52:58 DEBUG --- stderr --- 17:52:58 DEBUG 17:53:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458037" 17:53:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458037" 17:53:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458037" 17:53:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458037" 17:53:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458037" 17:53:57 INFO [http_cmd]: http status code OK 17:53:57 DEBUG --- status code --- 17:53:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:57 DEBUG --- http response --- 17:53:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:53:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458037" 17:53:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458038" 17:53:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458038" 17:53:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458038" 17:53:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458038" 17:53:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458038" 17:53:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458038" 17:53:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458038" 17:53:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734458038" 17:53:58 INFO [http_cmd]: http status code OK 17:53:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458038" 17:53:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458038" 17:53:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734458038" 17:53:58 INFO [http_cmd]: http status code OK 17:53:58 DEBUG --- status code --- 17:53:58 INFO [http_cmd]: http status code OK 17:53:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458038" 17:53:58 INFO [http_cmd]: http status code OK 17:53:58 INFO [http_cmd]: http status code OK 17:53:58 INFO [http_cmd]: http status code OK 17:53:58 INFO [http_cmd]: http status code OK 17:53:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734458038" 17:53:58 INFO [http_cmd]: http status code OK 17:53:58 INFO [http_cmd]: http status code OK 17:53:58 INFO [http_cmd]: http status code OK 17:53:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458038" 17:53:58 DEBUG --- status code --- 17:53:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:58 DEBUG --- status code --- 17:53:58 DEBUG --- status code --- 17:53:58 INFO [http_cmd]: http status code OK 17:53:58 INFO [http_cmd]: http status code OK 17:53:58 INFO [http_cmd]: http status code OK 17:53:58 DEBUG --- status code --- 17:53:58 DEBUG --- status code --- 17:53:58 DEBUG --- status code --- 17:53:58 DEBUG --- status code --- 17:53:58 INFO [http_cmd]: http status code OK 17:53:58 DEBUG --- status code --- 17:53:58 DEBUG --- status code --- 17:53:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:58 INFO [http_cmd]: http status code OK 17:53:58 DEBUG --- http response --- 17:53:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:58 DEBUG --- status code --- 17:53:58 DEBUG --- status code --- 17:53:58 DEBUG --- status code --- 17:53:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:58 DEBUG --- status code --- 17:53:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:58 DEBUG --- http response --- 17:53:58 DEBUG --- status code --- 17:53:58 INFO [http_cmd]: http status code OK 17:53:58 INFO [http_cmd]: http status code OK 17:53:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:53:58 DEBUG --- http response --- 17:53:58 DEBUG --- http response --- 17:53:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:58 DEBUG --- http response --- 17:53:58 DEBUG --- http response --- 17:53:58 DEBUG --- http response --- 17:53:58 DEBUG --- http response --- 17:53:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:58 DEBUG --- http response --- 17:53:58 DEBUG --- http response --- 17:53:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:53:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:58 DEBUG --- status code --- 17:53:58 DEBUG --- status code --- 17:53:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:53:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:53:58 DEBUG --- http response --- 17:53:58 DEBUG --- http response --- 17:53:58 DEBUG --- http response --- 17:53:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:53:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:53:58 INFO [http_cmd]: http status code OK 17:53:58 INFO [http_cmd]: http status code OK 17:53:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:53:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:53:58 DEBUG --- http response --- 17:53:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:53:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:53:58 DEBUG --- http response --- 17:53:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:53:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:53:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:53:58 DEBUG --- status code --- 17:53:58 DEBUG --- status code --- 17:53:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:53:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:53:58 DEBUG --- http response --- 17:53:58 DEBUG --- http response --- 17:53:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:53:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:53:58 DEBUG --- http response --- 17:53:58 DEBUG --- http response --- 17:53:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:53:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:53:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734458038" 17:53:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458038" 17:53:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458038" 17:53:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734458038" 17:53:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458038" 17:53:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458038" 17:53:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458038" 17:53:58 INFO [http_cmd]: http status code OK 17:53:58 DEBUG --- status code --- 17:53:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:58 DEBUG --- http response --- 17:53:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:53:58 INFO [http_cmd]: http status code OK 17:53:58 INFO [http_cmd]: http status code OK 17:53:58 DEBUG --- status code --- 17:53:58 DEBUG --- status code --- 17:53:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:58 DEBUG --- http response --- 17:53:58 DEBUG --- http response --- 17:53:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:53:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:53:58 INFO [http_cmd]: http status code OK 17:53:58 DEBUG --- status code --- 17:53:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:58 DEBUG --- http response --- 17:53:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:53:58 INFO [http_cmd]: http status code OK 17:53:58 INFO [http_cmd]: http status code OK 17:53:58 ERROR [http_cmd]: ERROR 17:53:58 DEBUG --- status code --- 17:53:58 DEBUG --- status code --- 17:53:58 ERROR --- status code --- 17:53:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:58 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:53:58 DEBUG --- http response --- 17:53:58 DEBUG --- http response --- 17:53:58 ERROR --- http response --- 17:53:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:53:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:53:58 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:53:58 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:53:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458038" 17:53:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734458038" 17:53:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458038" 17:53:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734458038" 17:53:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458038" 17:53:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458038" 17:53:58 INFO [http_cmd]: http status code OK 17:53:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458038" 17:53:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458038" 17:53:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458038" 17:53:58 DEBUG --- status code --- 17:53:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458038" 17:53:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458038" 17:53:58 INFO 17:53:58 INFO [http_cmd]: http status code OK 17:53:58 INFO 17:53:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:58 INFO [http_cmd]: http status code OK 17:53:58 INFO [http_cmd]: http status code OK 17:53:58 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 17:53:58 DEBUG --- status code --- 17:53:58 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 17:53:58 DEBUG --- http response --- 17:53:58 DEBUG --- status code --- 17:53:58 DEBUG --- status code --- 17:53:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:53:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:53:58 INFO [http_cmd]: http status code OK 17:53:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 17:53:58 INFO [http_cmd]: http status code OK 17:53:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:58 DEBUG --- http response --- 17:53:58 DEBUG --- status code --- 17:53:58 DEBUG ----- output ----- 17:53:58 DEBUG --- status code --- 17:53:58 INFO [http_cmd]: http status code OK 17:53:58 DEBUG --- http response --- 17:53:58 DEBUG --- http response --- 17:53:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:53:58 INFO [http_cmd]: http status code OK 17:53:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:58 INFO [http_cmd]: http status code OK 17:53:58 DEBUG { 17:53:58 INFO [http_cmd]: http status code OK 17:53:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:58 DEBUG --- status code --- 17:53:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:53:58 DEBUG ----- output ----- 17:53:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 17:53:58 DEBUG --- status code --- 17:53:58 INFO [http_cmd]: http status code OK 17:53:58 DEBUG --- http response --- 17:53:58 DEBUG --- status code --- 17:53:58 DEBUG "status": "success", 17:53:58 DEBUG --- status code --- 17:53:58 DEBUG --- http response --- 17:53:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:58 DEBUG ----- output ----- 17:53:58 DEBUG { 17:53:58 DEBUG ----- output ----- 17:53:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:58 DEBUG --- status code --- 17:53:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:53:58 DEBUG "data": { 17:53:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:53:58 DEBUG --- http response --- 17:53:58 DEBUG { 17:53:58 DEBUG "status": "success", 17:53:58 DEBUG { 17:53:58 DEBUG --- http response --- 17:53:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:58 DEBUG --- http response --- 17:53:58 DEBUG ----- output ----- 17:53:58 DEBUG { 17:53:58 DEBUG "status": "success", 17:53:58 DEBUG "data": { 17:53:58 DEBUG ----- output ----- 17:53:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 17:53:58 DEBUG "data": { 17:53:58 DEBUG "status": "success", 17:53:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:53:58 DEBUG --- http response --- 17:53:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:53:58 DEBUG ----- output ----- 17:53:58 DEBUG --- http response --- 17:53:58 DEBUG "status": "success", 17:53:58 DEBUG "resultType": "vector", 17:53:58 DEBUG { 17:53:58 DEBUG ----- output ----- 17:53:58 DEBUG "resultType": "vector", 17:53:58 DEBUG "data": { 17:53:58 DEBUG ----- output ----- 17:53:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 17:53:58 DEBUG "resultType": "vector", 17:53:58 DEBUG { 17:53:58 DEBUG "data": { 17:53:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:53:58 DEBUG "result": [ 17:53:58 DEBUG "status": "success", 17:53:58 DEBUG { 17:53:58 DEBUG "result": [ 17:53:58 DEBUG "resultType": "vector", 17:53:58 DEBUG { 17:53:58 DEBUG ----- output ----- 17:53:58 DEBUG "result": [ 17:53:58 DEBUG "status": "success", 17:53:58 DEBUG "resultType": "vector", 17:53:58 DEBUG ----- output ----- 17:53:58 DEBUG { 17:53:58 DEBUG "data": { 17:53:58 DEBUG "status": "success", 17:53:58 DEBUG { 17:53:58 DEBUG "result": [ 17:53:58 DEBUG "status": "success", 17:53:58 DEBUG { 17:53:58 DEBUG { 17:53:58 DEBUG "data": { 17:53:58 DEBUG "result": [ 17:53:58 DEBUG { 17:53:58 DEBUG "metric": { 17:53:58 DEBUG "resultType": "vector", 17:53:58 DEBUG "data": { 17:53:58 DEBUG "metric": { 17:53:58 DEBUG { 17:53:58 DEBUG "data": { 17:53:58 DEBUG "status": "success", 17:53:58 DEBUG "metric": { 17:53:58 DEBUG "resultType": "vector", 17:53:58 DEBUG { 17:53:58 DEBUG "status": "success", 17:53:58 DEBUG "instance": "10.142.0.11:9100" 17:53:58 DEBUG "result": [ 17:53:58 DEBUG "resultType": "vector", 17:53:58 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 17:53:58 DEBUG "metric": { 17:53:58 DEBUG "resultType": "vector", 17:53:58 DEBUG "data": { 17:53:58 DEBUG "pod": "ds-cts-0" 17:53:58 DEBUG "result": [ 17:53:58 DEBUG "metric": { 17:53:58 DEBUG "data": { 17:53:58 DEBUG }, 17:53:58 DEBUG { 17:53:58 DEBUG "result": [ 17:53:58 DEBUG }, 17:53:58 DEBUG "pod": "ds-cts-0" 17:53:58 DEBUG "result": [ 17:53:58 DEBUG "resultType": "vector", 17:53:58 DEBUG }, 17:53:58 DEBUG { 17:53:58 DEBUG "instance": "10.142.0.11:9100" 17:53:58 DEBUG "resultType": "vector", 17:53:58 DEBUG "value": [ 17:53:58 DEBUG "metric": { 17:53:58 DEBUG { 17:53:58 DEBUG "value": [ 17:53:58 DEBUG }, 17:53:58 DEBUG { 17:53:58 DEBUG "result": [ 17:53:58 DEBUG "value": [ 17:53:58 DEBUG "metric": { 17:53:58 DEBUG }, 17:53:58 DEBUG "result": [ 17:53:58 DEBUG 17:53:58 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 17:53:58 DEBUG "metric": { 17:53:58 DEBUG 17:53:58 DEBUG "value": [ 17:53:58 DEBUG "metric": { 17:53:58 DEBUG { 17:53:58 DEBUG 17:53:58 DEBUG "instance": "10.142.0.11:9100" 17:53:58 DEBUG "value": [ 17:53:58 DEBUG { 17:53:58 DEBUG 17:53:58 DEBUG }, 17:53:58 DEBUG "pod": "ds-cts-0" 17:53:58 DEBUG 17:53:58 DEBUG 17:53:58 DEBUG "instance": "10.142.0.11:9100" 17:53:58 DEBUG "metric": { 17:53:58 DEBUG 17:53:58 DEBUG }, 17:53:58 DEBUG 17:53:58 DEBUG "metric": { 17:53:58 DEBUG 17:53:58 DEBUG "value": [ 17:53:58 DEBUG }, 17:53:58 DEBUG 17:53:58 DEBUG [...] 17:53:58 DEBUG }, 17:53:58 DEBUG "pod": "ds-cts-0" 17:53:58 DEBUG 17:53:58 DEBUG "value": [ 17:53:58 DEBUG 17:53:58 DEBUG "instance": "10.142.0.11:9100" 17:53:58 DEBUG [...] 17:53:58 DEBUG 17:53:58 DEBUG "value": [ 17:53:58 DEBUG 17:53:58 DEBUG 17:53:58 DEBUG "value": [ 17:53:58 DEBUG }, 17:53:58 DEBUG [...] 17:53:58 DEBUG 17:53:58 DEBUG 17:53:58 DEBUG }, 17:53:58 DEBUG 17:53:58 DEBUG 17:53:58 DEBUG 17:53:58 DEBUG 17:53:58 DEBUG 17:53:58 DEBUG 17:53:58 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 17:53:58 DEBUG "value": [ 17:53:58 DEBUG 17:53:58 DEBUG 17:53:58 DEBUG "value": [ 17:53:58 DEBUG 17:53:58 DEBUG 17:53:58 DEBUG 17:53:58 DEBUG [...] 17:53:58 DEBUG 17:53:58 DEBUG }, 17:53:58 DEBUG 17:53:58 DEBUG 17:53:58 DEBUG 17:53:58 DEBUG 17:53:58 DEBUG [...] 17:53:58 DEBUG 17:53:58 DEBUG "instance": "10.142.0.51:9100" 17:53:58 DEBUG 17:53:58 DEBUG 17:53:58 DEBUG 17:53:58 DEBUG "value": [ 17:53:58 DEBUG 17:53:58 DEBUG 17:53:58 DEBUG [...] 17:53:58 DEBUG 17:53:58 DEBUG 17:53:58 DEBUG 17:53:58 DEBUG }, 17:53:58 DEBUG [...] 17:53:58 DEBUG 17:53:58 DEBUG 17:53:58 DEBUG 1734458038, 17:53:58 DEBUG "2388631552" 17:53:58 DEBUG ] 17:53:58 DEBUG } 17:53:58 DEBUG ] 17:53:58 DEBUG } 17:53:58 DEBUG } 17:53:58 DEBUG "value": [ 17:53:58 DEBUG 17:53:58 DEBUG 17:53:58 DEBUG [...] 17:53:58 DEBUG [...] 17:53:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 17:53:58 DEBUG 17:53:58 DEBUG 17:53:58 DEBUG [...] 17:53:58 DEBUG 17:53:58 DEBUG 1734458038, 17:53:58 DEBUG 17:53:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 17:53:58 DEBUG 17:53:58 DEBUG 17:53:58 DEBUG 17:53:58 DEBUG 17:53:58 DEBUG 17:53:58 DEBUG 17:53:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 17:53:58 DEBUG "0.0010000000000001561" 17:53:58 DEBUG 17:53:58 DEBUG }, 17:53:58 DEBUG "value": [ 17:53:58 DEBUG }, 17:53:58 DEBUG "instance": "10.142.0.51:9100" 17:53:58 DEBUG 17:53:58 DEBUG [...] 17:53:58 DEBUG 17:53:58 DEBUG }, 17:53:58 DEBUG ] 17:53:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 17:53:58 DEBUG 17:53:58 DEBUG 1734458038, 17:53:58 DEBUG "value": [ 17:53:58 DEBUG }, 17:53:58 DEBUG "instance": "10.142.0.51:9100" 17:53:58 DEBUG 17:53:58 DEBUG 17:53:58 DEBUG 17:53:58 DEBUG "value": [ 17:53:58 DEBUG }, 17:53:58 DEBUG 17:53:58 DEBUG "0.004831936067767892" 17:53:58 DEBUG 1734458038, 17:53:58 DEBUG "value": [ 17:53:58 DEBUG }, 17:53:58 DEBUG "value": [ 17:53:58 DEBUG } 17:53:58 DEBUG "instance": "10.142.0.51:9100" 17:53:58 DEBUG }, 17:53:58 DEBUG "value": [ 17:53:58 DEBUG 17:53:58 DEBUG ] 17:53:58 DEBUG "689467392" 17:53:58 DEBUG 1734458038, 17:53:58 DEBUG 17:53:58 DEBUG 1734458038, 17:53:58 DEBUG ] 17:53:58 DEBUG } 17:53:58 DEBUG } 17:53:58 DEBUG 1734458038, 17:53:58 DEBUG "122139.13333333333" 17:53:58 DEBUG ] 17:53:58 DEBUG } 17:53:58 DEBUG ] 17:53:58 DEBUG } 17:53:58 DEBUG "126431.23333333334" 17:53:58 DEBUG ] 17:53:58 DEBUG "value": [ 17:53:58 DEBUG "instance": "10.142.0.51:9100" 17:53:58 DEBUG } 17:53:58 DEBUG ] 17:53:58 DEBUG "27306.666666666664" 17:53:58 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 17:53:58 DEBUG } 17:53:58 DEBUG 1734458038, 17:53:58 DEBUG } 17:53:58 DEBUG ] 17:53:58 DEBUG } 17:53:58 DEBUG ] 17:53:58 DEBUG } 17:53:58 DEBUG ] 17:53:58 DEBUG } 17:53:58 DEBUG } 17:53:58 DEBUG 1734458038, 17:53:58 DEBUG }, 17:53:58 DEBUG } 17:53:58 DEBUG } 17:53:58 DEBUG ] 17:53:58 DEBUG }, 17:53:58 DEBUG "1393.0922625050262" 17:53:58 DEBUG "0" 17:53:58 DEBUG ] 17:53:58 DEBUG } 17:53:58 DEBUG ] 17:53:58 DEBUG } 17:53:58 DEBUG } 17:53:58 DEBUG "value": [ 17:53:58 DEBUG } 17:53:58 DEBUG } 17:53:58 DEBUG "value": [ 17:53:58 DEBUG ] 17:53:58 DEBUG 1734458038, 17:53:58 DEBUG ] 17:53:58 DEBUG 1734458038, 17:53:58 DEBUG } 17:53:58 DEBUG "0.0004666666666688949" 17:53:58 DEBUG } 17:53:58 DEBUG "1.0882354267867875" 17:53:58 DEBUG ] 17:53:58 DEBUG ] 17:53:58 DEBUG } 17:53:58 DEBUG ] 17:53:58 DEBUG } 17:53:58 DEBUG } 17:53:58 DEBUG } 17:53:58 DEBUG } 17:53:58 DEBUG ] 17:53:58 DEBUG ] 17:53:58 DEBUG } 17:53:58 DEBUG } 17:53:58 DEBUG } 17:53:58 DEBUG } 17:53:58 INFO [loop_until]: OK (rc = 0) 17:53:58 DEBUG --- stdout --- 17:53:58 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 13m 400Mi ds-idrepo-0 11m 576Mi rcs-5d696ff5b-7ttvr 5m 290Mi rcs-5d696ff5b-wwlbr 4m 328Mi 17:53:58 DEBUG --- stderr --- 17:53:58 DEBUG 17:53:58 INFO [loop_until]: OK (rc = 0) 17:53:58 DEBUG --- stdout --- 17:53:58 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 104m 0% 2263Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1166m 3% 2801Mi 9% 17:53:58 DEBUG --- stderr --- 17:53:58 DEBUG 17:54:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458097" 17:54:58 INFO [http_cmd]: http status code OK 17:54:58 DEBUG --- status code --- 17:54:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:58 DEBUG --- http response --- 17:54:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:54:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458098" 17:54:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458098" 17:54:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458098" 17:54:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458098" 17:54:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458098" 17:54:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458098" 17:54:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458098" 17:54:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458098" 17:54:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458098" 17:54:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734458098" 17:54:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458098" 17:54:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458098" 17:54:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458098" 17:54:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458098" 17:54:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458098" 17:54:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734458098" 17:54:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458098" 17:54:58 INFO [http_cmd]: http status code OK 17:54:58 DEBUG --- status code --- 17:54:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734458098" 17:54:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458098" 17:54:58 DEBUG --- http response --- 17:54:58 INFO [http_cmd]: http status code OK 17:54:58 INFO [http_cmd]: http status code OK 17:54:58 INFO [http_cmd]: http status code OK 17:54:58 INFO [http_cmd]: http status code OK 17:54:58 INFO [http_cmd]: http status code OK 17:54:58 INFO [http_cmd]: http status code OK 17:54:58 INFO [http_cmd]: http status code OK 17:54:58 INFO [http_cmd]: http status code OK 17:54:58 INFO [http_cmd]: http status code OK 17:54:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734458098" 17:54:58 INFO [http_cmd]: http status code OK 17:54:58 INFO [http_cmd]: http status code OK 17:54:58 INFO [http_cmd]: http status code OK 17:54:58 INFO [http_cmd]: http status code OK 17:54:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:54:58 INFO [http_cmd]: http status code OK 17:54:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458098" 17:54:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458098" 17:54:58 DEBUG --- status code --- 17:54:58 DEBUG --- status code --- 17:54:58 INFO [http_cmd]: http status code OK 17:54:58 INFO [http_cmd]: http status code OK 17:54:58 DEBUG --- status code --- 17:54:58 DEBUG --- status code --- 17:54:58 DEBUG --- status code --- 17:54:58 DEBUG --- status code --- 17:54:58 DEBUG --- status code --- 17:54:58 DEBUG --- status code --- 17:54:58 DEBUG --- status code --- 17:54:58 DEBUG --- status code --- 17:54:58 DEBUG --- status code --- 17:54:58 DEBUG --- status code --- 17:54:58 DEBUG --- status code --- 17:54:58 DEBUG --- status code --- 17:54:58 INFO [http_cmd]: http status code OK 17:54:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458098" 17:54:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458098" 17:54:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734458098" 17:54:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458098" 17:54:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:58 DEBUG --- status code --- 17:54:58 DEBUG --- status code --- 17:54:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:58 INFO [http_cmd]: http status code OK 17:54:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:58 DEBUG --- status code --- 17:54:58 DEBUG --- http response --- 17:54:58 DEBUG --- http response --- 17:54:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:58 DEBUG --- http response --- 17:54:58 INFO [http_cmd]: http status code OK 17:54:58 INFO [http_cmd]: http status code OK 17:54:58 DEBUG --- http response --- 17:54:58 DEBUG --- http response --- 17:54:58 DEBUG --- status code --- 17:54:58 DEBUG --- http response --- 17:54:58 DEBUG --- http response --- 17:54:58 DEBUG --- http response --- 17:54:58 DEBUG --- http response --- 17:54:58 DEBUG --- http response --- 17:54:58 DEBUG --- http response --- 17:54:58 DEBUG --- http response --- 17:54:58 DEBUG --- http response --- 17:54:58 DEBUG --- http response --- 17:54:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:58 DEBUG --- http response --- 17:54:58 INFO [http_cmd]: http status code OK 17:54:58 ERROR [http_cmd]: ERROR 17:54:58 INFO [http_cmd]: http status code OK 17:54:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:54:58 DEBUG --- http response --- 17:54:58 DEBUG --- http response --- 17:54:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:54:58 DEBUG --- status code --- 17:54:58 INFO [http_cmd]: http status code OK 17:54:58 DEBUG --- status code --- 17:54:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:54:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:54:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:54:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:54:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:54:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:54:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:54:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:54:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:54:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:54:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:54:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:54:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:54:58 DEBUG --- status code --- 17:54:58 ERROR --- status code --- 17:54:58 DEBUG --- status code --- 17:54:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:54:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:54:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:58 DEBUG --- status code --- 17:54:58 INFO [http_cmd]: http status code OK 17:54:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:58 DEBUG --- http response --- 17:54:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:58 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:54:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:58 DEBUG --- http response --- 17:54:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:58 DEBUG --- status code --- 17:54:58 DEBUG --- http response --- 17:54:58 DEBUG --- http response --- 17:54:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:54:58 ERROR --- http response --- 17:54:58 DEBUG --- http response --- 17:54:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:54:58 DEBUG --- http response --- 17:54:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:54:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:54:58 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:54:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:54:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:54:58 DEBUG --- http response --- 17:54:58 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:54:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:54:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734458098" 17:54:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458098" 17:54:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458098" 17:54:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458098" 17:54:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458098" 17:54:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458098" 17:54:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458098" 17:54:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458098" 17:54:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458098" 17:54:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458098" 17:54:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734458098" 17:54:58 INFO [http_cmd]: http status code OK 17:54:58 INFO [http_cmd]: http status code OK 17:54:58 INFO [http_cmd]: http status code OK 17:54:58 INFO [http_cmd]: http status code OK 17:54:58 INFO [http_cmd]: http status code OK 17:54:58 INFO [http_cmd]: http status code OK 17:54:58 DEBUG --- status code --- 17:54:58 DEBUG --- status code --- 17:54:58 DEBUG --- status code --- 17:54:58 DEBUG --- status code --- 17:54:58 DEBUG --- status code --- 17:54:58 DEBUG --- status code --- 17:54:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:58 INFO [http_cmd]: http status code OK 17:54:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:58 INFO [http_cmd]: http status code OK 17:54:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:58 DEBUG --- http response --- 17:54:58 DEBUG --- http response --- 17:54:58 DEBUG --- status code --- 17:54:58 DEBUG --- http response --- 17:54:58 DEBUG --- status code --- 17:54:58 DEBUG --- http response --- 17:54:58 DEBUG --- http response --- 17:54:58 DEBUG --- http response --- 17:54:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:54:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:54:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:58 INFO [http_cmd]: http status code OK 17:54:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:54:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 17:54:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:54:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 17:54:58 DEBUG ----- output ----- 17:54:58 INFO [http_cmd]: http status code OK 17:54:58 DEBUG ----- output ----- 17:54:58 DEBUG --- http response --- 17:54:58 DEBUG --- status code --- 17:54:58 DEBUG ----- output ----- 17:54:58 DEBUG --- http response --- 17:54:58 DEBUG ----- output ----- 17:54:58 DEBUG ----- output ----- 17:54:58 DEBUG ----- output ----- 17:54:58 DEBUG { 17:54:58 DEBUG --- status code --- 17:54:58 DEBUG { 17:54:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 17:54:58 DEBUG { 17:54:58 DEBUG { 17:54:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:54:58 DEBUG { 17:54:58 DEBUG { 17:54:58 DEBUG "status": "success", 17:54:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:58 DEBUG "status": "success", 17:54:58 DEBUG --- http response --- 17:54:58 DEBUG ----- output ----- 17:54:58 DEBUG "status": "success", 17:54:58 DEBUG "status": "success", 17:54:58 DEBUG ----- output ----- 17:54:58 DEBUG "status": "success", 17:54:58 DEBUG "status": "success", 17:54:58 DEBUG "data": { 17:54:58 DEBUG --- http response --- 17:54:58 DEBUG "data": { 17:54:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:54:58 DEBUG { 17:54:58 DEBUG "data": { 17:54:58 DEBUG "data": { 17:54:58 DEBUG { 17:54:58 DEBUG "data": { 17:54:58 DEBUG "data": { 17:54:58 DEBUG "resultType": "vector", 17:54:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 17:54:58 DEBUG "resultType": "vector", 17:54:58 DEBUG ----- output ----- 17:54:58 DEBUG "status": "success", 17:54:58 DEBUG "resultType": "vector", 17:54:58 DEBUG "resultType": "vector", 17:54:58 INFO [http_cmd]: http status code OK 17:54:58 DEBUG "status": "success", 17:54:58 DEBUG "resultType": "vector", 17:54:58 DEBUG "resultType": "vector", 17:54:58 DEBUG "result": [ 17:54:58 DEBUG ----- output ----- 17:54:58 DEBUG "result": [ 17:54:58 DEBUG { 17:54:58 DEBUG "data": { 17:54:58 DEBUG "result": [ 17:54:58 DEBUG "result": [ 17:54:58 DEBUG --- status code --- 17:54:58 DEBUG "data": { 17:54:58 DEBUG "result": [ 17:54:58 DEBUG "result": [ 17:54:58 DEBUG { 17:54:58 DEBUG { 17:54:58 DEBUG { 17:54:58 DEBUG "status": "success", 17:54:58 DEBUG "resultType": "vector", 17:54:58 DEBUG { 17:54:58 DEBUG { 17:54:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:58 DEBUG "resultType": "vector", 17:54:58 DEBUG { 17:54:58 DEBUG { 17:54:58 DEBUG "metric": { 17:54:58 DEBUG "status": "success", 17:54:58 DEBUG "metric": { 17:54:58 DEBUG "data": { 17:54:58 DEBUG "result": [ 17:54:58 DEBUG "metric": { 17:54:58 DEBUG "metric": { 17:54:58 DEBUG --- http response --- 17:54:58 DEBUG "result": [ 17:54:58 DEBUG "metric": { 17:54:58 DEBUG "metric": { 17:54:58 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 17:54:58 DEBUG "data": { 17:54:58 DEBUG "instance": "10.142.0.11:9100" 17:54:58 DEBUG "resultType": "vector", 17:54:58 DEBUG { 17:54:58 DEBUG "instance": "10.142.0.11:9100" 17:54:58 DEBUG "pod": "ds-cts-0" 17:54:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:54:58 DEBUG { 17:54:58 DEBUG "instance": "10.142.0.11:9100" 17:54:58 DEBUG "pod": "ds-cts-0" 17:54:58 DEBUG }, 17:54:58 DEBUG "resultType": "vector", 17:54:58 DEBUG }, 17:54:58 DEBUG "result": [ 17:54:58 DEBUG "metric": { 17:54:58 DEBUG }, 17:54:58 DEBUG }, 17:54:58 DEBUG ----- output ----- 17:54:58 DEBUG "metric": { 17:54:58 DEBUG }, 17:54:58 DEBUG }, 17:54:58 DEBUG "value": [ 17:54:58 DEBUG "result": [ 17:54:58 DEBUG "value": [ 17:54:58 DEBUG { 17:54:58 DEBUG "pod": "ds-cts-0" 17:54:58 DEBUG "value": [ 17:54:58 DEBUG "value": [ 17:54:58 DEBUG { 17:54:58 DEBUG "instance": "10.142.0.11:9100" 17:54:58 DEBUG "value": [ 17:54:58 DEBUG "value": [ 17:54:58 DEBUG 17:54:58 DEBUG { 17:54:58 DEBUG 17:54:58 DEBUG "metric": { 17:54:58 DEBUG }, 17:54:58 DEBUG 17:54:58 DEBUG 17:54:58 DEBUG "status": "success", 17:54:58 DEBUG }, 17:54:58 DEBUG 17:54:58 DEBUG 17:54:58 DEBUG 17:54:58 DEBUG "metric": { 17:54:58 DEBUG 17:54:58 DEBUG "instance": "10.142.0.11:9100" 17:54:58 DEBUG "value": [ 17:54:58 DEBUG 17:54:58 DEBUG 17:54:58 DEBUG "data": { 17:54:58 DEBUG "value": [ 17:54:58 DEBUG 17:54:58 DEBUG 17:54:58 DEBUG 17:54:58 DEBUG "pod": "ds-cts-0" 17:54:58 DEBUG 17:54:58 DEBUG }, 17:54:58 DEBUG 17:54:58 DEBUG 17:54:58 DEBUG 17:54:58 DEBUG "resultType": "vector", 17:54:58 DEBUG 17:54:58 DEBUG 17:54:58 DEBUG 17:54:58 DEBUG [...] 17:54:58 DEBUG }, 17:54:58 DEBUG [...] 17:54:58 DEBUG "value": [ 17:54:58 DEBUG 17:54:58 DEBUG [...] 17:54:58 DEBUG [...] 17:54:58 DEBUG "result": [ 17:54:58 DEBUG 17:54:58 DEBUG [...] 17:54:58 DEBUG 17:54:58 DEBUG 17:54:58 DEBUG "value": [ 17:54:58 DEBUG 17:54:58 DEBUG 17:54:58 DEBUG 17:54:58 DEBUG 17:54:58 DEBUG 17:54:58 DEBUG { 17:54:58 DEBUG 17:54:58 DEBUG [...] 17:54:58 DEBUG 17:54:58 DEBUG 17:54:58 DEBUG 17:54:58 DEBUG 17:54:58 DEBUG [...] 17:54:58 DEBUG 17:54:58 DEBUG 17:54:58 DEBUG 17:54:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 17:54:58 DEBUG [...] 17:54:58 DEBUG 17:54:58 DEBUG 17:54:58 DEBUG 17:54:58 DEBUG 17:54:58 DEBUG 17:54:58 DEBUG 17:54:58 DEBUG 17:54:58 DEBUG 17:54:58 DEBUG "metric": { 17:54:58 DEBUG }, 17:54:58 DEBUG 17:54:58 DEBUG 17:54:58 DEBUG "instance": "10.142.0.51:9100" 17:54:58 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 17:54:58 DEBUG 17:54:58 DEBUG "instance": "10.142.0.51:9100" 17:54:58 DEBUG 17:54:58 DEBUG 17:54:58 DEBUG 17:54:58 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 17:54:58 DEBUG "value": [ 17:54:58 DEBUG 17:54:58 DEBUG 17:54:58 DEBUG }, 17:54:58 DEBUG }, 17:54:58 DEBUG [...] 17:54:58 DEBUG }, 17:54:58 DEBUG [...] 17:54:58 DEBUG "instance": "10.142.0.51:9100" 17:54:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 17:54:58 DEBUG }, 17:54:58 DEBUG 1734458098, 17:54:58 DEBUG 17:54:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 17:54:58 DEBUG "value": [ 17:54:58 DEBUG "value": [ 17:54:58 DEBUG 17:54:58 DEBUG "value": [ 17:54:58 DEBUG 17:54:58 DEBUG 17:54:58 DEBUG 17:54:58 DEBUG "instance": "10.142.0.51:9100" 17:54:58 DEBUG }, 17:54:58 DEBUG "value": [ 17:54:58 DEBUG 1734458098, 17:54:58 DEBUG "0.00043333333333303397" 17:54:58 DEBUG ] 17:54:58 DEBUG } 17:54:58 DEBUG 1734458098, 17:54:58 DEBUG }, 17:54:58 DEBUG }, 17:54:58 DEBUG "value": [ 17:54:58 DEBUG "0.015305137977814777" 17:54:58 DEBUG "instance": "10.142.0.51:9100" 17:54:58 DEBUG }, 17:54:58 DEBUG 1734458098, 17:54:58 DEBUG 1734458098, 17:54:58 DEBUG 17:54:58 DEBUG ] 17:54:58 DEBUG "0.0003333333333333854" 17:54:58 DEBUG "value": [ 17:54:58 DEBUG "value": [ 17:54:58 DEBUG 17:54:58 DEBUG ] 17:54:58 DEBUG }, 17:54:58 DEBUG "value": [ 17:54:58 DEBUG "133671.13333333333" 17:54:58 DEBUG "2597236736" 17:54:58 DEBUG 17:54:58 DEBUG } 17:54:58 DEBUG ] 17:54:58 DEBUG 1734458098, 17:54:58 DEBUG 1734458098, 17:54:58 DEBUG "0" 17:54:58 DEBUG ] 17:54:58 DEBUG } 17:54:58 DEBUG ] 17:54:58 DEBUG 1734458098, 17:54:58 DEBUG ] 17:54:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 17:54:58 DEBUG } 17:54:58 DEBUG } 17:54:58 DEBUG "129447.8666666667" 17:54:58 DEBUG 17:54:58 DEBUG } 17:54:58 DEBUG "value": [ 17:54:58 DEBUG } 17:54:58 DEBUG ] 17:54:58 DEBUG "694300672" 17:54:58 DEBUG } 17:54:58 DEBUG ] 17:54:58 DEBUG } 17:54:58 DEBUG } 17:54:58 DEBUG ] 17:54:58 DEBUG ] 17:54:58 DEBUG 1734458098, 17:54:58 DEBUG } 17:54:58 DEBUG } 17:54:58 DEBUG ] 17:54:58 DEBUG }, 17:54:58 DEBUG ] 17:54:58 DEBUG 17:54:58 DEBUG } 17:54:58 DEBUG } 17:54:58 DEBUG "19797.333333333332" 17:54:58 DEBUG ] 17:54:58 DEBUG } 17:54:58 DEBUG ] 17:54:58 DEBUG } 17:54:58 DEBUG } 17:54:58 DEBUG [...] 17:54:58 DEBUG } 17:54:58 DEBUG ] 17:54:58 DEBUG } 17:54:58 DEBUG "value": [ 17:54:58 DEBUG } 17:54:58 DEBUG ] 17:54:58 DEBUG 17:54:58 DEBUG } 17:54:58 DEBUG } 17:54:58 DEBUG 1734458098, 17:54:58 DEBUG } 17:54:58 DEBUG } 17:54:58 DEBUG 17:54:58 DEBUG ] 17:54:58 DEBUG "1361.827736764823" 17:54:58 DEBUG } 17:54:58 DEBUG 17:54:58 DEBUG } 17:54:58 DEBUG } 17:54:58 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 17:54:58 DEBUG ] 17:54:58 DEBUG }, 17:54:58 DEBUG } 17:54:58 DEBUG "value": [ 17:54:58 DEBUG ] 17:54:58 DEBUG 1734458098, 17:54:58 DEBUG } 17:54:58 DEBUG "1.0615573793898905" 17:54:58 DEBUG } 17:54:58 DEBUG ] 17:54:58 DEBUG } 17:54:58 DEBUG ] 17:54:58 DEBUG } 17:54:58 DEBUG } 17:54:58 INFO 17:54:58 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 17:54:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:54:58 INFO 17:54:58 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 17:54:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:54:58 INFO [loop_until]: OK (rc = 0) 17:54:58 DEBUG --- stdout --- 17:54:58 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 8m 404Mi ds-idrepo-0 16m 585Mi rcs-5d696ff5b-7ttvr 5m 290Mi rcs-5d696ff5b-wwlbr 7m 330Mi 17:54:58 DEBUG --- stderr --- 17:54:58 DEBUG 17:54:58 INFO [loop_until]: OK (rc = 0) 17:54:58 DEBUG --- stdout --- 17:54:58 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 100m 0% 2294Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1152m 3% 3529Mi 12% 17:54:58 DEBUG --- stderr --- 17:54:58 DEBUG 17:55:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458158" 17:55:58 INFO [http_cmd]: http status code OK 17:55:58 DEBUG --- status code --- 17:55:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:58 DEBUG --- http response --- 17:55:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:55:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458158" 17:55:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458158" 17:55:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458158" 17:55:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458158" 17:55:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734458158" 17:55:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458158" 17:55:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734458158" 17:55:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734458158" 17:55:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458158" 17:55:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458158" 17:55:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458158" 17:55:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458158" 17:55:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458158" 17:55:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458158" 17:55:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458158" 17:55:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458158" 17:55:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458158" 17:55:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734458158" 17:55:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458158" 17:55:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734458158" 17:55:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458158" 17:55:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458158" 17:55:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458158" 17:55:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458158" 17:55:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458158" 17:55:58 INFO [http_cmd]: http status code OK 17:55:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458158" 17:55:58 INFO [http_cmd]: http status code OK 17:55:58 INFO [http_cmd]: http status code OK 17:55:58 INFO [http_cmd]: http status code OK 17:55:58 INFO [http_cmd]: http status code OK 17:55:58 INFO [http_cmd]: http status code OK 17:55:58 INFO [http_cmd]: http status code OK 17:55:58 INFO [http_cmd]: http status code OK 17:55:58 INFO [http_cmd]: http status code OK 17:55:58 INFO [http_cmd]: http status code OK 17:55:58 INFO [http_cmd]: http status code OK 17:55:58 DEBUG --- status code --- 17:55:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:58 DEBUG --- http response --- 17:55:58 INFO [http_cmd]: http status code OK 17:55:58 INFO [http_cmd]: http status code OK 17:55:58 DEBUG --- status code --- 17:55:58 DEBUG --- status code --- 17:55:58 DEBUG --- status code --- 17:55:58 DEBUG --- status code --- 17:55:58 DEBUG --- status code --- 17:55:58 DEBUG --- status code --- 17:55:58 DEBUG --- status code --- 17:55:58 DEBUG --- status code --- 17:55:58 DEBUG --- status code --- 17:55:58 DEBUG --- status code --- 17:55:58 INFO [http_cmd]: http status code OK 17:55:58 INFO [http_cmd]: http status code OK 17:55:58 INFO [http_cmd]: http status code OK 17:55:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:55:58 DEBUG --- status code --- 17:55:58 DEBUG --- status code --- 17:55:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:58 DEBUG --- http response --- 17:55:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:58 DEBUG --- status code --- 17:55:58 DEBUG --- status code --- 17:55:58 DEBUG --- status code --- 17:55:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:58 DEBUG --- http response --- 17:55:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:55:58 DEBUG --- http response --- 17:55:58 DEBUG --- http response --- 17:55:58 DEBUG --- http response --- 17:55:58 DEBUG --- http response --- 17:55:58 DEBUG --- http response --- 17:55:58 DEBUG --- http response --- 17:55:58 DEBUG --- http response --- 17:55:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:58 DEBUG --- http response --- 17:55:58 DEBUG --- http response --- 17:55:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:55:58 DEBUG --- http response --- 17:55:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:55:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:55:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:55:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:55:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:55:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:55:58 INFO [http_cmd]: http status code OK 17:55:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:55:58 INFO [http_cmd]: http status code OK 17:55:58 ERROR [http_cmd]: ERROR 17:55:58 INFO [http_cmd]: http status code OK 17:55:58 DEBUG --- http response --- 17:55:58 DEBUG --- http response --- 17:55:58 DEBUG --- http response --- 17:55:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:55:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:55:58 INFO [http_cmd]: http status code OK 17:55:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:55:58 INFO [http_cmd]: http status code OK 17:55:58 INFO [http_cmd]: http status code OK 17:55:58 INFO [http_cmd]: http status code OK 17:55:58 INFO [http_cmd]: http status code OK 17:55:58 DEBUG --- status code --- 17:55:58 DEBUG --- status code --- 17:55:58 INFO [http_cmd]: http status code OK 17:55:58 DEBUG --- status code --- 17:55:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:58 DEBUG --- http response --- 17:55:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:55:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:55:58 DEBUG --- status code --- 17:55:58 DEBUG --- status code --- 17:55:58 DEBUG --- status code --- 17:55:58 DEBUG --- status code --- 17:55:58 DEBUG --- status code --- 17:55:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:58 ERROR --- status code --- 17:55:58 DEBUG --- status code --- 17:55:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:55:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:55:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:58 DEBUG --- http response --- 17:55:58 DEBUG --- http response --- 17:55:58 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:55:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:58 DEBUG --- http response --- 17:55:58 DEBUG --- http response --- 17:55:58 DEBUG --- http response --- 17:55:58 DEBUG --- http response --- 17:55:58 DEBUG --- http response --- 17:55:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:55:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:55:58 ERROR --- http response --- 17:55:58 DEBUG --- http response --- 17:55:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:55:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:55:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:55:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:55:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:55:58 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:55:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:55:58 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:55:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458158" 17:55:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734458158" 17:55:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458158" 17:55:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458158" 17:55:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458158" 17:55:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458158" 17:55:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458158" 17:55:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734458158" 17:55:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458158" 17:55:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458158" 17:55:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458158" 17:55:58 INFO [http_cmd]: http status code OK 17:55:58 DEBUG --- status code --- 17:55:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:58 DEBUG --- http response --- 17:55:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 17:55:58 INFO [http_cmd]: http status code OK 17:55:58 DEBUG ----- output ----- 17:55:58 DEBUG --- status code --- 17:55:58 INFO [http_cmd]: http status code OK 17:55:58 INFO [http_cmd]: http status code OK 17:55:58 INFO [http_cmd]: http status code OK 17:55:58 DEBUG { 17:55:58 INFO [http_cmd]: http status code OK 17:55:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:58 INFO [http_cmd]: http status code OK 17:55:58 INFO [http_cmd]: http status code OK 17:55:58 DEBUG --- status code --- 17:55:58 DEBUG --- status code --- 17:55:58 DEBUG --- status code --- 17:55:58 DEBUG "status": "success", 17:55:58 DEBUG --- status code --- 17:55:58 DEBUG --- http response --- 17:55:58 INFO [http_cmd]: http status code OK 17:55:58 DEBUG --- status code --- 17:55:58 DEBUG --- status code --- 17:55:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:58 DEBUG "data": { 17:55:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:55:58 DEBUG --- status code --- 17:55:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:58 DEBUG --- http response --- 17:55:58 DEBUG --- http response --- 17:55:58 DEBUG --- http response --- 17:55:58 DEBUG "resultType": "vector", 17:55:58 DEBUG --- http response --- 17:55:58 DEBUG ----- output ----- 17:55:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:58 DEBUG --- http response --- 17:55:58 DEBUG --- http response --- 17:55:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:55:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 17:55:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:55:58 DEBUG "result": [ 17:55:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:55:58 DEBUG { 17:55:58 DEBUG --- http response --- 17:55:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:55:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 17:55:58 DEBUG ----- output ----- 17:55:58 DEBUG ----- output ----- 17:55:58 INFO [http_cmd]: http status code OK 17:55:58 DEBUG ----- output ----- 17:55:58 DEBUG { 17:55:58 DEBUG ----- output ----- 17:55:58 DEBUG "status": "success", 17:55:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 17:55:58 DEBUG ----- output ----- 17:55:58 DEBUG ----- output ----- 17:55:58 DEBUG { 17:55:58 DEBUG { 17:55:58 DEBUG --- status code --- 17:55:58 DEBUG { 17:55:58 DEBUG "metric": { 17:55:58 DEBUG { 17:55:58 DEBUG "data": { 17:55:58 DEBUG ----- output ----- 17:55:58 DEBUG { 17:55:58 DEBUG { 17:55:58 DEBUG "status": "success", 17:55:58 DEBUG "status": "success", 17:55:58 INFO [http_cmd]: http status code OK 17:55:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:58 DEBUG "status": "success", 17:55:58 DEBUG "pod": "ds-cts-0" 17:55:58 DEBUG "status": "success", 17:55:58 DEBUG "resultType": "vector", 17:55:58 DEBUG { 17:55:58 DEBUG "status": "success", 17:55:58 DEBUG "status": "success", 17:55:58 DEBUG "data": { 17:55:58 DEBUG "data": { 17:55:58 DEBUG --- status code --- 17:55:58 DEBUG --- http response --- 17:55:58 DEBUG "data": { 17:55:58 DEBUG }, 17:55:58 DEBUG "data": { 17:55:58 DEBUG "result": [ 17:55:58 DEBUG "status": "success", 17:55:58 DEBUG "data": { 17:55:58 DEBUG "data": { 17:55:58 DEBUG "resultType": "vector", 17:55:58 DEBUG "resultType": "vector", 17:55:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:58 DEBUG "resultType": "vector", 17:55:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:55:58 DEBUG "value": [ 17:55:58 DEBUG "resultType": "vector", 17:55:58 DEBUG { 17:55:58 DEBUG "data": { 17:55:58 DEBUG "resultType": "vector", 17:55:58 DEBUG "resultType": "vector", 17:55:58 DEBUG "result": [ 17:55:58 DEBUG "result": [ 17:55:58 DEBUG --- http response --- 17:55:58 DEBUG "result": [ 17:55:58 DEBUG ----- output ----- 17:55:58 DEBUG 17:55:58 DEBUG "result": [ 17:55:58 DEBUG "metric": { 17:55:58 DEBUG "resultType": "vector", 17:55:58 DEBUG "result": [ 17:55:58 DEBUG "result": [ 17:55:58 DEBUG { 17:55:58 DEBUG { 17:55:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:55:58 DEBUG { 17:55:58 DEBUG { 17:55:58 DEBUG 17:55:58 DEBUG { 17:55:58 DEBUG "instance": "10.142.0.11:9100" 17:55:58 DEBUG "result": [ 17:55:58 DEBUG { 17:55:58 DEBUG { 17:55:58 DEBUG "metric": { 17:55:58 DEBUG "metric": { 17:55:58 DEBUG ----- output ----- 17:55:58 DEBUG "metric": { 17:55:58 DEBUG "status": "success", 17:55:58 DEBUG 17:55:58 DEBUG "metric": { 17:55:58 DEBUG }, 17:55:58 DEBUG { 17:55:58 DEBUG "metric": { 17:55:58 DEBUG "pod": "ds-cts-0" 17:55:58 DEBUG }, 17:55:58 DEBUG "value": [ 17:55:58 DEBUG 17:55:58 DEBUG { 17:55:58 DEBUG "data": { 17:55:58 DEBUG [...] 17:55:58 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 17:55:58 DEBUG "value": [ 17:55:58 DEBUG "metric": { 17:55:58 DEBUG "metric": { 17:55:58 DEBUG "instance": "10.142.0.11:9100" 17:55:58 DEBUG "pod": "ds-cts-0" 17:55:58 DEBUG }, 17:55:58 DEBUG "value": [ 17:55:58 DEBUG 17:55:58 DEBUG 17:55:58 DEBUG 17:55:58 DEBUG 17:55:58 DEBUG 17:55:58 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 17:55:58 DEBUG "pod": "ds-cts-0" 17:55:58 DEBUG }, 17:55:58 DEBUG "instance": "10.142.0.11:9100" 17:55:58 DEBUG 17:55:58 DEBUG 17:55:58 DEBUG "resultType": "vector", 17:55:58 DEBUG "status": "success", 17:55:58 DEBUG [...] 17:55:58 DEBUG 17:55:58 DEBUG 17:55:58 DEBUG }, 17:55:58 DEBUG }, 17:55:58 DEBUG "value": [ 17:55:58 DEBUG }, 17:55:58 DEBUG [...] 17:55:58 DEBUG }, 17:55:58 DEBUG "result": [ 17:55:58 DEBUG "data": { 17:55:58 DEBUG 17:55:58 DEBUG 17:55:58 DEBUG 17:55:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 17:55:58 DEBUG }, 17:55:58 DEBUG "value": [ 17:55:58 DEBUG 1734458158, 17:55:58 DEBUG "0.01574253969773548" 17:55:58 DEBUG "value": [ 17:55:58 DEBUG { 17:55:58 DEBUG "resultType": "vector", 17:55:58 DEBUG 17:55:58 DEBUG 17:55:58 DEBUG "value": [ 17:55:58 DEBUG "value": [ 17:55:58 DEBUG 17:55:58 DEBUG "value": [ 17:55:58 DEBUG 17:55:58 DEBUG ] 17:55:58 DEBUG 17:55:58 DEBUG "metric": { 17:55:58 DEBUG "result": [ 17:55:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 17:55:58 DEBUG [...] 17:55:58 DEBUG 17:55:58 DEBUG 17:55:58 DEBUG 17:55:58 DEBUG 17:55:58 DEBUG 17:55:58 DEBUG } 17:55:58 DEBUG 17:55:58 DEBUG "instance": "10.142.0.11:9100" 17:55:58 DEBUG { 17:55:58 DEBUG }, 17:55:58 DEBUG 17:55:58 DEBUG 17:55:58 DEBUG 17:55:58 DEBUG 17:55:58 DEBUG 17:55:58 DEBUG 17:55:58 DEBUG ] 17:55:58 DEBUG 17:55:58 DEBUG }, 17:55:58 DEBUG "metric": { 17:55:58 DEBUG "value": [ 17:55:58 DEBUG 17:55:58 DEBUG 17:55:58 DEBUG [...] 17:55:58 DEBUG 17:55:58 DEBUG 17:55:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 17:55:58 DEBUG } 17:55:58 DEBUG [...] 17:55:58 DEBUG "value": [ 17:55:58 DEBUG "instance": "10.142.0.11:9100" 17:55:58 DEBUG 1734458158, 17:55:58 DEBUG 17:55:58 DEBUG [...] 17:55:58 DEBUG 17:55:58 DEBUG 17:55:58 DEBUG 17:55:58 DEBUG }, 17:55:58 DEBUG } 17:55:58 DEBUG 17:55:58 DEBUG 17:55:58 DEBUG }, 17:55:58 DEBUG "0" 17:55:58 DEBUG "instance": "10.142.0.51:9100" 17:55:58 DEBUG 17:55:58 DEBUG 17:55:58 DEBUG [...] 17:55:58 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 17:55:58 DEBUG "value": [ 17:55:58 DEBUG 17:55:58 DEBUG 17:55:58 DEBUG "value": [ 17:55:58 DEBUG ] 17:55:58 DEBUG }, 17:55:58 DEBUG [...] 17:55:58 DEBUG 17:55:58 DEBUG 17:55:58 DEBUG }, 17:55:58 DEBUG 1734458158, 17:55:58 DEBUG 17:55:58 DEBUG 17:55:58 DEBUG 17:55:58 DEBUG } 17:55:58 DEBUG "value": [ 17:55:58 DEBUG 17:55:58 DEBUG "instance": "10.142.0.51:9100" 17:55:58 DEBUG 17:55:58 DEBUG "value": [ 17:55:58 DEBUG "1177.6279302087983" 17:55:58 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 17:55:58 DEBUG [...] 17:55:58 DEBUG 17:55:58 DEBUG ] 17:55:58 DEBUG 1734458158, 17:55:58 DEBUG 17:55:58 DEBUG }, 17:55:58 DEBUG 17:55:58 DEBUG 1734458158, 17:55:58 DEBUG ] 17:55:58 DEBUG }, 17:55:58 DEBUG 17:55:58 DEBUG 17:55:58 DEBUG } 17:55:58 DEBUG "0.00039999999999954146" 17:55:58 DEBUG 17:55:58 DEBUG "value": [ 17:55:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 17:55:58 DEBUG "2623676416" 17:55:58 DEBUG } 17:55:58 DEBUG "value": [ 17:55:58 DEBUG 17:55:58 DEBUG [...] 17:55:58 DEBUG } 17:55:58 DEBUG ] 17:55:58 DEBUG "instance": "10.142.0.51:9100" 17:55:58 DEBUG 1734458158, 17:55:58 DEBUG "0.0006666666666667708" 17:55:58 DEBUG ] 17:55:58 DEBUG ] 17:55:58 DEBUG 1734458158, 17:55:58 DEBUG 17:55:58 DEBUG 17:55:58 DEBUG } 17:55:58 DEBUG }, 17:55:58 DEBUG }, 17:55:58 DEBUG ] 17:55:58 DEBUG } 17:55:58 DEBUG } 17:55:58 DEBUG "1.075389900773259" 17:55:58 DEBUG "instance": "10.142.0.51:9100" 17:55:58 DEBUG 17:55:58 DEBUG ] 17:55:58 DEBUG "value": [ 17:55:58 DEBUG "value": [ 17:55:58 DEBUG } 17:55:58 DEBUG ] 17:55:58 DEBUG } 17:55:58 DEBUG } 17:55:58 DEBUG }, 17:55:58 DEBUG 17:55:58 DEBUG } 17:55:58 DEBUG 1734458158, 17:55:58 DEBUG 1734458158, 17:55:58 DEBUG ] 17:55:58 DEBUG ] 17:55:58 DEBUG } 17:55:58 DEBUG "value": [ 17:55:58 DEBUG "instance": "10.142.0.51:9100" 17:55:58 DEBUG } 17:55:58 DEBUG "129820.93333333335" 17:55:58 DEBUG "694530048" 17:55:58 DEBUG } 17:55:58 DEBUG } 17:55:58 DEBUG 1734458158, 17:55:58 DEBUG }, 17:55:58 DEBUG ] 17:55:58 DEBUG ] 17:55:58 DEBUG } 17:55:58 DEBUG ] 17:55:58 DEBUG "127869.66666666667" 17:55:58 DEBUG "value": [ 17:55:58 DEBUG } 17:55:58 DEBUG } 17:55:58 DEBUG } 17:55:58 DEBUG ] 17:55:58 DEBUG 1734458158, 17:55:58 DEBUG ] 17:55:58 DEBUG ] 17:55:58 DEBUG } 17:55:58 DEBUG } 17:55:58 DEBUG "26214.4" 17:55:58 DEBUG ] 17:55:58 DEBUG } 17:55:58 DEBUG ] 17:55:58 DEBUG } 17:55:58 DEBUG } 17:55:58 DEBUG ] 17:55:58 DEBUG } 17:55:58 DEBUG } 17:55:58 DEBUG } 17:55:58 DEBUG } 17:55:58 DEBUG } 17:55:58 DEBUG } 17:55:58 INFO 17:55:58 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 17:55:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:55:58 INFO 17:55:58 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 17:55:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:55:58 INFO [loop_until]: OK (rc = 0) 17:55:58 DEBUG --- stdout --- 17:55:58 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 17m 410Mi ds-idrepo-0 9m 590Mi rcs-5d696ff5b-7ttvr 7m 292Mi rcs-5d696ff5b-wwlbr 5m 330Mi 17:55:58 DEBUG --- stderr --- 17:55:58 DEBUG 17:55:58 INFO [loop_until]: OK (rc = 0) 17:55:58 DEBUG --- stdout --- 17:55:58 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 105m 0% 2262Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1160m 3% 3547Mi 12% 17:55:58 DEBUG --- stderr --- 17:55:58 DEBUG 17:56:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458218" 17:56:58 INFO [http_cmd]: http status code OK 17:56:58 DEBUG --- status code --- 17:56:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:58 DEBUG --- http response --- 17:56:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:56:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734458218" 17:56:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458218" 17:56:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458218" 17:56:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458218" 17:56:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458218" 17:56:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458218" 17:56:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458218" 17:56:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458218" 17:56:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734458218" 17:56:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458218" 17:56:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458218" 17:56:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734458218" 17:56:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458218" 17:56:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458218" 17:56:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458218" 17:56:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458218" 17:56:58 INFO [http_cmd]: http status code OK 17:56:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458218" 17:56:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458218" 17:56:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458218" 17:56:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734458218" 17:56:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458218" 17:56:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458218" 17:56:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458218" 17:56:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734458218" 17:56:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458218" 17:56:58 INFO [http_cmd]: http status code OK 17:56:58 INFO [http_cmd]: http status code OK 17:56:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458218" 17:56:58 INFO [http_cmd]: http status code OK 17:56:58 INFO [http_cmd]: http status code OK 17:56:58 INFO [http_cmd]: http status code OK 17:56:58 INFO [http_cmd]: http status code OK 17:56:58 INFO [http_cmd]: http status code OK 17:56:58 INFO [http_cmd]: http status code OK 17:56:58 INFO [http_cmd]: http status code OK 17:56:58 INFO [http_cmd]: http status code OK 17:56:58 INFO [http_cmd]: http status code OK 17:56:58 DEBUG --- status code --- 17:56:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:58 INFO [http_cmd]: http status code OK 17:56:58 DEBUG --- status code --- 17:56:58 INFO [http_cmd]: http status code OK 17:56:58 INFO [http_cmd]: http status code OK 17:56:58 DEBUG --- status code --- 17:56:58 INFO [http_cmd]: http status code OK 17:56:58 INFO [http_cmd]: http status code OK 17:56:58 DEBUG --- status code --- 17:56:58 INFO [http_cmd]: http status code OK 17:56:58 DEBUG --- status code --- 17:56:58 INFO [http_cmd]: http status code OK 17:56:58 DEBUG --- status code --- 17:56:58 INFO [http_cmd]: http status code OK 17:56:58 INFO [http_cmd]: http status code OK 17:56:58 DEBUG --- status code --- 17:56:58 INFO [http_cmd]: http status code OK 17:56:58 DEBUG --- status code --- 17:56:58 DEBUG --- status code --- 17:56:58 DEBUG --- status code --- 17:56:58 DEBUG --- status code --- 17:56:58 DEBUG --- status code --- 17:56:58 DEBUG --- http response --- 17:56:58 DEBUG --- status code --- 17:56:58 DEBUG --- status code --- 17:56:58 INFO [http_cmd]: http status code OK 17:56:58 INFO [http_cmd]: http status code OK 17:56:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:58 DEBUG --- status code --- 17:56:58 DEBUG --- status code --- 17:56:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:58 DEBUG --- status code --- 17:56:58 DEBUG --- status code --- 17:56:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:58 DEBUG --- status code --- 17:56:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:58 DEBUG --- status code --- 17:56:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:58 DEBUG --- status code --- 17:56:58 ERROR [http_cmd]: ERROR 17:56:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:58 DEBUG --- http response --- 17:56:58 DEBUG --- status code --- 17:56:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:58 DEBUG --- http response --- 17:56:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:56:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:58 DEBUG --- status code --- 17:56:58 DEBUG --- status code --- 17:56:58 DEBUG --- http response --- 17:56:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:58 DEBUG --- http response --- 17:56:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:58 DEBUG --- http response --- 17:56:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:58 DEBUG --- http response --- 17:56:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:58 DEBUG --- http response --- 17:56:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:58 ERROR --- status code --- 17:56:58 INFO [http_cmd]: http status code OK 17:56:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:56:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:58 DEBUG --- http response --- 17:56:58 DEBUG --- http response --- 17:56:58 DEBUG --- http response --- 17:56:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:56:58 DEBUG --- http response --- 17:56:58 DEBUG --- http response --- 17:56:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:56:58 DEBUG --- http response --- 17:56:58 DEBUG --- http response --- 17:56:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:56:58 DEBUG --- http response --- 17:56:58 DEBUG --- http response --- 17:56:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:56:58 DEBUG --- http response --- 17:56:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:56:58 DEBUG --- http response --- 17:56:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:56:58 DEBUG --- http response --- 17:56:58 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:56:58 DEBUG --- status code --- 17:56:58 DEBUG --- http response --- 17:56:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:56:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:56:58 DEBUG --- http response --- 17:56:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:56:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:56:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:56:58 DEBUG --- http response --- 17:56:58 DEBUG --- http response --- 17:56:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:56:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:56:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:56:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:56:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:56:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:56:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:56:58 ERROR --- http response --- 17:56:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:56:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:56:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:56:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:56:58 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:56:58 DEBUG --- http response --- 17:56:58 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:56:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:56:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458218" 17:56:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458218" 17:56:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458218" 17:56:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734458218" 17:56:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458218" 17:56:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458218" 17:56:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734458218" 17:56:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458218" 17:56:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458218" 17:56:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458218" 17:56:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458218" 17:56:58 INFO [http_cmd]: http status code OK 17:56:58 DEBUG --- status code --- 17:56:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:58 DEBUG --- http response --- 17:56:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 17:56:58 DEBUG ----- output ----- 17:56:58 DEBUG { 17:56:58 DEBUG "status": "success", 17:56:58 DEBUG "data": { 17:56:58 DEBUG "resultType": "vector", 17:56:58 DEBUG "result": [ 17:56:58 DEBUG { 17:56:58 DEBUG "metric": { 17:56:58 DEBUG "pod": "ds-cts-0" 17:56:58 DEBUG }, 17:56:58 DEBUG "value": [ 17:56:58 DEBUG 17:56:58 DEBUG 17:56:58 DEBUG 17:56:58 DEBUG [...] 17:56:58 DEBUG 17:56:58 DEBUG 17:56:58 DEBUG 17:56:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 17:56:58 DEBUG }, 17:56:58 DEBUG "value": [ 17:56:58 DEBUG 1734458218, 17:56:58 DEBUG "0.004935460938035512" 17:56:58 DEBUG ] 17:56:58 DEBUG } 17:56:58 INFO [http_cmd]: http status code OK 17:56:58 INFO [http_cmd]: http status code OK 17:56:58 DEBUG ] 17:56:58 DEBUG --- status code --- 17:56:58 DEBUG --- status code --- 17:56:58 DEBUG } 17:56:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:58 DEBUG } 17:56:58 DEBUG --- http response --- 17:56:58 DEBUG --- http response --- 17:56:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 17:56:58 DEBUG ----- output ----- 17:56:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 17:56:58 DEBUG { 17:56:58 DEBUG ----- output ----- 17:56:58 DEBUG "status": "success", 17:56:58 DEBUG { 17:56:58 DEBUG "data": { 17:56:58 DEBUG "status": "success", 17:56:58 DEBUG "resultType": "vector", 17:56:58 DEBUG "data": { 17:56:58 DEBUG "result": [ 17:56:58 DEBUG "resultType": "vector", 17:56:58 DEBUG { 17:56:58 DEBUG "result": [ 17:56:58 DEBUG "metric": { 17:56:58 DEBUG { 17:56:58 DEBUG "pod": "ds-cts-0" 17:56:58 DEBUG "metric": { 17:56:58 DEBUG }, 17:56:58 DEBUG "pod": "ds-cts-0" 17:56:58 DEBUG "value": [ 17:56:58 DEBUG }, 17:56:58 DEBUG 17:56:58 DEBUG "value": [ 17:56:58 DEBUG 17:56:58 DEBUG 17:56:58 DEBUG 17:56:58 DEBUG 17:56:58 DEBUG [...] 17:56:58 DEBUG 17:56:58 DEBUG 17:56:58 DEBUG [...] 17:56:58 DEBUG 17:56:58 DEBUG 17:56:58 DEBUG 17:56:58 DEBUG 17:56:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 17:56:58 DEBUG 17:56:58 DEBUG }, 17:56:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 17:56:58 DEBUG "value": [ 17:56:58 DEBUG }, 17:56:58 DEBUG 1734458218, 17:56:58 DEBUG "value": [ 17:56:58 DEBUG "0" 17:56:58 DEBUG 1734458218, 17:56:58 DEBUG ] 17:56:58 DEBUG "1229.1225226727436" 17:56:58 DEBUG } 17:56:58 DEBUG ] 17:56:58 DEBUG ] 17:56:58 DEBUG } 17:56:58 DEBUG } 17:56:58 DEBUG ] 17:56:58 DEBUG } 17:56:58 DEBUG } 17:56:58 DEBUG } 17:56:58 INFO [http_cmd]: http status code OK 17:56:58 DEBUG --- status code --- 17:56:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:58 DEBUG --- http response --- 17:56:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:56:58 DEBUG ----- output ----- 17:56:58 DEBUG { 17:56:58 DEBUG "status": "success", 17:56:58 DEBUG "data": { 17:56:58 DEBUG "resultType": "vector", 17:56:58 DEBUG "result": [ 17:56:58 DEBUG { 17:56:58 DEBUG "metric": { 17:56:58 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 17:56:58 DEBUG }, 17:56:58 DEBUG "value": [ 17:56:58 DEBUG 17:56:58 DEBUG 17:56:58 DEBUG 17:56:58 DEBUG [...] 17:56:58 DEBUG 17:56:58 DEBUG 17:56:58 DEBUG 17:56:58 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 17:56:58 DEBUG }, 17:56:58 DEBUG "value": [ 17:56:58 DEBUG 1734458218, 17:56:58 DEBUG "2642972672" 17:56:58 DEBUG ] 17:56:58 DEBUG } 17:56:58 DEBUG ] 17:56:58 INFO [http_cmd]: http status code OK 17:56:58 INFO [http_cmd]: http status code OK 17:56:58 INFO [http_cmd]: http status code OK 17:56:58 INFO [http_cmd]: http status code OK 17:56:58 DEBUG } 17:56:58 INFO [http_cmd]: http status code OK 17:56:58 DEBUG --- status code --- 17:56:58 DEBUG --- status code --- 17:56:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:58 DEBUG --- http response --- 17:56:58 DEBUG --- status code --- 17:56:58 DEBUG --- status code --- 17:56:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:58 DEBUG --- status code --- 17:56:58 DEBUG } 17:56:58 INFO [http_cmd]: http status code OK 17:56:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:56:58 INFO [http_cmd]: http status code OK 17:56:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:58 DEBUG --- http response --- 17:56:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:58 DEBUG --- status code --- 17:56:58 DEBUG ----- output ----- 17:56:58 DEBUG --- status code --- 17:56:58 DEBUG --- http response --- 17:56:58 DEBUG --- http response --- 17:56:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:56:58 DEBUG --- http response --- 17:56:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:58 DEBUG { 17:56:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:56:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:56:58 DEBUG ----- output ----- 17:56:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:56:58 DEBUG --- http response --- 17:56:58 DEBUG "status": "success", 17:56:58 DEBUG --- http response --- 17:56:58 DEBUG ----- output ----- 17:56:58 DEBUG ----- output ----- 17:56:58 DEBUG { 17:56:58 DEBUG ----- output ----- 17:56:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 17:56:58 DEBUG "data": { 17:56:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:56:58 DEBUG { 17:56:58 DEBUG { 17:56:58 DEBUG "status": "success", 17:56:58 DEBUG { 17:56:58 DEBUG ----- output ----- 17:56:58 DEBUG "resultType": "vector", 17:56:58 DEBUG ----- output ----- 17:56:58 DEBUG "status": "success", 17:56:58 DEBUG "status": "success", 17:56:58 DEBUG "data": { 17:56:58 DEBUG "status": "success", 17:56:58 DEBUG { 17:56:58 DEBUG "result": [ 17:56:58 DEBUG { 17:56:58 DEBUG "data": { 17:56:58 DEBUG "data": { 17:56:58 DEBUG "resultType": "vector", 17:56:58 DEBUG "data": { 17:56:58 DEBUG "status": "success", 17:56:58 DEBUG { 17:56:58 DEBUG "status": "success", 17:56:58 DEBUG "resultType": "vector", 17:56:58 DEBUG "resultType": "vector", 17:56:58 DEBUG "result": [ 17:56:58 DEBUG "resultType": "vector", 17:56:58 DEBUG "data": { 17:56:58 DEBUG "metric": { 17:56:58 DEBUG "data": { 17:56:58 DEBUG "result": [ 17:56:58 DEBUG "result": [ 17:56:58 DEBUG { 17:56:58 DEBUG "result": [ 17:56:58 DEBUG "resultType": "vector", 17:56:58 DEBUG "instance": "10.142.0.11:9100" 17:56:58 DEBUG "resultType": "vector", 17:56:58 DEBUG { 17:56:58 DEBUG { 17:56:58 DEBUG "metric": { 17:56:58 DEBUG { 17:56:58 DEBUG "result": [ 17:56:58 DEBUG }, 17:56:58 DEBUG "result": [ 17:56:58 DEBUG "metric": { 17:56:58 DEBUG "metric": { 17:56:58 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 17:56:58 DEBUG "metric": { 17:56:58 DEBUG { 17:56:58 DEBUG "value": [ 17:56:58 DEBUG { 17:56:58 DEBUG "instance": "10.142.0.11:9100" 17:56:58 DEBUG "instance": "10.142.0.11:9100" 17:56:58 DEBUG }, 17:56:58 DEBUG "instance": "10.142.0.11:9100" 17:56:58 DEBUG "metric": { 17:56:58 DEBUG 17:56:58 DEBUG "metric": { 17:56:58 DEBUG }, 17:56:58 DEBUG }, 17:56:58 DEBUG "value": [ 17:56:58 DEBUG }, 17:56:58 DEBUG "pod": "ds-cts-0" 17:56:58 DEBUG 17:56:58 DEBUG "instance": "10.142.0.11:9100" 17:56:58 DEBUG "value": [ 17:56:58 DEBUG "value": [ 17:56:58 DEBUG 17:56:58 DEBUG "value": [ 17:56:58 DEBUG }, 17:56:58 DEBUG 17:56:58 DEBUG }, 17:56:58 DEBUG 17:56:58 DEBUG 17:56:58 DEBUG 17:56:58 DEBUG 17:56:58 DEBUG "value": [ 17:56:58 DEBUG [...] 17:56:58 DEBUG "value": [ 17:56:58 DEBUG 17:56:58 DEBUG 17:56:58 DEBUG 17:56:58 DEBUG [...] 17:56:58 DEBUG 17:56:58 DEBUG 17:56:58 DEBUG 17:56:58 DEBUG 17:56:58 DEBUG 17:56:58 DEBUG 17:56:58 DEBUG 17:56:58 DEBUG 17:56:58 DEBUG 17:56:58 DEBUG 17:56:58 DEBUG [...] 17:56:58 DEBUG [...] 17:56:58 DEBUG 17:56:58 DEBUG 17:56:58 DEBUG 17:56:58 DEBUG 17:56:58 DEBUG 17:56:58 DEBUG 17:56:58 DEBUG 17:56:58 DEBUG [...] 17:56:58 DEBUG 17:56:58 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 17:56:58 DEBUG "instance": "10.142.0.51:9100" 17:56:58 DEBUG [...] 17:56:58 DEBUG 17:56:58 DEBUG 17:56:58 DEBUG 17:56:58 DEBUG [...] 17:56:58 DEBUG }, 17:56:58 DEBUG }, 17:56:58 DEBUG 17:56:58 DEBUG 17:56:58 DEBUG 17:56:58 DEBUG "instance": "10.142.0.51:9100" 17:56:58 DEBUG 17:56:58 DEBUG "value": [ 17:56:58 DEBUG "value": [ 17:56:58 DEBUG 17:56:58 DEBUG "instance": "10.142.0.51:9100" 17:56:58 DEBUG 17:56:58 DEBUG }, 17:56:58 DEBUG 17:56:58 DEBUG 1734458218, 17:56:58 DEBUG 1734458218, 17:56:58 DEBUG 17:56:58 DEBUG }, 17:56:58 DEBUG 17:56:58 DEBUG "value": [ 17:56:58 DEBUG 17:56:58 DEBUG "1.0704942834391287" 17:56:58 DEBUG "0.0003333333333333854" 17:56:58 DEBUG "instance": "10.142.0.51:9100" 17:56:58 DEBUG "value": [ 17:56:58 DEBUG "instance": "10.142.0.51:9100" 17:56:58 DEBUG 1734458218, 17:56:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 17:56:58 DEBUG ] 17:56:58 DEBUG ] 17:56:58 DEBUG }, 17:56:58 DEBUG 1734458218, 17:56:58 DEBUG "35498.666666666664" 17:56:58 DEBUG ] 17:56:58 DEBUG }, 17:56:58 DEBUG } 17:56:58 DEBUG } 17:56:58 DEBUG "value": [ 17:56:58 DEBUG }, 17:56:58 DEBUG "0.00046666666666700016" 17:56:58 DEBUG } 17:56:58 DEBUG "value": [ 17:56:58 DEBUG ] 17:56:58 DEBUG ] 17:56:58 DEBUG 1734458218, 17:56:58 DEBUG "value": [ 17:56:58 DEBUG ] 17:56:58 DEBUG ] 17:56:58 DEBUG 1734458218, 17:56:58 DEBUG } 17:56:58 DEBUG } 17:56:58 DEBUG "128596.09999999998" 17:56:58 DEBUG 1734458218, 17:56:58 DEBUG } 17:56:58 DEBUG } 17:56:58 DEBUG "694931456" 17:56:58 DEBUG } 17:56:58 DEBUG } 17:56:58 DEBUG ] 17:56:58 DEBUG "132669.46666666667" 17:56:58 DEBUG ] 17:56:58 DEBUG } 17:56:58 DEBUG ] 17:56:58 DEBUG } 17:56:58 DEBUG ] 17:56:58 DEBUG } 17:56:58 DEBUG } 17:56:58 DEBUG ] 17:56:58 DEBUG } 17:56:58 DEBUG } 17:56:58 DEBUG ] 17:56:58 DEBUG } 17:56:58 DEBUG ] 17:56:58 DEBUG } 17:56:58 DEBUG } 17:56:58 DEBUG } 17:56:58 DEBUG } 17:56:58 DEBUG } 17:56:58 INFO 17:56:58 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 17:56:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:56:58 INFO 17:56:58 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 17:56:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:56:58 INFO [loop_until]: OK (rc = 0) 17:56:58 DEBUG --- stdout --- 17:56:58 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 8m 414Mi ds-idrepo-0 9m 598Mi rcs-5d696ff5b-7ttvr 3m 292Mi rcs-5d696ff5b-wwlbr 4m 331Mi 17:56:58 DEBUG --- stderr --- 17:56:58 DEBUG 17:56:58 INFO [loop_until]: OK (rc = 0) 17:56:58 DEBUG --- stdout --- 17:56:58 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 105m 0% 2281Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1144m 3% 3557Mi 12% 17:56:58 DEBUG --- stderr --- 17:56:58 DEBUG 17:57:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458278" 17:57:58 INFO [http_cmd]: http status code OK 17:57:58 DEBUG --- status code --- 17:57:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:58 DEBUG --- http response --- 17:57:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:57:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734458278" 17:57:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458278" 17:57:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458278" 17:57:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458278" 17:57:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458278" 17:57:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458278" 17:57:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458278" 17:57:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458278" 17:57:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734458278" 17:57:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458278" 17:57:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458278" 17:57:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458278" 17:57:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458278" 17:57:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458278" 17:57:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734458278" 17:57:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458278" 17:57:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458278" 17:57:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734458278" 17:57:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458278" 17:57:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458278" 17:57:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458278" 17:57:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458278" 17:57:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734458278" 17:57:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458278" 17:57:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458278" 17:57:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458278" 17:57:58 INFO [http_cmd]: http status code OK 17:57:58 INFO [http_cmd]: http status code OK 17:57:58 INFO [http_cmd]: http status code OK 17:57:58 INFO [http_cmd]: http status code OK 17:57:58 INFO [http_cmd]: http status code OK 17:57:58 INFO [http_cmd]: http status code OK 17:57:58 INFO [http_cmd]: http status code OK 17:57:58 INFO [http_cmd]: http status code OK 17:57:58 INFO [http_cmd]: http status code OK 17:57:58 INFO [http_cmd]: http status code OK 17:57:58 INFO [http_cmd]: http status code OK 17:57:58 INFO [http_cmd]: http status code OK 17:57:58 INFO [http_cmd]: http status code OK 17:57:58 INFO [http_cmd]: http status code OK 17:57:58 INFO [http_cmd]: http status code OK 17:57:58 INFO [http_cmd]: http status code OK 17:57:58 DEBUG --- status code --- 17:57:58 INFO [http_cmd]: http status code OK 17:57:58 DEBUG --- status code --- 17:57:58 DEBUG --- status code --- 17:57:58 DEBUG --- status code --- 17:57:58 INFO [http_cmd]: http status code OK 17:57:58 DEBUG --- status code --- 17:57:58 DEBUG --- status code --- 17:57:58 DEBUG --- status code --- 17:57:58 DEBUG --- status code --- 17:57:58 DEBUG --- status code --- 17:57:58 DEBUG --- status code --- 17:57:58 DEBUG --- status code --- 17:57:58 DEBUG --- status code --- 17:57:58 DEBUG --- status code --- 17:57:58 DEBUG --- status code --- 17:57:58 DEBUG --- status code --- 17:57:58 DEBUG --- status code --- 17:57:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:58 DEBUG --- http response --- 17:57:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:58 DEBUG --- status code --- 17:57:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:58 DEBUG --- status code --- 17:57:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:57:58 DEBUG --- http response --- 17:57:58 DEBUG --- http response --- 17:57:58 DEBUG --- http response --- 17:57:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:58 DEBUG --- http response --- 17:57:58 DEBUG --- http response --- 17:57:58 DEBUG --- http response --- 17:57:58 DEBUG --- http response --- 17:57:58 DEBUG --- http response --- 17:57:58 DEBUG --- http response --- 17:57:58 DEBUG --- http response --- 17:57:58 DEBUG --- http response --- 17:57:58 DEBUG --- http response --- 17:57:58 DEBUG --- http response --- 17:57:58 DEBUG --- http response --- 17:57:58 INFO [http_cmd]: http status code OK 17:57:58 DEBUG --- http response --- 17:57:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:57:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:57:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:57:58 DEBUG --- http response --- 17:57:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:57:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:57:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:57:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:57:58 INFO [http_cmd]: http status code OK 17:57:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:57:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:57:58 ERROR [http_cmd]: ERROR 17:57:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:57:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:57:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:57:58 INFO [http_cmd]: http status code OK 17:57:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:57:58 INFO [http_cmd]: http status code OK 17:57:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:57:58 DEBUG --- status code --- 17:57:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:57:58 DEBUG --- http response --- 17:57:58 INFO [http_cmd]: http status code OK 17:57:58 INFO [http_cmd]: http status code OK 17:57:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:57:58 INFO [http_cmd]: http status code OK 17:57:58 DEBUG --- status code --- 17:57:58 ERROR --- status code --- 17:57:58 DEBUG --- status code --- 17:57:58 DEBUG --- status code --- 17:57:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:57:58 DEBUG --- status code --- 17:57:58 DEBUG --- status code --- 17:57:58 DEBUG --- status code --- 17:57:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:58 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:57:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:58 DEBUG --- http response --- 17:57:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:58 DEBUG --- http response --- 17:57:58 DEBUG --- http response --- 17:57:58 DEBUG --- http response --- 17:57:58 DEBUG --- http response --- 17:57:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:57:58 DEBUG --- http response --- 17:57:58 DEBUG --- http response --- 17:57:58 ERROR --- http response --- 17:57:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:57:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:57:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:57:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:57:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:57:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:57:58 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:57:58 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:57:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458278" 17:57:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458278" 17:57:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458278" 17:57:58 INFO [http_cmd]: http status code OK 17:57:58 DEBUG --- status code --- 17:57:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:58 DEBUG --- http response --- 17:57:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 17:57:58 DEBUG ----- output ----- 17:57:58 DEBUG { 17:57:58 DEBUG "status": "success", 17:57:58 DEBUG "data": { 17:57:58 DEBUG "resultType": "vector", 17:57:58 DEBUG "result": [ 17:57:58 DEBUG { 17:57:58 DEBUG "metric": { 17:57:58 DEBUG "pod": "ds-cts-0" 17:57:58 DEBUG }, 17:57:58 DEBUG "value": [ 17:57:58 DEBUG 17:57:58 DEBUG 17:57:58 DEBUG 17:57:58 DEBUG [...] 17:57:58 DEBUG 17:57:58 DEBUG 17:57:58 DEBUG 17:57:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 17:57:58 DEBUG }, 17:57:58 DEBUG "value": [ 17:57:58 DEBUG 1734458278, 17:57:58 DEBUG "0.008015026163510953" 17:57:58 DEBUG ] 17:57:58 DEBUG } 17:57:58 DEBUG ] 17:57:58 DEBUG } 17:57:58 DEBUG } 17:57:58 INFO [http_cmd]: http status code OK 17:57:58 DEBUG --- status code --- 17:57:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:58 DEBUG --- http response --- 17:57:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 17:57:58 DEBUG ----- output ----- 17:57:58 DEBUG { 17:57:58 DEBUG "status": "success", 17:57:58 DEBUG "data": { 17:57:58 DEBUG "resultType": "vector", 17:57:58 DEBUG "result": [ 17:57:58 DEBUG { 17:57:58 DEBUG "metric": { 17:57:58 DEBUG "pod": "ds-cts-0" 17:57:58 DEBUG }, 17:57:58 DEBUG "value": [ 17:57:58 DEBUG 17:57:58 DEBUG 17:57:58 DEBUG 17:57:58 DEBUG [...] 17:57:58 DEBUG 17:57:58 INFO [http_cmd]: http status code OK 17:57:58 DEBUG 17:57:58 DEBUG --- status code --- 17:57:58 DEBUG 17:57:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 17:57:58 DEBUG --- http response --- 17:57:58 DEBUG }, 17:57:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 17:57:58 DEBUG "value": [ 17:57:58 DEBUG ----- output ----- 17:57:58 DEBUG 1734458278, 17:57:58 DEBUG { 17:57:58 DEBUG "0" 17:57:58 DEBUG "status": "success", 17:57:58 DEBUG ] 17:57:58 DEBUG "data": { 17:57:58 DEBUG } 17:57:58 DEBUG "resultType": "vector", 17:57:58 DEBUG ] 17:57:58 DEBUG "result": [ 17:57:58 DEBUG } 17:57:58 DEBUG { 17:57:58 DEBUG } 17:57:58 DEBUG "metric": { 17:57:58 DEBUG "pod": "ds-cts-0" 17:57:58 DEBUG }, 17:57:58 DEBUG "value": [ 17:57:58 DEBUG 17:57:58 DEBUG 17:57:58 DEBUG 17:57:58 DEBUG [...] 17:57:58 DEBUG 17:57:58 DEBUG 17:57:58 DEBUG 17:57:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 17:57:58 DEBUG }, 17:57:58 DEBUG "value": [ 17:57:58 DEBUG 1734458278, 17:57:58 DEBUG "1230.1537613756186" 17:57:58 DEBUG ] 17:57:58 DEBUG } 17:57:58 DEBUG ] 17:57:58 DEBUG } 17:57:58 DEBUG } 17:57:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734458278" 17:57:58 INFO [http_cmd]: http status code OK 17:57:58 DEBUG --- status code --- 17:57:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:58 DEBUG --- http response --- 17:57:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:57:58 DEBUG ----- output ----- 17:57:58 DEBUG { 17:57:58 DEBUG "status": "success", 17:57:58 DEBUG "data": { 17:57:58 DEBUG "resultType": "vector", 17:57:58 DEBUG "result": [ 17:57:58 DEBUG { 17:57:58 DEBUG "metric": { 17:57:58 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 17:57:58 DEBUG }, 17:57:58 DEBUG "value": [ 17:57:58 DEBUG 17:57:58 DEBUG 17:57:58 DEBUG 17:57:58 DEBUG [...] 17:57:58 DEBUG 17:57:58 DEBUG 17:57:58 DEBUG 17:57:58 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 17:57:58 DEBUG }, 17:57:58 DEBUG "value": [ 17:57:58 DEBUG 1734458278, 17:57:58 DEBUG "2656702464" 17:57:58 DEBUG ] 17:57:58 DEBUG } 17:57:58 DEBUG ] 17:57:58 DEBUG } 17:57:58 DEBUG } 17:57:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458278" 17:57:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734458278" 17:57:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458278" 17:57:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458278" 17:57:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458278" 17:57:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458278" 17:57:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458278" 17:57:58 INFO [http_cmd]: http status code OK 17:57:58 DEBUG --- status code --- 17:57:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:58 DEBUG --- http response --- 17:57:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:57:58 DEBUG ----- output ----- 17:57:58 DEBUG { 17:57:58 DEBUG "status": "success", 17:57:58 DEBUG "data": { 17:57:58 DEBUG "resultType": "vector", 17:57:58 DEBUG "result": [ 17:57:58 DEBUG { 17:57:58 DEBUG "metric": { 17:57:58 DEBUG "instance": "10.142.0.11:9100" 17:57:58 DEBUG }, 17:57:58 DEBUG "value": [ 17:57:58 DEBUG 17:57:58 DEBUG 17:57:58 DEBUG 17:57:58 INFO [http_cmd]: http status code OK 17:57:58 DEBUG [...] 17:57:58 DEBUG --- status code --- 17:57:58 DEBUG 17:57:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:58 DEBUG --- http response --- 17:57:58 DEBUG 17:57:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:57:58 DEBUG 17:57:58 INFO [http_cmd]: http status code OK 17:57:58 INFO [http_cmd]: http status code OK 17:57:58 DEBUG --- status code --- 17:57:58 INFO [http_cmd]: http status code OK 17:57:58 DEBUG "instance": "10.142.0.51:9100" 17:57:58 DEBUG --- status code --- 17:57:58 DEBUG ----- output ----- 17:57:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:58 DEBUG --- status code --- 17:57:58 DEBUG }, 17:57:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:58 DEBUG { 17:57:58 DEBUG --- http response --- 17:57:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:58 DEBUG --- http response --- 17:57:58 DEBUG --- http response --- 17:57:58 DEBUG "status": "success", 17:57:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:57:58 DEBUG "value": [ 17:57:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:57:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:57:58 INFO [http_cmd]: http status code OK 17:57:58 INFO [http_cmd]: http status code OK 17:57:58 DEBUG "data": { 17:57:58 DEBUG ----- output ----- 17:57:58 DEBUG 1734458278, 17:57:58 DEBUG ----- output ----- 17:57:58 DEBUG ----- output ----- 17:57:58 DEBUG --- status code --- 17:57:58 DEBUG --- status code --- 17:57:58 DEBUG "resultType": "vector", 17:57:58 DEBUG { 17:57:58 DEBUG "0" 17:57:58 DEBUG { 17:57:58 DEBUG { 17:57:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:58 DEBUG "result": [ 17:57:58 DEBUG "status": "success", 17:57:58 DEBUG ] 17:57:58 DEBUG "status": "success", 17:57:58 DEBUG "status": "success", 17:57:58 DEBUG --- http response --- 17:57:58 DEBUG --- http response --- 17:57:58 DEBUG { 17:57:58 DEBUG "data": { 17:57:58 DEBUG } 17:57:58 DEBUG "data": { 17:57:58 DEBUG "data": { 17:57:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 17:57:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:57:58 DEBUG "metric": { 17:57:58 DEBUG "resultType": "vector", 17:57:58 DEBUG ] 17:57:58 DEBUG "resultType": "vector", 17:57:58 DEBUG "resultType": "vector", 17:57:58 DEBUG ----- output ----- 17:57:58 DEBUG ----- output ----- 17:57:58 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 17:57:58 DEBUG "result": [ 17:57:58 DEBUG } 17:57:58 DEBUG "result": [ 17:57:58 DEBUG "result": [ 17:57:58 DEBUG { 17:57:58 DEBUG { 17:57:58 DEBUG "status": "success", 17:57:58 DEBUG "data": { 17:57:58 DEBUG } 17:57:58 DEBUG { 17:57:58 DEBUG { 17:57:58 DEBUG "status": "success", 17:57:58 DEBUG "data": { 17:57:58 DEBUG "resultType": "vector", 17:57:58 DEBUG "resultType": "vector", 17:57:58 DEBUG "metric": { 17:57:58 DEBUG "metric": { 17:57:58 DEBUG "instance": "10.142.0.11:9100" 17:57:58 DEBUG }, 17:57:58 DEBUG "result": [ 17:57:58 DEBUG "result": [ 17:57:58 DEBUG "instance": "10.142.0.11:9100" 17:57:58 DEBUG }, 17:57:58 DEBUG "value": [ 17:57:58 DEBUG "value": [ 17:57:58 DEBUG { 17:57:58 DEBUG { 17:57:58 DEBUG }, 17:57:58 DEBUG { 17:57:58 DEBUG 17:57:58 DEBUG 17:57:58 DEBUG "metric": { 17:57:58 DEBUG "metric": { 17:57:58 DEBUG "value": [ 17:57:58 DEBUG "metric": { 17:57:58 DEBUG 17:57:58 DEBUG 17:57:58 DEBUG "pod": "ds-cts-0" 17:57:58 DEBUG "instance": "10.142.0.11:9100" 17:57:58 DEBUG 17:57:58 DEBUG "instance": "10.142.0.11:9100" 17:57:58 DEBUG 17:57:58 DEBUG 17:57:58 DEBUG }, 17:57:58 DEBUG }, 17:57:58 DEBUG 17:57:58 DEBUG }, 17:57:58 DEBUG [...] 17:57:58 DEBUG [...] 17:57:58 DEBUG "value": [ 17:57:58 DEBUG "value": [ 17:57:58 DEBUG 17:57:58 DEBUG "value": [ 17:57:58 DEBUG 17:57:58 DEBUG 17:57:58 DEBUG 17:57:58 DEBUG 17:57:58 DEBUG [...] 17:57:58 DEBUG 17:57:58 DEBUG 17:57:58 DEBUG 17:57:58 DEBUG 17:57:58 DEBUG 17:57:58 DEBUG 17:57:58 DEBUG 17:57:58 DEBUG 17:57:58 DEBUG 17:57:58 DEBUG 17:57:58 DEBUG 17:57:58 DEBUG 17:57:58 DEBUG 17:57:58 DEBUG "instance": "10.142.0.51:9100" 17:57:58 DEBUG "instance": "10.142.0.51:9100" 17:57:58 DEBUG [...] 17:57:58 DEBUG [...] 17:57:58 DEBUG 17:57:58 DEBUG 17:57:58 DEBUG }, 17:57:58 DEBUG }, 17:57:58 DEBUG 17:57:58 DEBUG 17:57:58 DEBUG [...] 17:57:58 DEBUG 17:57:58 DEBUG "value": [ 17:57:58 DEBUG "value": [ 17:57:58 DEBUG 17:57:58 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 17:57:58 DEBUG 17:57:58 DEBUG "instance": "10.142.0.51:9100" 17:57:58 DEBUG 1734458278, 17:57:58 DEBUG 1734458278, 17:57:58 DEBUG 17:57:58 DEBUG }, 17:57:58 DEBUG 17:57:58 DEBUG }, 17:57:58 DEBUG "123244.13333333333" 17:57:58 DEBUG "126781.13333333335" 17:57:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 17:57:58 DEBUG }, 17:57:58 DEBUG "value": [ 17:57:58 DEBUG "value": [ 17:57:58 DEBUG ] 17:57:58 DEBUG ] 17:57:58 DEBUG "value": [ 17:57:58 DEBUG 17:57:58 DEBUG 1734458278, 17:57:58 DEBUG 1734458278, 17:57:58 DEBUG } 17:57:58 DEBUG } 17:57:58 DEBUG 1734458278, 17:57:58 DEBUG "instance": "10.142.0.51:9100" 17:57:58 DEBUG "695128064" 17:57:58 DEBUG "0.0005333333333363536" 17:57:58 DEBUG ] 17:57:58 DEBUG ] 17:57:58 DEBUG "1.0777588727466998" 17:57:58 DEBUG }, 17:57:58 DEBUG ] 17:57:58 DEBUG ] 17:57:58 DEBUG } 17:57:58 DEBUG } 17:57:58 DEBUG ] 17:57:58 DEBUG "value": [ 17:57:58 DEBUG } 17:57:58 DEBUG } 17:57:58 DEBUG } 17:57:58 DEBUG } 17:57:58 DEBUG } 17:57:58 DEBUG 1734458278, 17:57:58 DEBUG ] 17:57:58 DEBUG ] 17:57:58 DEBUG ] 17:57:58 DEBUG "24849.066666666666" 17:57:58 DEBUG } 17:57:58 DEBUG } 17:57:58 DEBUG } 17:57:58 DEBUG ] 17:57:58 DEBUG } 17:57:58 DEBUG } 17:57:58 DEBUG } 17:57:58 DEBUG } 17:57:58 DEBUG ] 17:57:58 DEBUG } 17:57:58 DEBUG } 17:57:58 INFO 17:57:58 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 17:57:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:57:58 INFO 17:57:58 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 17:57:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:57:58 INFO [loop_until]: OK (rc = 0) 17:57:58 DEBUG --- stdout --- 17:57:58 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 8m 419Mi ds-idrepo-0 10m 603Mi rcs-5d696ff5b-7ttvr 12m 293Mi rcs-5d696ff5b-wwlbr 4m 331Mi 17:57:58 DEBUG --- stderr --- 17:57:58 DEBUG 17:57:58 INFO [loop_until]: OK (rc = 0) 17:57:58 DEBUG --- stdout --- 17:57:58 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 103m 0% 2219Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1146m 3% 3566Mi 12% 17:57:58 DEBUG --- stderr --- 17:57:58 DEBUG 17:58:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458338" 17:58:58 INFO [http_cmd]: http status code OK 17:58:58 DEBUG --- status code --- 17:58:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:58 DEBUG --- http response --- 17:58:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:58:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734458338" 17:58:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458338" 17:58:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458338" 17:58:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458338" 17:58:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458338" 17:58:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458338" 17:58:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458338" 17:58:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458338" 17:58:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734458338" 17:58:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458338" 17:58:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458338" 17:58:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458338" 17:58:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458338" 17:58:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458338" 17:58:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734458338" 17:58:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458338" 17:58:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458338" 17:58:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458338" 17:58:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458338" 17:58:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734458338" 17:58:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458338" 17:58:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458338" 17:58:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734458338" 17:58:58 INFO [http_cmd]: http status code OK 17:58:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458338" 17:58:58 INFO [http_cmd]: http status code OK 17:58:58 INFO [http_cmd]: http status code OK 17:58:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458338" 17:58:58 INFO [http_cmd]: http status code OK 17:58:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458338" 17:58:58 INFO [http_cmd]: http status code OK 17:58:58 INFO [http_cmd]: http status code OK 17:58:58 INFO [http_cmd]: http status code OK 17:58:58 INFO [http_cmd]: http status code OK 17:58:58 INFO [http_cmd]: http status code OK 17:58:58 INFO [http_cmd]: http status code OK 17:58:58 INFO [http_cmd]: http status code OK 17:58:58 INFO [http_cmd]: http status code OK 17:58:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458338" 17:58:58 DEBUG --- status code --- 17:58:58 INFO [http_cmd]: http status code OK 17:58:58 INFO [http_cmd]: http status code OK 17:58:58 INFO [http_cmd]: http status code OK 17:58:58 INFO [http_cmd]: http status code OK 17:58:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458338" 17:58:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458338" 17:58:58 DEBUG --- status code --- 17:58:58 DEBUG --- status code --- 17:58:58 DEBUG --- status code --- 17:58:58 DEBUG --- status code --- 17:58:58 DEBUG --- status code --- 17:58:58 DEBUG --- status code --- 17:58:58 DEBUG --- status code --- 17:58:58 DEBUG --- status code --- 17:58:58 DEBUG --- status code --- 17:58:58 DEBUG --- status code --- 17:58:58 DEBUG --- status code --- 17:58:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:58 DEBUG --- status code --- 17:58:58 DEBUG --- status code --- 17:58:58 DEBUG --- status code --- 17:58:58 DEBUG --- status code --- 17:58:58 INFO [http_cmd]: http status code OK 17:58:58 INFO [http_cmd]: http status code OK 17:58:58 INFO [http_cmd]: http status code OK 17:58:58 ERROR [http_cmd]: ERROR 17:58:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:58 INFO [http_cmd]: http status code OK 17:58:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:58 INFO [http_cmd]: http status code OK 17:58:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:58 INFO [http_cmd]: http status code OK 17:58:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734458338" 17:58:58 DEBUG --- http response --- 17:58:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:58 INFO [http_cmd]: http status code OK 17:58:58 INFO [http_cmd]: http status code OK 17:58:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:58 INFO [http_cmd]: http status code OK 17:58:58 DEBUG --- status code --- 17:58:58 DEBUG --- status code --- 17:58:58 INFO [http_cmd]: http status code OK 17:58:58 INFO [http_cmd]: http status code OK 17:58:58 DEBUG --- status code --- 17:58:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:58 DEBUG --- http response --- 17:58:58 DEBUG --- http response --- 17:58:58 DEBUG --- http response --- 17:58:58 DEBUG --- status code --- 17:58:58 DEBUG --- http response --- 17:58:58 DEBUG --- http response --- 17:58:58 DEBUG --- status code --- 17:58:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:58 DEBUG --- http response --- 17:58:58 INFO [http_cmd]: http status code OK 17:58:58 DEBUG --- http response --- 17:58:58 DEBUG --- http response --- 17:58:58 DEBUG --- http response --- 17:58:58 DEBUG --- http response --- 17:58:58 DEBUG --- http response --- 17:58:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:58:58 DEBUG --- http response --- 17:58:58 DEBUG --- http response --- 17:58:58 DEBUG --- status code --- 17:58:58 DEBUG --- status code --- 17:58:58 DEBUG --- http response --- 17:58:58 DEBUG --- http response --- 17:58:58 DEBUG --- status code --- 17:58:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:58 DEBUG --- status code --- 17:58:58 DEBUG --- status code --- 17:58:58 ERROR --- status code --- 17:58:58 DEBUG --- status code --- 17:58:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 17:58:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:58:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:58:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:58:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:58:58 DEBUG --- http response --- 17:58:58 DEBUG --- http response --- 17:58:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:58:58 DEBUG --- status code --- 17:58:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:58:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:58:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:58:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:58:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:58:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:58:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:58:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:58:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:58:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:58 DEBUG --- http response --- 17:58:58 DEBUG --- http response --- 17:58:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:58 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:58:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:58 DEBUG ----- output ----- 17:58:58 DEBUG --- http response --- 17:58:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:58:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:58:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:58 INFO [http_cmd]: http status code OK 17:58:58 DEBUG --- http response --- 17:58:58 DEBUG --- http response --- 17:58:58 DEBUG --- http response --- 17:58:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:58:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:58:58 DEBUG --- http response --- 17:58:58 DEBUG --- http response --- 17:58:58 ERROR --- http response --- 17:58:58 DEBUG --- http response --- 17:58:58 DEBUG { 17:58:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:58:58 DEBUG --- http response --- 17:58:58 DEBUG --- status code --- 17:58:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:58:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 17:58:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:58:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:58:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458338" 17:58:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:58:58 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:58:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:58:58 DEBUG "status": "success", 17:58:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 17:58:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:58 DEBUG ----- output ----- 17:58:58 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:58:58 DEBUG "data": { 17:58:58 DEBUG ----- output ----- 17:58:58 DEBUG --- http response --- 17:58:58 DEBUG { 17:58:58 DEBUG "resultType": "vector", 17:58:58 DEBUG { 17:58:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:58:58 DEBUG "status": "success", 17:58:58 DEBUG "result": [ 17:58:58 DEBUG "status": "success", 17:58:58 DEBUG ----- output ----- 17:58:58 DEBUG "data": { 17:58:58 DEBUG { 17:58:58 DEBUG "data": { 17:58:58 DEBUG { 17:58:58 DEBUG "resultType": "vector", 17:58:58 DEBUG "metric": { 17:58:58 DEBUG "resultType": "vector", 17:58:58 DEBUG "status": "success", 17:58:58 DEBUG "result": [ 17:58:58 DEBUG "pod": "ds-cts-0" 17:58:58 DEBUG "result": [ 17:58:58 DEBUG "data": { 17:58:58 DEBUG { 17:58:58 DEBUG }, 17:58:58 DEBUG { 17:58:58 DEBUG "resultType": "vector", 17:58:58 DEBUG "metric": { 17:58:58 DEBUG "value": [ 17:58:58 DEBUG "metric": { 17:58:58 DEBUG "result": [ 17:58:58 DEBUG "pod": "ds-cts-0" 17:58:58 DEBUG 17:58:58 DEBUG "pod": "ds-cts-0" 17:58:58 DEBUG { 17:58:58 DEBUG }, 17:58:58 DEBUG 17:58:58 DEBUG }, 17:58:58 DEBUG "metric": { 17:58:58 DEBUG "value": [ 17:58:58 DEBUG 17:58:58 DEBUG "value": [ 17:58:58 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 17:58:58 DEBUG 17:58:58 DEBUG [...] 17:58:58 DEBUG 17:58:58 DEBUG }, 17:58:58 DEBUG 17:58:58 DEBUG 17:58:58 DEBUG 17:58:58 DEBUG "value": [ 17:58:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458338" 17:58:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458338" 17:58:58 DEBUG 17:58:58 DEBUG 17:58:58 DEBUG 17:58:58 DEBUG 17:58:58 DEBUG [...] 17:58:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458338" 17:58:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734458338" 17:58:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458338" 17:58:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458338" 17:58:58 DEBUG 17:58:58 DEBUG [...] 17:58:58 DEBUG 17:58:58 DEBUG 17:58:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 17:58:58 DEBUG 17:58:58 DEBUG 17:58:58 DEBUG 17:58:58 DEBUG 17:58:58 DEBUG 17:58:58 DEBUG [...] 17:58:58 DEBUG }, 17:58:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 17:58:58 INFO [http_cmd]: http status code OK 17:58:58 INFO [http_cmd]: http status code OK 17:58:58 INFO [http_cmd]: http status code OK 17:58:58 DEBUG 17:58:58 DEBUG 17:58:58 DEBUG "value": [ 17:58:58 DEBUG }, 17:58:58 DEBUG --- status code --- 17:58:58 DEBUG --- status code --- 17:58:58 DEBUG --- status code --- 17:58:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 17:58:58 DEBUG 17:58:58 DEBUG 1734458338, 17:58:58 DEBUG "value": [ 17:58:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:58 DEBUG }, 17:58:58 DEBUG 17:58:58 DEBUG "0" 17:58:58 DEBUG 1734458338, 17:58:58 DEBUG --- http response --- 17:58:58 DEBUG --- http response --- 17:58:58 DEBUG --- http response --- 17:58:58 DEBUG "value": [ 17:58:58 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 17:58:58 DEBUG }, 17:58:58 DEBUG "value": [ 17:58:58 DEBUG 1734458338, 17:58:58 DEBUG "2680967168" 17:58:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:58:58 DEBUG ----- output ----- 17:58:58 DEBUG { 17:58:58 DEBUG "status": "success", 17:58:58 DEBUG "data": { 17:58:58 DEBUG "resultType": "vector", 17:58:58 DEBUG "result": [ 17:58:58 DEBUG { 17:58:58 DEBUG ] 17:58:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:58:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:58:58 DEBUG 1734458338, 17:58:58 DEBUG ] 17:58:58 DEBUG "0.007862596089756691" 17:58:58 INFO [http_cmd]: http status code OK 17:58:58 INFO [http_cmd]: http status code OK 17:58:58 DEBUG "metric": { 17:58:58 DEBUG } 17:58:58 DEBUG ] 17:58:58 DEBUG ----- output ----- 17:58:58 DEBUG "1347.2724460153252" 17:58:58 DEBUG } 17:58:58 DEBUG ] 17:58:58 DEBUG --- status code --- 17:58:58 DEBUG --- status code --- 17:58:58 DEBUG "instance": "10.142.0.11:9100" 17:58:58 DEBUG ----- output ----- 17:58:58 DEBUG } 17:58:58 DEBUG } 17:58:58 INFO [http_cmd]: http status code OK 17:58:58 DEBUG ] 17:58:58 DEBUG } 17:58:58 DEBUG } 17:58:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:58 DEBUG }, 17:58:58 DEBUG { 17:58:58 DEBUG { 17:58:58 DEBUG --- status code --- 17:58:58 INFO [http_cmd]: http status code OK 17:58:58 DEBUG ] 17:58:58 DEBUG ] 17:58:58 DEBUG ] 17:58:58 DEBUG --- http response --- 17:58:58 DEBUG --- http response --- 17:58:58 DEBUG "value": [ 17:58:58 DEBUG "status": "success", 17:58:58 DEBUG "status": "success", 17:58:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:58 DEBUG --- status code --- 17:58:58 DEBUG } 17:58:58 DEBUG } 17:58:58 DEBUG } 17:58:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:58:58 DEBUG 17:58:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 17:58:58 DEBUG "data": { 17:58:58 DEBUG "data": { 17:58:58 DEBUG --- http response --- 17:58:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:58 DEBUG } 17:58:58 DEBUG } 17:58:58 DEBUG } 17:58:58 DEBUG ----- output ----- 17:58:58 DEBUG 17:58:58 DEBUG ----- output ----- 17:58:58 DEBUG "resultType": "vector", 17:58:58 DEBUG "resultType": "vector", 17:58:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:58:58 DEBUG --- http response --- 17:58:58 DEBUG { 17:58:58 DEBUG 17:58:58 DEBUG { 17:58:58 DEBUG "result": [ 17:58:58 DEBUG "result": [ 17:58:58 DEBUG ----- output ----- 17:58:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:58:58 DEBUG "status": "success", 17:58:58 DEBUG [...] 17:58:58 DEBUG "status": "success", 17:58:58 DEBUG { 17:58:58 DEBUG { 17:58:58 DEBUG { 17:58:58 DEBUG ----- output ----- 17:58:58 DEBUG "data": { 17:58:58 DEBUG 17:58:58 DEBUG "data": { 17:58:58 DEBUG "metric": { 17:58:58 DEBUG "metric": { 17:58:58 DEBUG "status": "success", 17:58:58 DEBUG { 17:58:58 DEBUG "resultType": "vector", 17:58:58 DEBUG 17:58:58 DEBUG "resultType": "vector", 17:58:58 DEBUG "instance": "10.142.0.11:9100" 17:58:58 DEBUG "instance": "10.142.0.11:9100" 17:58:58 DEBUG "data": { 17:58:58 DEBUG "status": "success", 17:58:58 DEBUG "result": [ 17:58:58 DEBUG 17:58:58 DEBUG "result": [ 17:58:58 DEBUG }, 17:58:58 DEBUG }, 17:58:58 DEBUG "resultType": "vector", 17:58:58 DEBUG "result": [ 17:58:58 DEBUG { 17:58:58 DEBUG "metric": { 17:58:58 DEBUG "instance": "10.142.0.11:9100" 17:58:58 DEBUG { 17:58:58 DEBUG "value": [ 17:58:58 DEBUG "data": { 17:58:58 DEBUG { 17:58:58 DEBUG "instance": "10.142.0.51:9100" 17:58:58 DEBUG "value": [ 17:58:58 DEBUG }, 17:58:58 DEBUG "metric": { 17:58:58 DEBUG 17:58:58 DEBUG "resultType": "vector", 17:58:58 DEBUG "metric": { 17:58:58 DEBUG }, 17:58:58 DEBUG 17:58:58 DEBUG "value": [ 17:58:58 DEBUG "pod": "ds-cts-0" 17:58:58 DEBUG 17:58:58 DEBUG "result": [ 17:58:58 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 17:58:58 DEBUG "value": [ 17:58:58 DEBUG 17:58:58 DEBUG 17:58:58 DEBUG [...] 17:58:58 DEBUG 17:58:58 DEBUG 17:58:58 DEBUG }, 17:58:58 DEBUG 1734458338, 17:58:58 DEBUG 17:58:58 DEBUG }, 17:58:58 DEBUG { 17:58:58 DEBUG 17:58:58 DEBUG [...] 17:58:58 DEBUG "value": [ 17:58:58 DEBUG "139204.60000000003" 17:58:58 DEBUG 17:58:58 DEBUG "value": [ 17:58:58 DEBUG "metric": { 17:58:58 DEBUG 17:58:58 DEBUG 17:58:58 DEBUG 17:58:58 DEBUG ] 17:58:58 DEBUG 17:58:58 DEBUG 17:58:58 DEBUG "instance": "10.142.0.11:9100" 17:58:58 DEBUG "instance": "10.142.0.51:9100" 17:58:58 DEBUG 17:58:58 DEBUG 17:58:58 DEBUG } 17:58:58 DEBUG [...] 17:58:58 DEBUG 17:58:58 DEBUG }, 17:58:58 DEBUG }, 17:58:58 DEBUG 17:58:58 DEBUG 17:58:58 DEBUG ] 17:58:58 DEBUG 17:58:58 DEBUG 17:58:58 DEBUG "value": [ 17:58:58 DEBUG "value": [ 17:58:58 DEBUG "instance": "10.142.0.51:9100" 17:58:58 DEBUG [...] 17:58:58 DEBUG } 17:58:58 DEBUG 17:58:58 DEBUG [...] 17:58:58 DEBUG 17:58:58 DEBUG 1734458338, 17:58:58 DEBUG }, 17:58:58 DEBUG 17:58:58 DEBUG } 17:58:58 DEBUG 17:58:58 DEBUG 17:58:58 DEBUG 17:58:58 DEBUG "0.0003333333333333854" 17:58:58 DEBUG "value": [ 17:58:58 DEBUG 17:58:58 DEBUG "instance": "10.142.0.51:9100" 17:58:58 DEBUG 17:58:58 DEBUG 17:58:58 DEBUG ] 17:58:58 DEBUG 1734458338, 17:58:58 DEBUG 17:58:58 DEBUG }, 17:58:58 DEBUG 17:58:58 DEBUG [...] 17:58:58 DEBUG } 17:58:58 DEBUG "135554.59999999998" 17:58:58 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 17:58:58 DEBUG "value": [ 17:58:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 17:58:58 DEBUG 17:58:58 DEBUG ] 17:58:58 DEBUG ] 17:58:58 DEBUG }, 17:58:58 DEBUG 1734458338, 17:58:58 DEBUG }, 17:58:58 DEBUG 17:58:58 DEBUG } 17:58:58 DEBUG } 17:58:58 DEBUG "value": [ 17:58:58 DEBUG "0.00036666666666983856" 17:58:58 DEBUG "value": [ 17:58:58 DEBUG 17:58:58 DEBUG } 17:58:58 DEBUG ] 17:58:58 DEBUG 1734458338, 17:58:58 DEBUG ] 17:58:58 DEBUG 1734458338, 17:58:58 DEBUG "instance": "10.142.0.51:9100" 17:58:58 DEBUG } 17:58:58 DEBUG "1.065435068568585" 17:58:58 DEBUG } 17:58:58 DEBUG "695361536" 17:58:58 DEBUG }, 17:58:58 DEBUG } 17:58:58 DEBUG ] 17:58:58 DEBUG ] 17:58:58 DEBUG ] 17:58:58 DEBUG "value": [ 17:58:58 DEBUG } 17:58:58 DEBUG } 17:58:58 DEBUG } 17:58:58 DEBUG 1734458338, 17:58:58 DEBUG ] 17:58:58 DEBUG } 17:58:58 DEBUG ] 17:58:58 DEBUG "19114.666666666668" 17:58:58 DEBUG } 17:58:58 DEBUG } 17:58:58 DEBUG ] 17:58:58 DEBUG } 17:58:58 DEBUG } 17:58:58 DEBUG } 17:58:58 DEBUG ] 17:58:58 DEBUG } 17:58:58 DEBUG } 17:58:58 INFO 17:58:58 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 17:58:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:58:58 INFO 17:58:58 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 17:58:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:58:58 INFO [loop_until]: OK (rc = 0) 17:58:58 DEBUG --- stdout --- 17:58:58 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 10m 423Mi ds-idrepo-0 8m 609Mi rcs-5d696ff5b-7ttvr 5m 293Mi rcs-5d696ff5b-wwlbr 4m 331Mi 17:58:58 DEBUG --- stderr --- 17:58:58 DEBUG 17:58:58 INFO [loop_until]: OK (rc = 0) 17:58:58 DEBUG --- stdout --- 17:58:58 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 106m 0% 2234Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1138m 3% 3577Mi 12% 17:58:58 DEBUG --- stderr --- 17:58:58 DEBUG 17:59:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458398" 17:59:58 INFO [http_cmd]: http status code OK 17:59:58 DEBUG --- status code --- 17:59:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:58 DEBUG --- http response --- 17:59:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:59:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734458398" 17:59:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458398" 17:59:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458398" 17:59:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458398" 17:59:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458398" 17:59:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734458398" 17:59:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458398" 17:59:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458398" 17:59:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458398" 17:59:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458398" 17:59:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458398" 17:59:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734458398" 17:59:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458398" 17:59:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458398" 17:59:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734458398" 17:59:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458398" 17:59:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458398" 17:59:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458398" 17:59:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458398" 17:59:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458398" 17:59:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458398" 17:59:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458398" 17:59:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734458398" 17:59:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458398" 17:59:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458398" 17:59:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458398" 17:59:58 INFO [http_cmd]: http status code OK 17:59:58 INFO [http_cmd]: http status code OK 17:59:58 INFO [http_cmd]: http status code OK 17:59:58 INFO [http_cmd]: http status code OK 17:59:58 INFO [http_cmd]: http status code OK 17:59:58 INFO [http_cmd]: http status code OK 17:59:58 INFO [http_cmd]: http status code OK 17:59:58 INFO [http_cmd]: http status code OK 17:59:58 INFO [http_cmd]: http status code OK 17:59:58 INFO [http_cmd]: http status code OK 17:59:58 INFO [http_cmd]: http status code OK 17:59:58 INFO [http_cmd]: http status code OK 17:59:58 INFO [http_cmd]: http status code OK 17:59:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734458398" 17:59:58 INFO [http_cmd]: http status code OK 17:59:58 INFO [http_cmd]: http status code OK 17:59:58 INFO [http_cmd]: http status code OK 17:59:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458398" 17:59:58 DEBUG --- status code --- 17:59:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458398" 17:59:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458398" 17:59:58 DEBUG --- status code --- 17:59:58 DEBUG --- status code --- 17:59:58 DEBUG --- status code --- 17:59:58 DEBUG --- status code --- 17:59:58 DEBUG --- status code --- 17:59:58 DEBUG --- status code --- 17:59:58 DEBUG --- status code --- 17:59:58 DEBUG --- status code --- 17:59:58 DEBUG --- status code --- 17:59:58 DEBUG --- status code --- 17:59:58 DEBUG --- status code --- 17:59:58 DEBUG --- status code --- 17:59:58 DEBUG --- status code --- 17:59:58 DEBUG --- status code --- 17:59:58 DEBUG --- status code --- 17:59:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:58 INFO [http_cmd]: http status code OK 17:59:58 INFO [http_cmd]: http status code OK 17:59:58 INFO [http_cmd]: http status code OK 17:59:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458398" 17:59:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:58 INFO [http_cmd]: http status code OK 17:59:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:58 INFO [http_cmd]: http status code OK 17:59:58 ERROR [http_cmd]: ERROR 17:59:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458398" 17:59:58 INFO [http_cmd]: http status code OK 17:59:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458398" 17:59:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:58 INFO [http_cmd]: http status code OK 17:59:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458398" 17:59:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:58 INFO [http_cmd]: http status code OK 17:59:58 INFO [http_cmd]: http status code OK 17:59:58 INFO [http_cmd]: http status code OK 17:59:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458398" 17:59:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:58 INFO [http_cmd]: http status code OK 17:59:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734458398" 17:59:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458398" 17:59:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:58 INFO [http_cmd]: http status code OK 17:59:58 DEBUG --- http response --- 17:59:58 INFO [http_cmd]: http status code OK 17:59:58 DEBUG --- http response --- 17:59:58 DEBUG --- status code --- 17:59:58 DEBUG --- status code --- 17:59:58 DEBUG --- status code --- 17:59:58 DEBUG --- http response --- 17:59:58 DEBUG --- status code --- 17:59:58 DEBUG --- http response --- 17:59:58 DEBUG --- http response --- 17:59:58 DEBUG --- http response --- 17:59:58 DEBUG --- http response --- 17:59:58 DEBUG --- http response --- 17:59:58 DEBUG --- http response --- 17:59:58 DEBUG --- status code --- 17:59:58 ERROR --- status code --- 17:59:58 DEBUG --- status code --- 17:59:58 DEBUG --- http response --- 17:59:58 DEBUG --- http response --- 17:59:58 DEBUG --- http response --- 17:59:58 DEBUG --- http response --- 17:59:58 DEBUG --- status code --- 17:59:58 DEBUG --- http response --- 17:59:58 DEBUG --- status code --- 17:59:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:58 DEBUG --- status code --- 17:59:58 DEBUG --- http response --- 17:59:58 DEBUG --- status code --- 17:59:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:58 DEBUG --- http response --- 17:59:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:59:58 DEBUG --- status code --- 17:59:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:59:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:59:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:59:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:59:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:59:58 INFO [http_cmd]: http status code OK 17:59:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:59:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:59:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:59:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:58 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:59:58 ERROR --- http response --- 17:59:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:59:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:59:58 INFO [http_cmd]: http status code OK 17:59:58 INFO [http_cmd]: http status code OK 17:59:58 INFO [http_cmd]: http status code OK 17:59:58 INFO [http_cmd]: http status code OK 17:59:58 DEBUG --- status code --- 17:59:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:58 DEBUG --- http response --- 17:59:58 INFO [http_cmd]: http status code OK 17:59:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:59:58 DEBUG --- status code --- 17:59:58 DEBUG --- http response --- 17:59:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:58 DEBUG --- http response --- 17:59:58 DEBUG --- status code --- 17:59:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 17:59:58 DEBUG ----- output ----- 17:59:58 DEBUG { 17:59:58 DEBUG "status": "success", 17:59:58 DEBUG "data": { 17:59:58 DEBUG "resultType": "vector", 17:59:58 DEBUG "result": [ 17:59:58 DEBUG { 17:59:58 DEBUG --- http response --- 17:59:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:58 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:59:58 DEBUG --- status code --- 17:59:58 DEBUG --- status code --- 17:59:58 DEBUG --- status code --- 17:59:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:59:58 INFO [http_cmd]: http status code OK 17:59:58 DEBUG --- status code --- 17:59:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:59:58 DEBUG --- status code --- 17:59:58 DEBUG --- http response --- 17:59:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:59:58 DEBUG --- http response --- 17:59:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 17:59:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:59:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:58 DEBUG --- http response --- 17:59:58 DEBUG --- http response --- 17:59:58 DEBUG --- http response --- 17:59:58 DEBUG --- http response --- 17:59:58 DEBUG --- status code --- 17:59:58 DEBUG "metric": { 17:59:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:59:58 DEBUG --- http response --- 17:59:58 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:59:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:59:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:58 DEBUG ----- output ----- 17:59:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:58 DEBUG --- http response --- 17:59:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:59:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:59:58 DEBUG ----- output ----- 17:59:58 DEBUG --- http response --- 17:59:58 DEBUG --- http response --- 17:59:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:59:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:59:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:59:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:59:58 DEBUG "pod": "ds-cts-0" 17:59:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:59:58 DEBUG --- http response --- 17:59:58 DEBUG --- http response --- 17:59:58 DEBUG --- http response --- 17:59:58 DEBUG --- http response --- 17:59:58 DEBUG { 17:59:58 DEBUG --- http response --- 17:59:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:59:58 DEBUG { 17:59:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 17:59:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:59:58 DEBUG --- http response --- 17:59:58 DEBUG }, 17:59:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 17:59:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:59:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:59:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:59:58 DEBUG "status": "success", 17:59:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:59:58 DEBUG "status": "success", 17:59:58 DEBUG ----- output ----- 17:59:58 DEBUG ----- output ----- 17:59:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 17:59:58 DEBUG "value": [ 17:59:58 DEBUG ----- output ----- 17:59:58 DEBUG ----- output ----- 17:59:58 DEBUG ----- output ----- 17:59:58 DEBUG ----- output ----- 17:59:58 DEBUG "data": { 17:59:58 DEBUG ----- output ----- 17:59:58 DEBUG "data": { 17:59:58 DEBUG { 17:59:58 DEBUG { 17:59:58 DEBUG ----- output ----- 17:59:58 DEBUG 17:59:58 DEBUG { 17:59:58 DEBUG { 17:59:58 DEBUG { 17:59:58 DEBUG { 17:59:58 DEBUG "resultType": "vector", 17:59:58 DEBUG { 17:59:58 DEBUG "resultType": "vector", 17:59:58 DEBUG "status": "success", 17:59:58 DEBUG "status": "success", 17:59:58 DEBUG { 17:59:58 DEBUG 17:59:58 DEBUG "status": "success", 17:59:58 DEBUG "status": "success", 17:59:58 DEBUG "status": "success", 17:59:58 DEBUG "status": "success", 17:59:58 DEBUG "result": [ 17:59:58 DEBUG "status": "success", 17:59:58 DEBUG "result": [ 17:59:58 DEBUG "data": { 17:59:58 DEBUG "data": { 17:59:58 DEBUG "status": "success", 17:59:58 DEBUG 17:59:58 DEBUG "data": { 17:59:58 DEBUG "data": { 17:59:58 DEBUG "data": { 17:59:58 DEBUG "data": { 17:59:58 DEBUG { 17:59:58 DEBUG "data": { 17:59:58 DEBUG { 17:59:58 DEBUG "resultType": "vector", 17:59:58 DEBUG "resultType": "vector", 17:59:58 DEBUG "data": { 17:59:58 DEBUG [...] 17:59:58 DEBUG "resultType": "vector", 17:59:58 DEBUG "resultType": "vector", 17:59:58 DEBUG "resultType": "vector", 17:59:58 DEBUG "resultType": "vector", 17:59:58 DEBUG "metric": { 17:59:58 DEBUG "resultType": "vector", 17:59:58 DEBUG "metric": { 17:59:58 DEBUG "result": [ 17:59:58 DEBUG "result": [ 17:59:58 DEBUG "resultType": "vector", 17:59:58 DEBUG 17:59:58 DEBUG "result": [ 17:59:58 DEBUG "result": [ 17:59:58 DEBUG "result": [ 17:59:58 DEBUG "result": [ 17:59:58 DEBUG "instance": "10.142.0.11:9100" 17:59:58 DEBUG "result": [ 17:59:58 DEBUG "pod": "ds-cts-0" 17:59:58 DEBUG { 17:59:58 DEBUG { 17:59:58 DEBUG "result": [ 17:59:58 DEBUG 17:59:58 DEBUG { 17:59:58 DEBUG { 17:59:58 DEBUG { 17:59:58 DEBUG { 17:59:58 DEBUG }, 17:59:58 DEBUG { 17:59:58 DEBUG }, 17:59:58 DEBUG "metric": { 17:59:58 DEBUG "metric": { 17:59:58 DEBUG { 17:59:58 DEBUG 17:59:58 DEBUG "metric": { 17:59:58 DEBUG "metric": { 17:59:58 DEBUG "metric": { 17:59:58 DEBUG "metric": { 17:59:58 DEBUG "value": [ 17:59:58 DEBUG "metric": { 17:59:58 DEBUG "value": [ 17:59:58 DEBUG "pod": "ds-cts-0" 17:59:58 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 17:59:58 DEBUG "metric": { 17:59:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 17:59:58 DEBUG "pod": "ds-cts-0" 17:59:58 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 17:59:58 DEBUG "instance": "10.142.0.11:9100" 17:59:58 DEBUG "instance": "10.142.0.11:9100" 17:59:58 DEBUG 17:59:58 DEBUG "instance": "10.142.0.11:9100" 17:59:58 DEBUG 17:59:58 DEBUG }, 17:59:58 DEBUG }, 17:59:58 DEBUG "instance": "10.142.0.11:9100" 17:59:58 DEBUG }, 17:59:58 DEBUG }, 17:59:58 DEBUG }, 17:59:58 DEBUG }, 17:59:58 DEBUG }, 17:59:58 DEBUG 17:59:58 DEBUG }, 17:59:58 DEBUG 17:59:58 DEBUG "value": [ 17:59:58 DEBUG "value": [ 17:59:58 DEBUG }, 17:59:58 DEBUG "value": [ 17:59:58 DEBUG "value": [ 17:59:58 DEBUG "value": [ 17:59:58 DEBUG "value": [ 17:59:58 DEBUG "value": [ 17:59:58 DEBUG 17:59:58 DEBUG "value": [ 17:59:58 DEBUG 17:59:58 DEBUG 17:59:58 DEBUG 17:59:58 DEBUG "value": [ 17:59:58 DEBUG 1734458398, 17:59:58 DEBUG 17:59:58 DEBUG 17:59:58 DEBUG 17:59:58 DEBUG 17:59:58 DEBUG [...] 17:59:58 DEBUG 17:59:58 DEBUG [...] 17:59:58 DEBUG 17:59:58 DEBUG 17:59:58 DEBUG 17:59:58 DEBUG "0.00454555292768962" 17:59:58 DEBUG 17:59:58 DEBUG 17:59:58 DEBUG 17:59:58 DEBUG 17:59:58 DEBUG 17:59:58 DEBUG 17:59:58 DEBUG 17:59:58 DEBUG 17:59:58 DEBUG 17:59:58 DEBUG 17:59:58 DEBUG ] 17:59:58 DEBUG 17:59:58 DEBUG 17:59:58 DEBUG 17:59:58 DEBUG 17:59:58 DEBUG 17:59:58 DEBUG 17:59:58 DEBUG 17:59:58 DEBUG [...] 17:59:58 DEBUG [...] 17:59:58 DEBUG 17:59:58 DEBUG } 17:59:58 DEBUG [...] 17:59:58 DEBUG [...] 17:59:58 DEBUG [...] 17:59:58 DEBUG [...] 17:59:58 DEBUG 17:59:58 DEBUG [...] 17:59:58 DEBUG 17:59:58 DEBUG 17:59:58 DEBUG 17:59:58 DEBUG [...] 17:59:58 DEBUG ] 17:59:58 DEBUG 17:59:58 DEBUG 17:59:58 DEBUG 17:59:58 DEBUG 17:59:58 DEBUG "instance": "10.142.0.51:9100" 17:59:58 DEBUG 17:59:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 17:59:58 DEBUG 17:59:58 DEBUG 17:59:58 DEBUG 17:59:58 DEBUG } 17:59:58 DEBUG 17:59:58 DEBUG 17:59:58 DEBUG 17:59:58 DEBUG 17:59:58 DEBUG }, 17:59:58 DEBUG 17:59:58 DEBUG }, 17:59:58 DEBUG 17:59:58 DEBUG 17:59:58 DEBUG 17:59:58 DEBUG } 17:59:58 DEBUG 17:59:58 DEBUG 17:59:58 DEBUG 17:59:58 DEBUG 17:59:58 DEBUG "value": [ 17:59:58 DEBUG 17:59:58 DEBUG "value": [ 17:59:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 17:59:58 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 17:59:58 DEBUG 17:59:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 17:59:58 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 17:59:58 DEBUG "instance": "10.142.0.51:9100" 17:59:58 DEBUG "instance": "10.142.0.51:9100" 17:59:58 DEBUG 1734458398, 17:59:58 DEBUG "instance": "10.142.0.51:9100" 17:59:58 DEBUG 1734458398, 17:59:58 DEBUG }, 17:59:58 DEBUG }, 17:59:58 DEBUG "instance": "10.142.0.51:9100" 17:59:58 DEBUG }, 17:59:58 DEBUG }, 17:59:58 DEBUG }, 17:59:58 DEBUG }, 17:59:58 DEBUG "15155.199999999999" 17:59:58 DEBUG }, 17:59:58 DEBUG "0" 17:59:58 DEBUG "value": [ 17:59:58 DEBUG "value": [ 17:59:58 DEBUG }, 17:59:58 DEBUG "value": [ 17:59:58 DEBUG "value": [ 17:59:58 DEBUG "value": [ 17:59:58 DEBUG "value": [ 17:59:58 DEBUG ] 17:59:58 DEBUG "value": [ 17:59:58 DEBUG ] 17:59:58 DEBUG 1734458398, 17:59:58 DEBUG 1734458398, 17:59:58 DEBUG "value": [ 17:59:58 DEBUG 1734458398, 17:59:58 DEBUG "134526.43333333332" 17:59:58 DEBUG ] 17:59:58 DEBUG 1734458398, 17:59:58 DEBUG } 17:59:58 DEBUG 1734458398, 17:59:58 DEBUG } 17:59:58 DEBUG "1340.0299804773704" 17:59:58 DEBUG "2701684736" 17:59:58 DEBUG 1734458398, 17:59:58 DEBUG 1734458398, 17:59:58 DEBUG } 17:59:58 DEBUG 1734458398, 17:59:58 DEBUG "130292.56666666664" 17:59:58 DEBUG ] 17:59:58 DEBUG "0.0013333333333333049" 17:59:58 DEBUG ] 17:59:58 DEBUG ] 17:59:58 DEBUG ] 17:59:58 DEBUG } 17:59:58 DEBUG "1.0682774610048404" 17:59:58 DEBUG ] 17:59:58 DEBUG "0.0011000000000000417" 17:59:58 DEBUG ] 17:59:58 DEBUG } 17:59:58 DEBUG ] 17:59:58 DEBUG } 17:59:58 DEBUG } 17:59:58 DEBUG ] 17:59:58 DEBUG } 17:59:58 DEBUG ] 17:59:58 DEBUG } 17:59:58 DEBUG ] 17:59:58 DEBUG } 17:59:58 DEBUG } 17:59:58 DEBUG } 17:59:58 DEBUG } 17:59:58 DEBUG "696152064" 17:59:58 DEBUG ] 17:59:58 DEBUG } 17:59:58 DEBUG } 17:59:58 DEBUG } 17:59:58 DEBUG } 17:59:58 DEBUG ] 17:59:58 DEBUG } 17:59:58 DEBUG } 17:59:58 DEBUG } 17:59:58 DEBUG ] 17:59:58 DEBUG ] 17:59:58 DEBUG ] 17:59:58 DEBUG } 17:59:58 DEBUG } 17:59:58 DEBUG } 17:59:58 DEBUG } 17:59:58 DEBUG ] 17:59:58 DEBUG } 17:59:58 DEBUG } 17:59:58 DEBUG } 17:59:58 DEBUG } 17:59:58 DEBUG ] 17:59:58 DEBUG } 17:59:58 DEBUG } 17:59:58 INFO 17:59:58 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 17:59:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:59:58 INFO 17:59:58 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 17:59:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:59:58 INFO [loop_until]: OK (rc = 0) 17:59:58 DEBUG --- stdout --- 17:59:58 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 18m 425Mi ds-idrepo-0 9m 614Mi rcs-5d696ff5b-7ttvr 5m 293Mi rcs-5d696ff5b-wwlbr 3m 331Mi 17:59:58 DEBUG --- stderr --- 17:59:58 DEBUG 17:59:58 INFO [loop_until]: OK (rc = 0) 17:59:58 DEBUG --- stdout --- 17:59:58 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 100m 0% 2268Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1147m 3% 3589Mi 12% 17:59:58 DEBUG --- stderr --- 17:59:58 DEBUG 18:00:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458458" 18:00:58 INFO [http_cmd]: http status code OK 18:00:58 DEBUG --- status code --- 18:00:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:00:58 DEBUG --- http response --- 18:00:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:00:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734458458" 18:00:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458458" 18:00:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458458" 18:00:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458458" 18:00:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734458458" 18:00:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458458" 18:00:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734458458" 18:00:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458458" 18:00:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458458" 18:00:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458458" 18:00:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458458" 18:00:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458458" 18:00:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458458" 18:00:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458458" 18:00:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458458" 18:00:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458458" 18:00:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458458" 18:00:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458458" 18:00:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458458" 18:00:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458458" 18:00:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734458458" 18:00:58 INFO [http_cmd]: http status code OK 18:00:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458458" 18:00:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458458" 18:00:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458458" 18:00:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734458458" 18:00:58 INFO [http_cmd]: http status code OK 18:00:58 INFO [http_cmd]: http status code OK 18:00:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458458" 18:00:58 INFO [http_cmd]: http status code OK 18:00:58 INFO [http_cmd]: http status code OK 18:00:58 INFO [http_cmd]: http status code OK 18:00:58 INFO [http_cmd]: http status code OK 18:00:58 INFO [http_cmd]: http status code OK 18:00:58 INFO [http_cmd]: http status code OK 18:00:58 INFO [http_cmd]: http status code OK 18:00:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458458" 18:00:58 INFO [http_cmd]: http status code OK 18:00:58 INFO [http_cmd]: http status code OK 18:00:58 DEBUG --- status code --- 18:00:58 INFO [http_cmd]: http status code OK 18:00:58 INFO [http_cmd]: http status code OK 18:00:58 INFO [http_cmd]: http status code OK 18:00:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458458" 18:00:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458458" 18:00:58 INFO [http_cmd]: http status code OK 18:00:58 INFO [http_cmd]: http status code OK 18:00:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458458" 18:00:58 INFO [http_cmd]: http status code OK 18:00:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458458" 18:00:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458458" 18:00:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734458458" 18:00:58 DEBUG --- status code --- 18:00:58 INFO [http_cmd]: http status code OK 18:00:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734458458" 18:00:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458458" 18:00:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458458" 18:00:58 DEBUG --- status code --- 18:00:58 INFO [http_cmd]: http status code OK 18:00:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458458" 18:00:58 DEBUG --- status code --- 18:00:58 INFO [http_cmd]: http status code OK 18:00:58 INFO [http_cmd]: http status code OK 18:00:58 DEBUG --- status code --- 18:00:58 DEBUG --- status code --- 18:00:58 DEBUG --- status code --- 18:00:58 DEBUG --- status code --- 18:00:58 DEBUG --- status code --- 18:00:58 DEBUG --- status code --- 18:00:58 DEBUG --- status code --- 18:00:58 DEBUG --- status code --- 18:00:58 DEBUG --- status code --- 18:00:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:00:58 DEBUG --- status code --- 18:00:58 DEBUG --- status code --- 18:00:58 DEBUG --- status code --- 18:00:58 DEBUG --- status code --- 18:00:58 INFO [http_cmd]: http status code OK 18:00:58 DEBUG --- status code --- 18:00:58 ERROR [http_cmd]: ERROR 18:00:58 DEBUG --- status code --- 18:00:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:00:58 DEBUG --- status code --- 18:00:58 INFO [http_cmd]: http status code OK 18:00:58 INFO [http_cmd]: http status code OK 18:00:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:00:58 INFO [http_cmd]: http status code OK 18:00:58 DEBUG --- status code --- 18:00:58 INFO [http_cmd]: http status code OK 18:00:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:00:58 INFO [http_cmd]: http status code OK 18:00:58 INFO [http_cmd]: http status code OK 18:00:58 DEBUG --- status code --- 18:00:58 INFO [http_cmd]: http status code OK 18:00:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:00:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:00:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:00:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:00:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:00:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:00:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:00:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:00:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:00:58 INFO [http_cmd]: http status code OK 18:00:58 INFO [http_cmd]: http status code OK 18:00:58 INFO [http_cmd]: http status code OK 18:00:58 DEBUG --- http response --- 18:00:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:00:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:00:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:00:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:00:58 DEBUG --- status code --- 18:00:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:00:58 ERROR --- status code --- 18:00:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:00:58 DEBUG --- http response --- 18:00:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:00:58 DEBUG --- status code --- 18:00:58 DEBUG --- status code --- 18:00:58 INFO [http_cmd]: http status code OK 18:00:58 INFO [http_cmd]: http status code OK 18:00:58 DEBUG --- http response --- 18:00:58 DEBUG --- status code --- 18:00:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:00:58 DEBUG --- status code --- 18:00:58 DEBUG --- http response --- 18:00:58 DEBUG --- status code --- 18:00:58 DEBUG --- status code --- 18:00:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:00:58 DEBUG --- status code --- 18:00:58 DEBUG --- http response --- 18:00:58 DEBUG --- http response --- 18:00:58 DEBUG --- http response --- 18:00:58 DEBUG --- http response --- 18:00:58 DEBUG --- http response --- 18:00:58 DEBUG --- http response --- 18:00:58 DEBUG --- http response --- 18:00:58 DEBUG --- http response --- 18:00:58 DEBUG --- http response --- 18:00:58 DEBUG --- status code --- 18:00:58 DEBUG --- status code --- 18:00:58 DEBUG --- status code --- 18:00:58 INFO [http_cmd]: http status code OK 18:00:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:00:58 DEBUG --- http response --- 18:00:58 DEBUG --- http response --- 18:00:58 DEBUG --- http response --- 18:00:58 DEBUG --- http response --- 18:00:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:00:58 DEBUG --- http response --- 18:00:58 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:00:58 DEBUG --- http response --- 18:00:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:00:58 DEBUG --- http response --- 18:00:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:00:58 DEBUG --- http response --- 18:00:58 DEBUG --- status code --- 18:00:58 DEBUG --- status code --- 18:00:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:00:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:00:58 DEBUG --- http response --- 18:00:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:00:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:00:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:00:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:00:58 DEBUG --- http response --- 18:00:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:00:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:00:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:00:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:00:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:00:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:00:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:00:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:00:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:00:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:00:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:00:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:00:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:00:58 DEBUG --- status code --- 18:00:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:00:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:00:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:00:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:00:58 DEBUG --- http response --- 18:00:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:00:58 ERROR --- http response --- 18:00:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:00:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:00:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:00:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:00:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:00:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:00:58 DEBUG --- http response --- 18:00:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:00:58 DEBUG --- http response --- 18:00:58 DEBUG --- http response --- 18:00:58 DEBUG --- http response --- 18:00:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:00:58 DEBUG --- http response --- 18:00:58 DEBUG --- http response --- 18:00:58 DEBUG --- http response --- 18:00:58 DEBUG --- http response --- 18:00:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:00:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:00:58 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:00:58 DEBUG --- http response --- 18:00:58 DEBUG --- http response --- 18:00:58 DEBUG --- http response --- 18:00:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:00:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:00:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:00:58 DEBUG ----- output ----- 18:00:58 DEBUG { 18:00:58 DEBUG "status": "success", 18:00:58 DEBUG "data": { 18:00:58 DEBUG "resultType": "vector", 18:00:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:00:58 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:00:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:00:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:00:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:00:58 DEBUG ----- output ----- 18:00:58 DEBUG ----- output ----- 18:00:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:00:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:00:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:00:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:00:58 DEBUG --- http response --- 18:00:58 DEBUG "result": [ 18:00:58 DEBUG ----- output ----- 18:00:58 DEBUG ----- output ----- 18:00:58 DEBUG ----- output ----- 18:00:58 DEBUG ----- output ----- 18:00:58 DEBUG { 18:00:58 DEBUG { 18:00:58 DEBUG ----- output ----- 18:00:58 DEBUG ----- output ----- 18:00:58 DEBUG ----- output ----- 18:00:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:00:58 DEBUG { 18:00:58 DEBUG { 18:00:58 DEBUG { 18:00:58 DEBUG { 18:00:58 DEBUG { 18:00:58 DEBUG "status": "success", 18:00:58 DEBUG "status": "success", 18:00:58 DEBUG { 18:00:58 DEBUG { 18:00:58 DEBUG { 18:00:58 DEBUG ----- output ----- 18:00:58 DEBUG "metric": { 18:00:58 DEBUG "status": "success", 18:00:58 DEBUG "status": "success", 18:00:58 DEBUG "status": "success", 18:00:58 DEBUG "status": "success", 18:00:58 DEBUG "data": { 18:00:58 DEBUG "data": { 18:00:58 DEBUG "status": "success", 18:00:58 DEBUG "status": "success", 18:00:58 DEBUG "status": "success", 18:00:58 DEBUG { 18:00:58 DEBUG "instance": "10.142.0.11:9100" 18:00:58 DEBUG "data": { 18:00:58 DEBUG "data": { 18:00:58 DEBUG "data": { 18:00:58 DEBUG "data": { 18:00:58 DEBUG "resultType": "vector", 18:00:58 DEBUG "resultType": "vector", 18:00:58 DEBUG "data": { 18:00:58 DEBUG "data": { 18:00:58 DEBUG "data": { 18:00:58 DEBUG "status": "success", 18:00:58 DEBUG }, 18:00:58 DEBUG "resultType": "vector", 18:00:58 DEBUG "resultType": "vector", 18:00:58 DEBUG "resultType": "vector", 18:00:58 DEBUG "resultType": "vector", 18:00:58 DEBUG "result": [ 18:00:58 DEBUG "result": [ 18:00:58 DEBUG "resultType": "vector", 18:00:58 DEBUG "resultType": "vector", 18:00:58 DEBUG "resultType": "vector", 18:00:58 DEBUG "data": { 18:00:58 DEBUG "value": [ 18:00:58 DEBUG "result": [ 18:00:58 DEBUG "result": [ 18:00:58 DEBUG "result": [ 18:00:58 DEBUG "result": [ 18:00:58 DEBUG { 18:00:58 DEBUG "metric": { 18:00:58 DEBUG "pod": "ds-cts-0" 18:00:58 DEBUG "result": [ 18:00:58 DEBUG "result": [ 18:00:58 DEBUG "resultType": "vector", 18:00:58 DEBUG 18:00:58 DEBUG { 18:00:58 DEBUG { 18:00:58 DEBUG { 18:00:58 DEBUG { 18:00:58 DEBUG { 18:00:58 DEBUG }, 18:00:58 DEBUG "result": [ 18:00:58 DEBUG { 18:00:58 DEBUG { 18:00:58 DEBUG "result": [ 18:00:58 DEBUG 18:00:58 DEBUG "metric": { 18:00:58 DEBUG "metric": { 18:00:58 DEBUG "metric": { 18:00:58 DEBUG "metric": { 18:00:58 DEBUG "metric": { 18:00:58 DEBUG "value": [ 18:00:58 DEBUG { 18:00:58 DEBUG "metric": { 18:00:58 DEBUG "metric": { 18:00:58 DEBUG { 18:00:58 DEBUG 18:00:58 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:00:58 DEBUG "pod": "ds-cts-0" 18:00:58 DEBUG "instance": "10.142.0.11:9100" 18:00:58 DEBUG "instance": "10.142.0.11:9100" 18:00:58 DEBUG "pod": "ds-cts-0" 18:00:58 DEBUG 18:00:58 DEBUG "metric": { 18:00:58 DEBUG "instance": "10.142.0.11:9100" 18:00:58 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:00:58 DEBUG "metric": { 18:00:58 DEBUG [...] 18:00:58 DEBUG }, 18:00:58 DEBUG }, 18:00:58 DEBUG }, 18:00:58 DEBUG }, 18:00:58 DEBUG }, 18:00:58 DEBUG 18:00:58 DEBUG "instance": "10.142.0.11:9100" 18:00:58 DEBUG }, 18:00:58 DEBUG }, 18:00:58 DEBUG "pod": "ds-cts-0" 18:00:58 DEBUG 18:00:58 DEBUG "value": [ 18:00:58 DEBUG "value": [ 18:00:58 DEBUG "value": [ 18:00:58 DEBUG "value": [ 18:00:58 DEBUG "value": [ 18:00:58 DEBUG 18:00:58 DEBUG }, 18:00:58 DEBUG "value": [ 18:00:58 DEBUG "value": [ 18:00:58 DEBUG }, 18:00:58 DEBUG 18:00:58 DEBUG 18:00:58 DEBUG 18:00:58 DEBUG 18:00:58 DEBUG 18:00:58 DEBUG 18:00:58 DEBUG [...] 18:00:58 DEBUG "value": [ 18:00:58 DEBUG 18:00:58 DEBUG 18:00:58 DEBUG 18:00:58 DEBUG [...] 18:00:58 DEBUG 18:00:58 DEBUG 18:00:58 DEBUG 18:00:58 DEBUG 18:00:58 DEBUG 18:00:58 DEBUG 18:00:58 DEBUG 18:00:58 DEBUG 18:00:58 DEBUG "value": [ 18:00:58 DEBUG 18:00:58 DEBUG 18:00:58 DEBUG 18:00:58 DEBUG 18:00:58 DEBUG "instance": "10.142.0.51:9100" 18:00:58 DEBUG 18:00:58 DEBUG 18:00:58 DEBUG 18:00:58 DEBUG 18:00:58 DEBUG 18:00:58 DEBUG 18:00:58 DEBUG "instance": "10.142.0.51:9100" 18:00:58 DEBUG 18:00:58 DEBUG 18:00:58 DEBUG 18:00:58 DEBUG }, 18:00:58 DEBUG [...] 18:00:58 DEBUG [...] 18:00:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:00:58 DEBUG }, 18:00:58 DEBUG "value": [ 18:00:58 DEBUG 18:00:58 DEBUG }, 18:00:58 DEBUG [...] 18:00:58 DEBUG [...] 18:00:58 DEBUG [...] 18:00:58 DEBUG "value": [ 18:00:58 DEBUG 18:00:58 DEBUG 18:00:58 DEBUG 18:00:58 DEBUG 18:00:58 DEBUG 1734458458, 18:00:58 DEBUG "0" 18:00:58 DEBUG ] 18:00:58 DEBUG 18:00:58 DEBUG 18:00:58 DEBUG 18:00:58 DEBUG 1734458458, 18:00:58 DEBUG 18:00:58 DEBUG 18:00:58 DEBUG 18:00:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:00:58 DEBUG [...] 18:00:58 DEBUG "value": [ 18:00:58 DEBUG } 18:00:58 DEBUG 18:00:58 DEBUG 18:00:58 DEBUG 18:00:58 DEBUG "0.0014999999999995833" 18:00:58 DEBUG 18:00:58 DEBUG 18:00:58 DEBUG 18:00:58 DEBUG }, 18:00:58 DEBUG 18:00:58 DEBUG 1734458458, 18:00:58 DEBUG ] 18:00:58 DEBUG 18:00:58 DEBUG 18:00:58 DEBUG 18:00:58 DEBUG ] 18:00:58 DEBUG "instance": "10.142.0.51:9100" 18:00:58 DEBUG [...] 18:00:58 DEBUG [...] 18:00:58 DEBUG "value": [ 18:00:58 DEBUG 18:00:58 DEBUG "27852.8" 18:00:58 DEBUG } 18:00:58 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:00:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:00:58 DEBUG "instance": "10.142.0.51:9100" 18:00:58 DEBUG } 18:00:58 DEBUG }, 18:00:58 DEBUG 18:00:58 DEBUG 18:00:58 DEBUG 1734458458, 18:00:58 DEBUG 18:00:58 DEBUG ] 18:00:58 DEBUG } 18:00:58 DEBUG }, 18:00:58 DEBUG }, 18:00:58 DEBUG }, 18:00:58 DEBUG ] 18:00:58 DEBUG "value": [ 18:00:58 DEBUG 18:00:58 DEBUG 18:00:58 DEBUG "1351.5898365767234" 18:00:58 DEBUG "instance": "10.142.0.51:9100" 18:00:58 DEBUG } 18:00:58 DEBUG "value": [ 18:00:58 DEBUG "value": [ 18:00:58 DEBUG 1734458458, 18:00:58 DEBUG } 18:00:58 DEBUG 1734458458, 18:00:58 DEBUG 18:00:58 DEBUG 18:00:58 DEBUG ] 18:00:58 DEBUG }, 18:00:58 DEBUG ] 18:00:58 DEBUG 1734458458, 18:00:58 DEBUG "value": [ 18:00:58 DEBUG "0.005276498477042216" 18:00:58 DEBUG } 18:00:58 DEBUG "131211.96666666667" 18:00:58 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:00:58 DEBUG }, 18:00:58 DEBUG } 18:00:58 DEBUG ] 18:00:58 DEBUG } 18:00:58 DEBUG "1.0668418822376695" 18:00:58 DEBUG 1734458458, 18:00:58 DEBUG "0.0009999999999999194" 18:00:58 DEBUG ] 18:00:58 DEBUG ] 18:00:58 DEBUG "value": [ 18:00:58 DEBUG "value": [ 18:00:58 DEBUG } 18:00:58 DEBUG } 18:00:58 DEBUG ] 18:00:58 DEBUG ] 18:00:58 DEBUG } 18:00:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:00:58 DEBUG } 18:00:58 DEBUG 1734458458, 18:00:58 DEBUG 1734458458, 18:00:58 DEBUG } 18:00:58 DEBUG } 18:00:58 DEBUG } 18:00:58 DEBUG ] 18:00:58 DEBUG }, 18:00:58 DEBUG ] 18:00:58 DEBUG "2723323904" 18:00:58 DEBUG "133190.86666666664" 18:00:58 DEBUG ] 18:00:58 DEBUG ] 18:00:58 DEBUG } 18:00:58 DEBUG "value": [ 18:00:58 DEBUG } 18:00:58 DEBUG ] 18:00:58 DEBUG ] 18:00:58 DEBUG } 18:00:58 DEBUG } 18:00:58 DEBUG } 18:00:58 DEBUG 1734458458, 18:00:58 DEBUG } 18:00:58 DEBUG } 18:00:58 DEBUG } 18:00:58 DEBUG } 18:00:58 DEBUG } 18:00:58 DEBUG "696565760" 18:00:58 DEBUG ] 18:00:58 DEBUG ] 18:00:58 DEBUG ] 18:00:58 DEBUG } 18:00:58 DEBUG } 18:00:58 DEBUG } 18:00:58 DEBUG } 18:00:58 DEBUG } 18:00:58 DEBUG ] 18:00:58 DEBUG } 18:00:58 DEBUG } 18:00:58 INFO 18:00:58 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:00:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:00:58 INFO 18:00:58 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:00:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:00:58 INFO [loop_until]: OK (rc = 0) 18:00:58 DEBUG --- stdout --- 18:00:58 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 8m 429Mi ds-idrepo-0 9m 622Mi rcs-5d696ff5b-7ttvr 4m 293Mi rcs-5d696ff5b-wwlbr 2m 331Mi 18:00:58 DEBUG --- stderr --- 18:00:58 DEBUG 18:00:58 INFO [loop_until]: OK (rc = 0) 18:00:58 DEBUG --- stdout --- 18:00:58 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 103m 0% 2267Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1133m 3% 3598Mi 12% 18:00:58 DEBUG --- stderr --- 18:00:58 DEBUG 18:01:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458518" 18:01:58 INFO [http_cmd]: http status code OK 18:01:58 DEBUG --- status code --- 18:01:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:01:58 DEBUG --- http response --- 18:01:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:01:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458518" 18:01:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734458518" 18:01:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458518" 18:01:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734458518" 18:01:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734458518" 18:01:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458518" 18:01:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458518" 18:01:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458518" 18:01:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458518" 18:01:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458518" 18:01:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458518" 18:01:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458518" 18:01:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458518" 18:01:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458518" 18:01:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458518" 18:01:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458518" 18:01:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734458518" 18:01:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458518" 18:01:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458518" 18:01:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458518" 18:01:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458518" 18:01:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734458518" 18:01:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458518" 18:01:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458518" 18:01:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458518" 18:01:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458518" 18:01:58 INFO [http_cmd]: http status code OK 18:01:58 INFO [http_cmd]: http status code OK 18:01:58 INFO [http_cmd]: http status code OK 18:01:58 INFO [http_cmd]: http status code OK 18:01:58 INFO [http_cmd]: http status code OK 18:01:58 INFO [http_cmd]: http status code OK 18:01:58 INFO [http_cmd]: http status code OK 18:01:58 INFO [http_cmd]: http status code OK 18:01:58 INFO [http_cmd]: http status code OK 18:01:58 INFO [http_cmd]: http status code OK 18:01:58 INFO [http_cmd]: http status code OK 18:01:58 INFO [http_cmd]: http status code OK 18:01:58 INFO [http_cmd]: http status code OK 18:01:58 INFO [http_cmd]: http status code OK 18:01:58 INFO [http_cmd]: http status code OK 18:01:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458518" 18:01:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458518" 18:01:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458518" 18:01:58 INFO [http_cmd]: http status code OK 18:01:58 INFO [http_cmd]: http status code OK 18:01:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458518" 18:01:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734458518" 18:01:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458518" 18:01:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458518" 18:01:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734458518" 18:01:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458518" 18:01:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458518" 18:01:58 INFO [http_cmd]: http status code OK 18:01:58 INFO [http_cmd]: http status code OK 18:01:58 DEBUG --- status code --- 18:01:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458518" 18:01:58 INFO [http_cmd]: http status code OK 18:01:58 ERROR [http_cmd]: ERROR 18:01:58 DEBUG --- status code --- 18:01:58 INFO [http_cmd]: http status code OK 18:01:58 DEBUG --- status code --- 18:01:58 DEBUG --- status code --- 18:01:58 DEBUG --- status code --- 18:01:58 DEBUG --- status code --- 18:01:58 DEBUG --- status code --- 18:01:58 DEBUG --- status code --- 18:01:58 INFO [http_cmd]: http status code OK 18:01:58 INFO [http_cmd]: http status code OK 18:01:58 DEBUG --- status code --- 18:01:58 DEBUG --- status code --- 18:01:58 DEBUG --- status code --- 18:01:58 DEBUG --- status code --- 18:01:58 DEBUG --- status code --- 18:01:58 DEBUG --- status code --- 18:01:58 DEBUG --- status code --- 18:01:58 DEBUG --- status code --- 18:01:58 INFO [http_cmd]: http status code OK 18:01:58 DEBUG --- status code --- 18:01:58 INFO [http_cmd]: http status code OK 18:01:58 INFO [http_cmd]: http status code OK 18:01:58 DEBUG --- status code --- 18:01:58 INFO [http_cmd]: http status code OK 18:01:58 DEBUG --- status code --- 18:01:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:01:58 INFO [http_cmd]: http status code OK 18:01:58 DEBUG --- status code --- 18:01:58 ERROR --- status code --- 18:01:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:01:58 DEBUG --- status code --- 18:01:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:01:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:01:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:01:58 INFO [http_cmd]: http status code OK 18:01:58 INFO [http_cmd]: http status code OK 18:01:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:01:58 INFO [http_cmd]: http status code OK 18:01:58 INFO [http_cmd]: http status code OK 18:01:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:01:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:01:58 DEBUG --- status code --- 18:01:58 DEBUG --- status code --- 18:01:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:01:58 DEBUG --- http response --- 18:01:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:01:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:01:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:01:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:01:58 INFO [http_cmd]: http status code OK 18:01:58 INFO [http_cmd]: http status code OK 18:01:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:01:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:01:58 DEBUG --- status code --- 18:01:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:01:58 DEBUG --- status code --- 18:01:58 DEBUG --- status code --- 18:01:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:01:58 DEBUG --- status code --- 18:01:58 INFO [http_cmd]: http status code OK 18:01:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:01:58 DEBUG --- http response --- 18:01:58 DEBUG --- status code --- 18:01:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:01:58 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:01:58 DEBUG --- http response --- 18:01:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:01:58 DEBUG --- http response --- 18:01:58 DEBUG --- http response --- 18:01:58 DEBUG --- http response --- 18:01:58 INFO [http_cmd]: http status code OK 18:01:58 DEBUG --- status code --- 18:01:58 DEBUG --- status code --- 18:01:58 DEBUG --- http response --- 18:01:58 DEBUG --- status code --- 18:01:58 DEBUG --- status code --- 18:01:58 DEBUG --- http response --- 18:01:58 DEBUG --- http response --- 18:01:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:01:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:01:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:01:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:01:58 DEBUG --- http response --- 18:01:58 DEBUG --- http response --- 18:01:58 DEBUG --- http response --- 18:01:58 DEBUG --- http response --- 18:01:58 DEBUG --- status code --- 18:01:58 DEBUG --- status code --- 18:01:58 DEBUG --- http response --- 18:01:58 DEBUG --- http response --- 18:01:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:01:58 DEBUG --- http response --- 18:01:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:01:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:01:58 DEBUG --- http response --- 18:01:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:01:58 DEBUG --- status code --- 18:01:58 DEBUG --- http response --- 18:01:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:01:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:01:58 DEBUG --- http response --- 18:01:58 ERROR --- http response --- 18:01:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:01:58 DEBUG --- http response --- 18:01:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:01:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:01:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:01:58 DEBUG --- status code --- 18:01:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:01:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:01:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:01:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:01:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:01:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:01:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:01:58 DEBUG --- http response --- 18:01:58 DEBUG --- http response --- 18:01:58 DEBUG --- http response --- 18:01:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:01:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:01:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:01:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:01:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:01:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:01:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:01:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:01:58 DEBUG --- http response --- 18:01:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:01:58 DEBUG --- http response --- 18:01:58 DEBUG --- http response --- 18:01:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:01:58 DEBUG --- http response --- 18:01:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:01:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:01:58 DEBUG --- http response --- 18:01:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:01:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:01:58 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:01:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:01:58 DEBUG --- http response --- 18:01:58 DEBUG --- http response --- 18:01:58 DEBUG --- http response --- 18:01:58 DEBUG --- http response --- 18:01:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:01:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:01:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:01:58 DEBUG --- http response --- 18:01:58 DEBUG --- http response --- 18:01:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:01:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:01:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:01:58 DEBUG --- http response --- 18:01:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:01:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:01:58 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:01:58 DEBUG --- http response --- 18:01:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:01:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:01:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:01:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:01:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:01:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:01:58 DEBUG ----- output ----- 18:01:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:01:58 DEBUG ----- output ----- 18:01:58 DEBUG ----- output ----- 18:01:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:01:58 DEBUG ----- output ----- 18:01:58 DEBUG ----- output ----- 18:01:58 DEBUG ----- output ----- 18:01:58 DEBUG ----- output ----- 18:01:58 DEBUG { 18:01:58 DEBUG ----- output ----- 18:01:58 DEBUG { 18:01:58 DEBUG ----- output ----- 18:01:58 DEBUG { 18:01:58 DEBUG { 18:01:58 DEBUG ----- output ----- 18:01:58 DEBUG { 18:01:58 DEBUG { 18:01:58 DEBUG ----- output ----- 18:01:58 DEBUG { 18:01:58 DEBUG "status": "success", 18:01:58 DEBUG { 18:01:58 DEBUG "status": "success", 18:01:58 DEBUG { 18:01:58 DEBUG "status": "success", 18:01:58 DEBUG "status": "success", 18:01:58 DEBUG { 18:01:58 DEBUG "status": "success", 18:01:58 DEBUG "status": "success", 18:01:58 DEBUG { 18:01:58 DEBUG "status": "success", 18:01:58 DEBUG "data": { 18:01:58 DEBUG "status": "success", 18:01:58 DEBUG "data": { 18:01:58 DEBUG "status": "success", 18:01:58 DEBUG "data": { 18:01:58 DEBUG "data": { 18:01:58 DEBUG "status": "success", 18:01:58 DEBUG "data": { 18:01:58 DEBUG "data": { 18:01:58 DEBUG "status": "success", 18:01:58 DEBUG "data": { 18:01:58 DEBUG "resultType": "vector", 18:01:58 DEBUG "data": { 18:01:58 DEBUG "resultType": "vector", 18:01:58 DEBUG "data": { 18:01:58 DEBUG "resultType": "vector", 18:01:58 DEBUG "resultType": "vector", 18:01:58 DEBUG "data": { 18:01:58 DEBUG "resultType": "vector", 18:01:58 DEBUG "resultType": "vector", 18:01:58 DEBUG "data": { 18:01:58 DEBUG "resultType": "vector", 18:01:58 DEBUG "result": [ 18:01:58 DEBUG "resultType": "vector", 18:01:58 DEBUG "result": [ 18:01:58 DEBUG "resultType": "vector", 18:01:58 DEBUG "result": [ 18:01:58 DEBUG "result": [ 18:01:58 DEBUG "resultType": "vector", 18:01:58 DEBUG "result": [ 18:01:58 DEBUG "result": [ 18:01:58 DEBUG "resultType": "vector", 18:01:58 DEBUG "result": [ 18:01:58 DEBUG { 18:01:58 DEBUG "result": [ 18:01:58 DEBUG { 18:01:58 DEBUG "result": [ 18:01:58 DEBUG { 18:01:58 DEBUG { 18:01:58 DEBUG "result": [ 18:01:58 DEBUG { 18:01:58 DEBUG { 18:01:58 DEBUG "result": [ 18:01:58 DEBUG { 18:01:58 DEBUG "metric": { 18:01:58 DEBUG { 18:01:58 DEBUG "metric": { 18:01:58 DEBUG { 18:01:58 DEBUG "metric": { 18:01:58 DEBUG "metric": { 18:01:58 DEBUG { 18:01:58 DEBUG "metric": { 18:01:58 DEBUG "metric": { 18:01:58 DEBUG { 18:01:58 DEBUG "metric": { 18:01:58 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:01:58 DEBUG "metric": { 18:01:58 DEBUG "instance": "10.142.0.11:9100" 18:01:58 DEBUG "metric": { 18:01:58 DEBUG "pod": "ds-cts-0" 18:01:58 DEBUG "pod": "ds-cts-0" 18:01:58 DEBUG "metric": { 18:01:58 DEBUG "pod": "ds-cts-0" 18:01:58 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:01:58 DEBUG "metric": { 18:01:58 DEBUG "instance": "10.142.0.11:9100" 18:01:58 DEBUG }, 18:01:58 DEBUG "pod": "ds-cts-0" 18:01:58 DEBUG }, 18:01:58 DEBUG "instance": "10.142.0.11:9100" 18:01:58 DEBUG }, 18:01:58 DEBUG }, 18:01:58 DEBUG "instance": "10.142.0.11:9100" 18:01:58 DEBUG }, 18:01:58 DEBUG }, 18:01:58 DEBUG "instance": "10.142.0.11:9100" 18:01:58 DEBUG }, 18:01:58 DEBUG "value": [ 18:01:58 DEBUG }, 18:01:58 DEBUG "value": [ 18:01:58 DEBUG }, 18:01:58 DEBUG "value": [ 18:01:58 DEBUG 18:01:58 DEBUG 18:01:58 DEBUG 18:01:58 DEBUG [...] 18:01:58 DEBUG 18:01:58 DEBUG 18:01:58 DEBUG 18:01:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:01:58 DEBUG }, 18:01:58 DEBUG "value": [ 18:01:58 DEBUG 1734458518, 18:01:58 DEBUG "1404.07889292503" 18:01:58 DEBUG ] 18:01:58 DEBUG } 18:01:58 DEBUG }, 18:01:58 DEBUG 18:01:58 DEBUG "value": [ 18:01:58 DEBUG "value": [ 18:01:58 DEBUG 18:01:58 DEBUG "value": [ 18:01:58 DEBUG "value": [ 18:01:58 DEBUG }, 18:01:58 DEBUG "value": [ 18:01:58 DEBUG "value": [ 18:01:58 DEBUG ] 18:01:58 DEBUG "value": [ 18:01:58 DEBUG 18:01:58 DEBUG 18:01:58 DEBUG [...] 18:01:58 DEBUG 18:01:58 DEBUG 18:01:58 DEBUG 18:01:58 DEBUG "value": [ 18:01:58 DEBUG 18:01:58 DEBUG 18:01:58 DEBUG } 18:01:58 DEBUG 18:01:58 DEBUG 18:01:58 DEBUG 18:01:58 DEBUG 18:01:58 DEBUG 18:01:58 DEBUG 18:01:58 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:01:58 DEBUG 18:01:58 DEBUG 18:01:58 DEBUG 18:01:58 DEBUG } 18:01:58 DEBUG 18:01:58 DEBUG 18:01:58 DEBUG 18:01:58 DEBUG 18:01:58 DEBUG 18:01:58 DEBUG 18:01:58 DEBUG }, 18:01:58 DEBUG 18:01:58 DEBUG 18:01:58 DEBUG 18:01:58 DEBUG 18:01:58 DEBUG [...] 18:01:58 DEBUG 18:01:58 DEBUG [...] 18:01:58 DEBUG 18:01:58 DEBUG [...] 18:01:58 DEBUG 18:01:58 DEBUG 18:01:58 DEBUG 18:01:58 DEBUG "instance": "10.142.0.51:9100" 18:01:58 DEBUG }, 18:01:58 DEBUG 18:01:58 DEBUG [...] 18:01:58 DEBUG 18:01:58 DEBUG 18:01:58 DEBUG "value": [ 18:01:58 DEBUG 18:01:58 DEBUG [...] 18:01:58 DEBUG [...] 18:01:58 DEBUG "value": [ 18:01:58 DEBUG 18:01:58 DEBUG [...] 18:01:58 DEBUG 18:01:58 DEBUG 18:01:58 DEBUG [...] 18:01:58 DEBUG 1734458518, 18:01:58 DEBUG [...] 18:01:58 DEBUG 18:01:58 DEBUG 18:01:58 DEBUG 1734458518, 18:01:58 DEBUG 18:01:58 DEBUG 18:01:58 DEBUG 18:01:58 DEBUG 18:01:58 DEBUG 18:01:58 DEBUG "2738814976" 18:01:58 DEBUG 18:01:58 DEBUG 18:01:58 DEBUG 18:01:58 DEBUG "0.0006333333333363574" 18:01:58 DEBUG ] 18:01:58 DEBUG } 18:01:58 DEBUG 18:01:58 DEBUG "instance": "10.142.0.51:9100" 18:01:58 DEBUG 18:01:58 DEBUG ] 18:01:58 DEBUG 18:01:58 DEBUG 18:01:58 DEBUG 18:01:58 DEBUG 18:01:58 DEBUG ] 18:01:58 DEBUG 18:01:58 DEBUG 18:01:58 DEBUG }, 18:01:58 DEBUG 18:01:58 DEBUG } 18:01:58 DEBUG 18:01:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:01:58 DEBUG }, 18:01:58 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:01:58 DEBUG } 18:01:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:01:58 DEBUG "instance": "10.142.0.51:9100" 18:01:58 DEBUG "value": [ 18:01:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:01:58 DEBUG ] 18:01:58 DEBUG "instance": "10.142.0.51:9100" 18:01:58 DEBUG "value": [ 18:01:58 DEBUG "instance": "10.142.0.51:9100" 18:01:58 DEBUG }, 18:01:58 DEBUG } 18:01:58 DEBUG }, 18:01:58 DEBUG }, 18:01:58 DEBUG 1734458518, 18:01:58 DEBUG }, 18:01:58 DEBUG } 18:01:58 DEBUG }, 18:01:58 DEBUG 1734458518, 18:01:58 DEBUG }, 18:01:58 DEBUG "value": [ 18:01:58 DEBUG "value": [ 18:01:58 DEBUG "value": [ 18:01:58 DEBUG "122831.63333333332" 18:01:58 DEBUG "value": [ 18:01:58 DEBUG } 18:01:58 DEBUG "value": [ 18:01:58 DEBUG "696807424" 18:01:58 DEBUG "value": [ 18:01:58 DEBUG 1734458518, 18:01:58 DEBUG 1734458518, 18:01:58 DEBUG 1734458518, 18:01:58 DEBUG ] 18:01:58 DEBUG 1734458518, 18:01:58 DEBUG 1734458518, 18:01:58 DEBUG ] 18:01:58 DEBUG 1734458518, 18:01:58 DEBUG "1.0673638734282627" 18:01:58 DEBUG "0.004029611875685436" 18:01:58 DEBUG "28398.933333333334" 18:01:58 DEBUG } 18:01:58 DEBUG ] 18:01:58 DEBUG } 18:01:58 DEBUG } 18:01:58 DEBUG "127504.16666666667" 18:01:58 DEBUG ] 18:01:58 DEBUG ] 18:01:58 DEBUG ] 18:01:58 DEBUG "0" 18:01:58 DEBUG "0.0003333333333333854" 18:01:58 DEBUG } 18:01:58 DEBUG ] 18:01:58 DEBUG } 18:01:58 DEBUG ] 18:01:58 DEBUG } 18:01:58 DEBUG ] 18:01:58 DEBUG ] 18:01:58 DEBUG ] 18:01:58 DEBUG } 18:01:58 DEBUG } 18:01:58 DEBUG } 18:01:58 DEBUG ] 18:01:58 DEBUG } 18:01:58 DEBUG } 18:01:58 DEBUG } 18:01:58 DEBUG ] 18:01:58 DEBUG } 18:01:58 DEBUG } 18:01:58 DEBUG ] 18:01:58 DEBUG ] 18:01:58 DEBUG } 18:01:58 DEBUG } 18:01:58 DEBUG } 18:01:58 DEBUG ] 18:01:58 DEBUG } 18:01:58 DEBUG } 18:01:58 DEBUG } 18:01:58 DEBUG } 18:01:58 DEBUG } 18:01:58 DEBUG } 18:01:58 DEBUG } 18:01:58 INFO 18:01:58 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:01:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:01:58 INFO 18:01:58 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:01:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:01:58 INFO [loop_until]: OK (rc = 0) 18:01:58 DEBUG --- stdout --- 18:01:58 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 7m 433Mi ds-idrepo-0 9m 627Mi rcs-5d696ff5b-7ttvr 4m 293Mi rcs-5d696ff5b-wwlbr 3m 331Mi 18:01:58 DEBUG --- stderr --- 18:01:58 DEBUG 18:01:58 INFO [loop_until]: OK (rc = 0) 18:01:58 DEBUG --- stdout --- 18:01:58 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 99m 0% 2257Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1151m 3% 3607Mi 12% 18:01:58 DEBUG --- stderr --- 18:01:58 DEBUG 18:02:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458578" 18:02:58 INFO [http_cmd]: http status code OK 18:02:58 DEBUG --- status code --- 18:02:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:02:58 DEBUG --- http response --- 18:02:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:02:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458578" 18:02:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458578" 18:02:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734458578" 18:02:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458578" 18:02:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458578" 18:02:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734458578" 18:02:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458578" 18:02:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458578" 18:02:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458578" 18:02:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734458578" 18:02:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458578" 18:02:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458578" 18:02:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458578" 18:02:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458578" 18:02:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734458578" 18:02:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458578" 18:02:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458578" 18:02:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458578" 18:02:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458578" 18:02:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734458578" 18:02:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458578" 18:02:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458578" 18:02:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458578" 18:02:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458578" 18:02:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458578" 18:02:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458578" 18:02:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458578" 18:02:58 INFO [http_cmd]: http status code OK 18:02:58 DEBUG --- status code --- 18:02:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:02:58 DEBUG --- http response --- 18:02:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:02:58 INFO [http_cmd]: http status code OK 18:02:58 DEBUG --- status code --- 18:02:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:02:58 DEBUG --- http response --- 18:02:58 INFO [http_cmd]: http status code OK 18:02:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:02:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734458578" 18:02:58 INFO [http_cmd]: http status code OK 18:02:58 INFO [http_cmd]: http status code OK 18:02:58 INFO [http_cmd]: http status code OK 18:02:58 DEBUG --- status code --- 18:02:58 INFO [http_cmd]: http status code OK 18:02:58 INFO [http_cmd]: http status code OK 18:02:58 INFO [http_cmd]: http status code OK 18:02:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458578" 18:02:58 INFO [http_cmd]: http status code OK 18:02:58 INFO [http_cmd]: http status code OK 18:02:58 ERROR [http_cmd]: ERROR 18:02:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458578" 18:02:58 INFO [http_cmd]: http status code OK 18:02:58 INFO [http_cmd]: http status code OK 18:02:58 INFO [http_cmd]: http status code OK 18:02:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458578" 18:02:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458578" 18:02:58 INFO [http_cmd]: http status code OK 18:02:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458578" 18:02:58 INFO [http_cmd]: http status code OK 18:02:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458578" 18:02:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458578" 18:02:58 INFO [http_cmd]: http status code OK 18:02:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734458578" 18:02:58 INFO [http_cmd]: http status code OK 18:02:58 INFO [http_cmd]: http status code OK 18:02:58 INFO [http_cmd]: http status code OK 18:02:58 INFO [http_cmd]: http status code OK 18:02:58 INFO [http_cmd]: http status code OK 18:02:58 INFO [http_cmd]: http status code OK 18:02:58 DEBUG --- status code --- 18:02:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458578" 18:02:58 INFO [http_cmd]: http status code OK 18:02:58 DEBUG --- status code --- 18:02:58 DEBUG --- status code --- 18:02:58 DEBUG --- status code --- 18:02:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:02:58 DEBUG --- status code --- 18:02:58 DEBUG --- status code --- 18:02:58 INFO [http_cmd]: http status code OK 18:02:58 DEBUG --- status code --- 18:02:58 DEBUG --- status code --- 18:02:58 DEBUG --- status code --- 18:02:58 ERROR --- status code --- 18:02:58 DEBUG --- status code --- 18:02:58 DEBUG --- status code --- 18:02:58 DEBUG --- status code --- 18:02:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:02:58 DEBUG --- status code --- 18:02:58 DEBUG --- status code --- 18:02:58 INFO [http_cmd]: http status code OK 18:02:58 DEBUG --- status code --- 18:02:58 DEBUG --- status code --- 18:02:58 DEBUG --- status code --- 18:02:58 DEBUG --- status code --- 18:02:58 DEBUG --- status code --- 18:02:58 INFO [http_cmd]: http status code OK 18:02:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:02:58 INFO [http_cmd]: http status code OK 18:02:58 INFO [http_cmd]: http status code OK 18:02:58 DEBUG --- status code --- 18:02:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:02:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:02:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:02:58 DEBUG --- http response --- 18:02:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:02:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:02:58 DEBUG --- status code --- 18:02:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:02:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:02:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:02:58 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:02:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:02:58 INFO [http_cmd]: http status code OK 18:02:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:02:58 INFO [http_cmd]: http status code OK 18:02:58 DEBUG --- status code --- 18:02:58 INFO [http_cmd]: http status code OK 18:02:58 DEBUG --- http response --- 18:02:58 INFO [http_cmd]: http status code OK 18:02:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:02:58 INFO [http_cmd]: http status code OK 18:02:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:02:58 INFO [http_cmd]: http status code OK 18:02:58 DEBUG --- status code --- 18:02:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:02:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:02:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:02:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:02:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:02:58 DEBUG --- status code --- 18:02:58 DEBUG --- http response --- 18:02:58 DEBUG --- status code --- 18:02:58 DEBUG --- status code --- 18:02:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:02:58 INFO [http_cmd]: http status code OK 18:02:58 DEBUG --- status code --- 18:02:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:02:58 DEBUG --- http response --- 18:02:58 DEBUG --- http response --- 18:02:58 DEBUG --- http response --- 18:02:58 DEBUG --- http response --- 18:02:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:02:58 DEBUG --- http response --- 18:02:58 DEBUG --- http response --- 18:02:58 DEBUG --- http response --- 18:02:58 ERROR --- http response --- 18:02:58 DEBUG --- http response --- 18:02:58 DEBUG --- status code --- 18:02:58 DEBUG --- http response --- 18:02:58 DEBUG --- status code --- 18:02:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:02:58 DEBUG --- status code --- 18:02:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:02:58 DEBUG --- status code --- 18:02:58 DEBUG --- http response --- 18:02:58 DEBUG --- status code --- 18:02:58 DEBUG --- http response --- 18:02:58 DEBUG --- status code --- 18:02:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:02:58 DEBUG --- http response --- 18:02:58 DEBUG --- http response --- 18:02:58 DEBUG --- http response --- 18:02:58 DEBUG --- http response --- 18:02:58 DEBUG --- http response --- 18:02:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:02:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:02:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:02:58 DEBUG --- http response --- 18:02:58 DEBUG --- http response --- 18:02:58 DEBUG --- http response --- 18:02:58 DEBUG --- http response --- 18:02:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:02:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:02:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:02:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:02:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:02:58 DEBUG --- http response --- 18:02:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:02:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:02:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:02:58 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:02:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:02:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:02:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:02:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:02:58 DEBUG --- http response --- 18:02:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:02:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:02:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:02:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:02:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:02:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:02:58 DEBUG --- http response --- 18:02:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:02:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:02:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:02:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:02:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:02:58 DEBUG --- http response --- 18:02:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:02:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:02:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:02:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:02:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:02:58 DEBUG ----- output ----- 18:02:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:02:58 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:02:58 DEBUG --- http response --- 18:02:58 DEBUG --- http response --- 18:02:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:02:58 DEBUG --- http response --- 18:02:58 DEBUG --- http response --- 18:02:58 DEBUG --- http response --- 18:02:58 DEBUG --- http response --- 18:02:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:02:58 DEBUG ----- output ----- 18:02:58 DEBUG ----- output ----- 18:02:58 DEBUG { 18:02:58 DEBUG "status": "success", 18:02:58 DEBUG --- http response --- 18:02:58 DEBUG { 18:02:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:02:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:02:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:02:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:02:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:02:58 DEBUG ----- output ----- 18:02:58 DEBUG { 18:02:58 DEBUG { 18:02:58 DEBUG "status": "success", 18:02:58 DEBUG ----- output ----- 18:02:58 DEBUG "data": { 18:02:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:02:58 DEBUG "status": "success", 18:02:58 DEBUG ----- output ----- 18:02:58 DEBUG ----- output ----- 18:02:58 DEBUG ----- output ----- 18:02:58 DEBUG ----- output ----- 18:02:58 DEBUG ----- output ----- 18:02:58 DEBUG { 18:02:58 DEBUG "status": "success", 18:02:58 DEBUG "data": { 18:02:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:02:58 DEBUG { 18:02:58 DEBUG "resultType": "vector", 18:02:58 DEBUG ----- output ----- 18:02:58 DEBUG "data": { 18:02:58 DEBUG { 18:02:58 DEBUG { 18:02:58 DEBUG { 18:02:58 DEBUG { 18:02:58 DEBUG "status": "success", 18:02:58 DEBUG "data": { 18:02:58 DEBUG "resultType": "vector", 18:02:58 DEBUG "result": [ 18:02:58 DEBUG "status": "success", 18:02:58 DEBUG "result": [ 18:02:58 DEBUG { 18:02:58 DEBUG "resultType": "vector", 18:02:58 DEBUG "status": "success", 18:02:58 DEBUG "status": "success", 18:02:58 DEBUG "status": "success", 18:02:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:02:58 DEBUG "status": "success", 18:02:58 DEBUG "data": { 18:02:58 DEBUG "resultType": "vector", 18:02:58 DEBUG { 18:02:58 DEBUG "data": { 18:02:58 DEBUG { 18:02:58 DEBUG "metric": { 18:02:58 DEBUG "status": "success", 18:02:58 DEBUG "data": { 18:02:58 DEBUG "data": { 18:02:58 DEBUG "data": { 18:02:58 DEBUG "data": { 18:02:58 DEBUG "resultType": "vector", 18:02:58 DEBUG "result": [ 18:02:58 DEBUG "metric": { 18:02:58 DEBUG "resultType": "vector", 18:02:58 DEBUG "result": [ 18:02:58 DEBUG "pod": "ds-cts-0" 18:02:58 DEBUG }, 18:02:58 DEBUG "value": [ 18:02:58 DEBUG "resultType": "vector", 18:02:58 DEBUG "resultType": "vector", 18:02:58 DEBUG "resultType": "vector", 18:02:58 DEBUG "result": [ 18:02:58 DEBUG { 18:02:58 DEBUG "instance": "10.142.0.11:9100" 18:02:58 DEBUG "result": [ 18:02:58 DEBUG { 18:02:58 DEBUG "data": { 18:02:58 DEBUG "resultType": "vector", 18:02:58 DEBUG 18:02:58 DEBUG "result": [ 18:02:58 DEBUG "result": [ 18:02:58 DEBUG "result": [ 18:02:58 DEBUG { 18:02:58 DEBUG "metric": { 18:02:58 DEBUG }, 18:02:58 DEBUG { 18:02:58 DEBUG "metric": { 18:02:58 DEBUG "resultType": "vector", 18:02:58 DEBUG "result": [ 18:02:58 DEBUG 18:02:58 DEBUG { 18:02:58 DEBUG { 18:02:58 DEBUG { 18:02:58 DEBUG "metric": { 18:02:58 DEBUG "instance": "10.142.0.11:9100" 18:02:58 DEBUG "value": [ 18:02:58 DEBUG "metric": { 18:02:58 DEBUG "instance": "10.142.0.11:9100" 18:02:58 DEBUG "result": [ 18:02:58 DEBUG { 18:02:58 DEBUG 18:02:58 DEBUG "metric": { 18:02:58 DEBUG "metric": { 18:02:58 DEBUG "metric": { 18:02:58 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:02:58 DEBUG }, 18:02:58 DEBUG 18:02:58 DEBUG "instance": "10.142.0.11:9100" 18:02:58 DEBUG }, 18:02:58 DEBUG { 18:02:58 DEBUG "metric": { 18:02:58 DEBUG [...] 18:02:58 DEBUG "pod": "ds-cts-0" 18:02:58 DEBUG "pod": "ds-cts-0" 18:02:58 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:02:58 DEBUG }, 18:02:58 DEBUG "value": [ 18:02:58 DEBUG 18:02:58 DEBUG }, 18:02:58 DEBUG "value": [ 18:02:58 DEBUG "metric": { 18:02:58 DEBUG "instance": "10.142.0.11:9100" 18:02:58 DEBUG 18:02:58 DEBUG }, 18:02:58 DEBUG }, 18:02:58 DEBUG }, 18:02:58 DEBUG "value": [ 18:02:58 DEBUG 18:02:58 DEBUG 18:02:58 DEBUG "value": [ 18:02:58 DEBUG 18:02:58 DEBUG "pod": "ds-cts-0" 18:02:58 DEBUG }, 18:02:58 DEBUG 18:02:58 DEBUG "value": [ 18:02:58 DEBUG "value": [ 18:02:58 DEBUG "value": [ 18:02:58 DEBUG 18:02:58 DEBUG 18:02:58 DEBUG [...] 18:02:58 DEBUG 18:02:58 DEBUG 18:02:58 DEBUG }, 18:02:58 DEBUG "value": [ 18:02:58 DEBUG 18:02:58 DEBUG 18:02:58 DEBUG 18:02:58 DEBUG 18:02:58 DEBUG 18:02:58 DEBUG 18:02:58 DEBUG 18:02:58 DEBUG 18:02:58 DEBUG 18:02:58 DEBUG "value": [ 18:02:58 DEBUG 18:02:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:02:58 DEBUG 18:02:58 DEBUG 18:02:58 DEBUG 18:02:58 DEBUG 18:02:58 DEBUG [...] 18:02:58 DEBUG 18:02:58 DEBUG 18:02:58 DEBUG [...] 18:02:58 DEBUG 18:02:58 DEBUG 18:02:58 DEBUG }, 18:02:58 DEBUG 18:02:58 DEBUG 18:02:58 DEBUG 18:02:58 DEBUG [...] 18:02:58 DEBUG 18:02:58 DEBUG 18:02:58 DEBUG [...] 18:02:58 DEBUG 18:02:58 DEBUG 18:02:58 DEBUG 18:02:58 DEBUG "value": [ 18:02:58 DEBUG [...] 18:02:58 DEBUG [...] 18:02:58 DEBUG [...] 18:02:58 DEBUG 18:02:58 DEBUG 18:02:58 DEBUG "instance": "10.142.0.51:9100" 18:02:58 DEBUG 18:02:58 DEBUG 18:02:58 DEBUG 18:02:58 DEBUG "instance": "10.142.0.51:9100" 18:02:58 DEBUG 1734458578, 18:02:58 DEBUG 18:02:58 DEBUG 18:02:58 DEBUG 18:02:58 DEBUG 18:02:58 DEBUG 18:02:58 DEBUG }, 18:02:58 DEBUG 18:02:58 DEBUG 18:02:58 DEBUG [...] 18:02:58 DEBUG }, 18:02:58 DEBUG "1230.4146741123825" 18:02:58 DEBUG 18:02:58 DEBUG 18:02:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:02:58 DEBUG }, 18:02:58 DEBUG "instance": "10.142.0.51:9100" 18:02:58 DEBUG "value": [ 18:02:58 DEBUG 18:02:58 DEBUG "instance": "10.142.0.51:9100" 18:02:58 DEBUG 18:02:58 DEBUG "value": [ 18:02:58 DEBUG ] 18:02:58 DEBUG 18:02:58 DEBUG 18:02:58 DEBUG 18:02:58 DEBUG "value": [ 18:02:58 DEBUG }, 18:02:58 DEBUG 1734458578, 18:02:58 DEBUG [...] 18:02:58 DEBUG }, 18:02:58 DEBUG 18:02:58 DEBUG 1734458578, 18:02:58 DEBUG } 18:02:58 DEBUG 18:02:58 DEBUG 18:02:58 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:02:58 DEBUG 1734458578, 18:02:58 DEBUG "value": [ 18:02:58 DEBUG "144574.3333333333" 18:02:58 DEBUG 18:02:58 DEBUG "value": [ 18:02:58 DEBUG 18:02:58 DEBUG "0.0003333333333333854" 18:02:58 DEBUG ] 18:02:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:02:58 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:02:58 DEBUG }, 18:02:58 DEBUG "0.005882286291600428" 18:02:58 DEBUG 1734458578, 18:02:58 DEBUG ] 18:02:58 DEBUG } 18:02:58 DEBUG 1734458578, 18:02:58 DEBUG "instance": "10.142.0.51:9100" 18:02:58 DEBUG ] 18:02:58 DEBUG } 18:02:58 DEBUG }, 18:02:58 DEBUG }, 18:02:58 DEBUG "value": [ 18:02:58 DEBUG ] 18:02:58 DEBUG "27170.13333333333" 18:02:58 DEBUG 18:02:58 DEBUG ] 18:02:58 DEBUG "140433.23333333334" 18:02:58 DEBUG }, 18:02:58 DEBUG } 18:02:58 DEBUG } 18:02:58 DEBUG "value": [ 18:02:58 DEBUG "value": [ 18:02:58 DEBUG 1734458578, 18:02:58 DEBUG } 18:02:58 DEBUG ] 18:02:58 DEBUG 18:02:58 DEBUG } 18:02:58 DEBUG ] 18:02:58 DEBUG "value": [ 18:02:58 DEBUG ] 18:02:58 DEBUG 1734458578, 18:02:58 DEBUG 1734458578, 18:02:58 DEBUG "2757021696" 18:02:58 DEBUG ] 18:02:58 DEBUG } 18:02:58 DEBUG ] 18:02:58 DEBUG } 18:02:58 DEBUG } 18:02:58 DEBUG 1734458578, 18:02:58 DEBUG } 18:02:58 DEBUG "697004032" 18:02:58 DEBUG "1.0627849147347264" 18:02:58 DEBUG ] 18:02:58 DEBUG } 18:02:58 DEBUG } 18:02:58 DEBUG } 18:02:58 DEBUG ] 18:02:58 DEBUG "0.0005666666666660566" 18:02:58 DEBUG } 18:02:58 DEBUG ] 18:02:58 DEBUG ] 18:02:58 DEBUG } 18:02:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:02:58 DEBUG } 18:02:58 DEBUG } 18:02:58 DEBUG ] 18:02:58 DEBUG } 18:02:58 DEBUG } 18:02:58 DEBUG ] 18:02:58 DEBUG }, 18:02:58 DEBUG } 18:02:58 DEBUG } 18:02:58 DEBUG ] 18:02:58 DEBUG ] 18:02:58 DEBUG } 18:02:58 DEBUG "value": [ 18:02:58 DEBUG ] 18:02:58 DEBUG } 18:02:58 DEBUG } 18:02:58 DEBUG } 18:02:58 DEBUG 1734458578, 18:02:58 DEBUG } 18:02:58 DEBUG } 18:02:58 DEBUG } 18:02:58 DEBUG "0" 18:02:58 DEBUG } 18:02:58 DEBUG ] 18:02:58 DEBUG } 18:02:58 DEBUG ] 18:02:58 DEBUG } 18:02:58 DEBUG } 18:02:58 INFO 18:02:58 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:02:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:02:58 INFO 18:02:58 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:02:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:02:58 INFO [loop_until]: OK (rc = 0) 18:02:58 DEBUG --- stdout --- 18:02:58 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 9m 436Mi ds-idrepo-0 10m 631Mi rcs-5d696ff5b-7ttvr 3m 293Mi rcs-5d696ff5b-wwlbr 2m 332Mi 18:02:58 DEBUG --- stderr --- 18:02:58 DEBUG 18:02:58 INFO [loop_until]: OK (rc = 0) 18:02:58 DEBUG --- stdout --- 18:02:58 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 105m 0% 2280Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1139m 3% 3616Mi 12% 18:02:58 DEBUG --- stderr --- 18:02:58 DEBUG 18:03:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458638" 18:03:58 INFO [http_cmd]: http status code OK 18:03:58 DEBUG --- status code --- 18:03:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:03:58 DEBUG --- http response --- 18:03:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:03:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458638" 18:03:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734458638" 18:03:58 INFO [http_cmd]: http status code OK 18:03:58 DEBUG --- status code --- 18:03:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:03:58 DEBUG --- http response --- 18:03:58 INFO [http_cmd]: http status code OK 18:03:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:03:58 DEBUG --- status code --- 18:03:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:03:58 DEBUG --- http response --- 18:03:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:03:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458638" 18:03:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458638" 18:03:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734458638" 18:03:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458638" 18:03:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458638" 18:03:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734458638" 18:03:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458638" 18:03:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458638" 18:03:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458638" 18:03:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458638" 18:03:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458638" 18:03:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458638" 18:03:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458638" 18:03:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458638" 18:03:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458638" 18:03:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734458638" 18:03:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458638" 18:03:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734458638" 18:03:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458638" 18:03:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458638" 18:03:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458638" 18:03:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458638" 18:03:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458638" 18:03:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458638" 18:03:58 INFO [http_cmd]: http status code OK 18:03:58 INFO [http_cmd]: http status code OK 18:03:58 INFO [http_cmd]: http status code OK 18:03:58 INFO [http_cmd]: http status code OK 18:03:58 INFO [http_cmd]: http status code OK 18:03:58 INFO [http_cmd]: http status code OK 18:03:58 INFO [http_cmd]: http status code OK 18:03:58 INFO [http_cmd]: http status code OK 18:03:58 INFO [http_cmd]: http status code OK 18:03:58 INFO [http_cmd]: http status code OK 18:03:58 INFO [http_cmd]: http status code OK 18:03:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458638" 18:03:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458638" 18:03:58 INFO [http_cmd]: http status code OK 18:03:58 INFO [http_cmd]: http status code OK 18:03:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458638" 18:03:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458638" 18:03:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734458638" 18:03:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734458638" 18:03:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458638" 18:03:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458638" 18:03:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458638" 18:03:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458638" 18:03:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458638" 18:03:58 INFO [http_cmd]: http status code OK 18:03:58 INFO [http_cmd]: http status code OK 18:03:58 ERROR [http_cmd]: ERROR 18:03:58 INFO [http_cmd]: http status code OK 18:03:58 DEBUG --- status code --- 18:03:58 INFO [http_cmd]: http status code OK 18:03:58 DEBUG --- status code --- 18:03:58 DEBUG --- status code --- 18:03:58 INFO [http_cmd]: http status code OK 18:03:58 DEBUG --- status code --- 18:03:58 DEBUG --- status code --- 18:03:58 DEBUG --- status code --- 18:03:58 DEBUG --- status code --- 18:03:58 DEBUG --- status code --- 18:03:58 DEBUG --- status code --- 18:03:58 INFO [http_cmd]: http status code OK 18:03:58 DEBUG --- status code --- 18:03:58 DEBUG --- status code --- 18:03:58 DEBUG --- status code --- 18:03:58 DEBUG --- status code --- 18:03:58 INFO [http_cmd]: http status code OK 18:03:58 INFO [http_cmd]: http status code OK 18:03:58 INFO [http_cmd]: http status code OK 18:03:58 INFO [http_cmd]: http status code OK 18:03:58 INFO [http_cmd]: http status code OK 18:03:58 DEBUG --- status code --- 18:03:58 DEBUG --- status code --- 18:03:58 INFO [http_cmd]: http status code OK 18:03:58 ERROR --- status code --- 18:03:58 DEBUG --- status code --- 18:03:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:03:58 DEBUG --- status code --- 18:03:58 INFO [http_cmd]: http status code OK 18:03:58 INFO [http_cmd]: http status code OK 18:03:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:03:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:03:58 DEBUG --- http response --- 18:03:58 INFO [http_cmd]: http status code OK 18:03:58 DEBUG --- status code --- 18:03:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:03:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:03:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:03:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:03:58 INFO [http_cmd]: http status code OK 18:03:58 DEBUG --- status code --- 18:03:58 INFO [http_cmd]: http status code OK 18:03:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:03:58 DEBUG --- status code --- 18:03:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:03:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:03:58 INFO [http_cmd]: http status code OK 18:03:58 INFO [http_cmd]: http status code OK 18:03:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:03:58 DEBUG --- http response --- 18:03:58 DEBUG --- status code --- 18:03:58 DEBUG --- status code --- 18:03:58 DEBUG --- status code --- 18:03:58 DEBUG --- status code --- 18:03:58 DEBUG --- status code --- 18:03:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:03:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:03:58 DEBUG --- http response --- 18:03:58 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:03:58 INFO [http_cmd]: http status code OK 18:03:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:03:58 DEBUG --- http response --- 18:03:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:03:58 DEBUG --- status code --- 18:03:58 DEBUG --- status code --- 18:03:58 DEBUG --- http response --- 18:03:58 INFO [http_cmd]: http status code OK 18:03:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:03:58 DEBUG --- status code --- 18:03:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:03:58 DEBUG --- http response --- 18:03:58 DEBUG --- http response --- 18:03:58 DEBUG --- http response --- 18:03:58 DEBUG --- http response --- 18:03:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:03:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:03:58 DEBUG --- status code --- 18:03:58 DEBUG --- http response --- 18:03:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:03:58 DEBUG --- http response --- 18:03:58 DEBUG --- http response --- 18:03:58 DEBUG --- status code --- 18:03:58 DEBUG --- status code --- 18:03:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:03:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:03:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:03:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:03:58 DEBUG --- http response --- 18:03:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:03:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:03:58 DEBUG --- http response --- 18:03:58 DEBUG --- status code --- 18:03:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:03:58 ERROR --- http response --- 18:03:58 DEBUG --- status code --- 18:03:58 DEBUG --- http response --- 18:03:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:03:58 DEBUG --- http response --- 18:03:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:03:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:03:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:03:58 DEBUG --- status code --- 18:03:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:03:58 DEBUG --- http response --- 18:03:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:03:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:03:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:03:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:03:58 DEBUG --- http response --- 18:03:58 DEBUG --- http response --- 18:03:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:03:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:03:58 DEBUG --- http response --- 18:03:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:03:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:03:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:03:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:03:58 DEBUG --- http response --- 18:03:58 DEBUG --- http response --- 18:03:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:03:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:03:58 DEBUG --- http response --- 18:03:58 DEBUG --- http response --- 18:03:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:03:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:03:58 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:03:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:03:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:03:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:03:58 DEBUG --- http response --- 18:03:58 DEBUG --- http response --- 18:03:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:03:58 DEBUG --- http response --- 18:03:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:03:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:03:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:03:58 DEBUG --- http response --- 18:03:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:03:58 DEBUG --- http response --- 18:03:58 DEBUG --- http response --- 18:03:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:03:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:03:58 DEBUG --- http response --- 18:03:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:03:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:03:58 DEBUG --- http response --- 18:03:58 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:03:58 DEBUG --- http response --- 18:03:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:03:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:03:58 DEBUG ----- output ----- 18:03:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:03:58 DEBUG ----- output ----- 18:03:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:03:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:03:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:03:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:03:58 DEBUG ----- output ----- 18:03:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:03:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:03:58 DEBUG ----- output ----- 18:03:58 DEBUG { 18:03:58 DEBUG { 18:03:58 DEBUG ----- output ----- 18:03:58 DEBUG { 18:03:58 DEBUG ----- output ----- 18:03:58 DEBUG ----- output ----- 18:03:58 DEBUG ----- output ----- 18:03:58 DEBUG { 18:03:58 DEBUG ----- output ----- 18:03:58 DEBUG ----- output ----- 18:03:58 DEBUG --- http response --- 18:03:58 DEBUG "status": "success", 18:03:58 DEBUG "status": "success", 18:03:58 DEBUG { 18:03:58 DEBUG "status": "success", 18:03:58 DEBUG "data": { 18:03:58 DEBUG "resultType": "vector", 18:03:58 DEBUG "result": [ 18:03:58 DEBUG { 18:03:58 DEBUG "metric": { 18:03:58 DEBUG "instance": "10.142.0.11:9100" 18:03:58 DEBUG "data": { 18:03:58 DEBUG "data": { 18:03:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:03:58 DEBUG "status": "success", 18:03:58 DEBUG { 18:03:58 DEBUG { 18:03:58 DEBUG { 18:03:58 DEBUG "status": "success", 18:03:58 DEBUG { 18:03:58 DEBUG { 18:03:58 DEBUG }, 18:03:58 DEBUG "resultType": "vector", 18:03:58 DEBUG "resultType": "vector", 18:03:58 DEBUG "result": [ 18:03:58 DEBUG "data": { 18:03:58 DEBUG "status": "success", 18:03:58 DEBUG "status": "success", 18:03:58 DEBUG "status": "success", 18:03:58 DEBUG "data": { 18:03:58 DEBUG "status": "success", 18:03:58 DEBUG "status": "success", 18:03:58 DEBUG "value": [ 18:03:58 DEBUG "result": [ 18:03:58 DEBUG { 18:03:58 DEBUG { 18:03:58 DEBUG "resultType": "vector", 18:03:58 DEBUG "data": { 18:03:58 DEBUG "data": { 18:03:58 DEBUG "data": { 18:03:58 DEBUG "resultType": "vector", 18:03:58 DEBUG "data": { 18:03:58 DEBUG "data": { 18:03:58 DEBUG 18:03:58 DEBUG 18:03:58 DEBUG 18:03:58 DEBUG "metric": { 18:03:58 DEBUG "result": [ 18:03:58 DEBUG "resultType": "vector", 18:03:58 DEBUG "resultType": "vector", 18:03:58 DEBUG "resultType": "vector", 18:03:58 DEBUG "result": [ 18:03:58 DEBUG { 18:03:58 DEBUG "resultType": "vector", 18:03:58 DEBUG "result": [ 18:03:58 DEBUG "metric": { 18:03:58 DEBUG [...] 18:03:58 DEBUG "instance": "10.142.0.11:9100" 18:03:58 DEBUG { 18:03:58 DEBUG "result": [ 18:03:58 DEBUG "result": [ 18:03:58 DEBUG "result": [ 18:03:58 DEBUG "resultType": "vector", 18:03:58 DEBUG "metric": { 18:03:58 DEBUG "instance": "10.142.0.11:9100" 18:03:58 DEBUG }, 18:03:58 DEBUG "pod": "ds-cts-0" 18:03:58 DEBUG 18:03:58 DEBUG }, 18:03:58 DEBUG "metric": { 18:03:58 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:03:58 DEBUG }, 18:03:58 DEBUG "value": [ 18:03:58 DEBUG 18:03:58 DEBUG 18:03:58 DEBUG 18:03:58 DEBUG "value": [ 18:03:58 DEBUG }, 18:03:58 DEBUG 18:03:58 DEBUG "value": [ 18:03:58 DEBUG { 18:03:58 DEBUG { 18:03:58 DEBUG { 18:03:58 DEBUG "result": [ 18:03:58 DEBUG ----- output ----- 18:03:58 DEBUG { 18:03:58 DEBUG [...] 18:03:58 DEBUG 18:03:58 DEBUG "value": [ 18:03:58 DEBUG 18:03:58 DEBUG 18:03:58 DEBUG "metric": { 18:03:58 DEBUG "metric": { 18:03:58 DEBUG "metric": { 18:03:58 DEBUG { 18:03:58 DEBUG { 18:03:58 DEBUG "metric": { 18:03:58 DEBUG 18:03:58 DEBUG 18:03:58 DEBUG 18:03:58 DEBUG "instance": "10.142.0.51:9100" 18:03:58 DEBUG 18:03:58 DEBUG "pod": "ds-cts-0" 18:03:58 DEBUG "pod": "ds-cts-0" 18:03:58 DEBUG "instance": "10.142.0.11:9100" 18:03:58 DEBUG "metric": { 18:03:58 DEBUG "status": "success", 18:03:58 DEBUG "instance": "10.142.0.11:9100" 18:03:58 DEBUG 18:03:58 DEBUG 18:03:58 DEBUG 18:03:58 DEBUG }, 18:03:58 DEBUG 18:03:58 DEBUG }, 18:03:58 DEBUG }, 18:03:58 DEBUG }, 18:03:58 DEBUG "pod": "ds-cts-0" 18:03:58 DEBUG "data": { 18:03:58 DEBUG }, 18:03:58 DEBUG 18:03:58 DEBUG [...] 18:03:58 DEBUG 18:03:58 DEBUG "value": [ 18:03:58 DEBUG [...] 18:03:58 DEBUG "value": [ 18:03:58 DEBUG "value": [ 18:03:58 DEBUG "value": [ 18:03:58 DEBUG }, 18:03:58 DEBUG "resultType": "vector", 18:03:58 DEBUG "value": [ 18:03:58 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:03:58 DEBUG 18:03:58 DEBUG [...] 18:03:58 DEBUG 1734458638, 18:03:58 DEBUG 18:03:58 DEBUG 18:03:58 DEBUG 18:03:58 DEBUG 18:03:58 DEBUG "value": [ 18:03:58 DEBUG "result": [ 18:03:58 DEBUG 18:03:58 DEBUG }, 18:03:58 DEBUG 18:03:58 DEBUG 18:03:58 DEBUG "123752.39999999997" 18:03:58 DEBUG 18:03:58 DEBUG 18:03:58 DEBUG 18:03:58 DEBUG 18:03:58 DEBUG 18:03:58 DEBUG { 18:03:58 DEBUG 18:03:58 DEBUG "value": [ 18:03:58 DEBUG 18:03:58 DEBUG 18:03:58 DEBUG ] 18:03:58 DEBUG 18:03:58 DEBUG 18:03:58 DEBUG 18:03:58 DEBUG 18:03:58 DEBUG 18:03:58 DEBUG "metric": { 18:03:58 DEBUG 18:03:58 DEBUG 1734458638, 18:03:58 DEBUG "instance": "10.142.0.51:9100" 18:03:58 DEBUG 18:03:58 DEBUG } 18:03:58 DEBUG "instance": "10.142.0.51:9100" 18:03:58 DEBUG [...] 18:03:58 DEBUG [...] 18:03:58 DEBUG [...] 18:03:58 DEBUG 18:03:58 DEBUG [...] 18:03:58 DEBUG 18:03:58 DEBUG "2777214976" 18:03:58 DEBUG }, 18:03:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:03:58 DEBUG ] 18:03:58 DEBUG }, 18:03:58 DEBUG 18:03:58 DEBUG 18:03:58 DEBUG 18:03:58 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:03:58 DEBUG [...] 18:03:58 DEBUG 18:03:58 DEBUG ] 18:03:58 DEBUG "value": [ 18:03:58 DEBUG }, 18:03:58 DEBUG } 18:03:58 DEBUG "value": [ 18:03:58 DEBUG 18:03:58 DEBUG 18:03:58 DEBUG 18:03:58 DEBUG }, 18:03:58 DEBUG 18:03:58 DEBUG 18:03:58 DEBUG } 18:03:58 DEBUG 1734458638, 18:03:58 DEBUG "value": [ 18:03:58 DEBUG } 18:03:58 DEBUG 1734458638, 18:03:58 DEBUG 18:03:58 DEBUG 18:03:58 DEBUG 18:03:58 DEBUG "value": [ 18:03:58 DEBUG 18:03:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:03:58 DEBUG ] 18:03:58 DEBUG "0.0003333333333333854" 18:03:58 DEBUG 1734458638, 18:03:58 DEBUG "127703.70000000001" 18:03:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:03:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:03:58 DEBUG "instance": "10.142.0.51:9100" 18:03:58 DEBUG 18:03:58 DEBUG 18:03:58 DEBUG }, 18:03:58 DEBUG } 18:03:58 DEBUG ] 18:03:58 DEBUG "1967.9918984249389" 18:03:58 DEBUG ] 18:03:58 DEBUG }, 18:03:58 DEBUG }, 18:03:58 DEBUG }, 18:03:58 DEBUG 18:03:58 DEBUG "instance": "10.142.0.51:9100" 18:03:58 DEBUG "value": [ 18:03:58 DEBUG } 18:03:58 DEBUG } 18:03:58 DEBUG ] 18:03:58 DEBUG } 18:03:58 DEBUG "value": [ 18:03:58 DEBUG "value": [ 18:03:58 DEBUG "value": [ 18:03:58 DEBUG 1734458638, 18:03:58 DEBUG "25122.13333333333" 18:03:58 DEBUG ] 18:03:58 DEBUG } 18:03:58 DEBUG ] 18:03:58 DEBUG } 18:03:58 DEBUG } 18:03:58 DEBUG 1734458638, 18:03:58 DEBUG "697503744" 18:03:58 DEBUG ] 18:03:58 DEBUG }, 18:03:58 DEBUG ] 18:03:58 DEBUG } 18:03:58 DEBUG ] 18:03:58 DEBUG 1734458638, 18:03:58 DEBUG 18:03:58 DEBUG } 18:03:58 DEBUG 1734458638, 18:03:58 DEBUG "value": [ 18:03:58 DEBUG } 18:03:58 DEBUG ] 18:03:58 DEBUG } 18:03:58 DEBUG } 18:03:58 DEBUG "0" 18:03:58 DEBUG ] 18:03:58 DEBUG "0.006036226879070053" 18:03:58 DEBUG 1734458638, 18:03:58 DEBUG } 18:03:58 DEBUG } 18:03:58 DEBUG } 18:03:58 DEBUG ] 18:03:58 DEBUG } 18:03:58 DEBUG ] 18:03:58 DEBUG "0.0005666666666660566" 18:03:58 DEBUG [...] 18:03:58 DEBUG } 18:03:58 DEBUG } 18:03:58 DEBUG } 18:03:58 DEBUG ] 18:03:58 DEBUG 18:03:58 DEBUG ] 18:03:58 DEBUG ] 18:03:58 DEBUG } 18:03:58 DEBUG 18:03:58 DEBUG } 18:03:58 DEBUG } 18:03:58 DEBUG ] 18:03:58 DEBUG 18:03:58 DEBUG } 18:03:58 DEBUG } 18:03:58 DEBUG } 18:03:58 DEBUG } 18:03:58 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:03:58 DEBUG }, 18:03:58 DEBUG "value": [ 18:03:58 DEBUG 1734458638, 18:03:58 DEBUG "1.0628577061911157" 18:03:58 DEBUG ] 18:03:58 DEBUG } 18:03:58 DEBUG ] 18:03:58 DEBUG } 18:03:58 DEBUG } 18:03:58 INFO 18:03:58 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:03:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:03:58 INFO 18:03:58 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:03:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:03:58 INFO [loop_until]: OK (rc = 0) 18:03:58 DEBUG --- stdout --- 18:03:58 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 8m 441Mi ds-idrepo-0 8m 637Mi rcs-5d696ff5b-7ttvr 6m 293Mi rcs-5d696ff5b-wwlbr 5m 332Mi 18:03:58 DEBUG --- stderr --- 18:03:58 DEBUG 18:03:58 INFO [loop_until]: OK (rc = 0) 18:03:58 DEBUG --- stdout --- 18:03:58 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 104m 0% 2271Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1134m 3% 4387Mi 15% 18:03:58 DEBUG --- stderr --- 18:03:58 DEBUG 18:04:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458698" 18:04:58 INFO [http_cmd]: http status code OK 18:04:58 DEBUG --- status code --- 18:04:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:04:58 DEBUG --- http response --- 18:04:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:04:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458698" 18:04:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734458698" 18:04:58 INFO [http_cmd]: http status code OK 18:04:58 DEBUG --- status code --- 18:04:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:04:58 INFO [http_cmd]: http status code OK 18:04:58 DEBUG --- http response --- 18:04:58 DEBUG --- status code --- 18:04:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:04:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:04:58 DEBUG --- http response --- 18:04:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:04:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458698" 18:04:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458698" 18:04:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458698" 18:04:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734458698" 18:04:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458698" 18:04:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458698" 18:04:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458698" 18:04:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458698" 18:04:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458698" 18:04:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458698" 18:04:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458698" 18:04:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458698" 18:04:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458698" 18:04:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458698" 18:04:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458698" 18:04:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458698" 18:04:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734458698" 18:04:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458698" 18:04:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734458698" 18:04:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458698" 18:04:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458698" 18:04:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458698" 18:04:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458698" 18:04:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734458698" 18:04:58 INFO [http_cmd]: http status code OK 18:04:58 INFO [http_cmd]: http status code OK 18:04:58 INFO [http_cmd]: http status code OK 18:04:58 INFO [http_cmd]: http status code OK 18:04:58 INFO [http_cmd]: http status code OK 18:04:58 INFO [http_cmd]: http status code OK 18:04:58 INFO [http_cmd]: http status code OK 18:04:58 INFO [http_cmd]: http status code OK 18:04:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458698" 18:04:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734458698" 18:04:58 DEBUG --- status code --- 18:04:58 INFO [http_cmd]: http status code OK 18:04:58 INFO [http_cmd]: http status code OK 18:04:58 INFO [http_cmd]: http status code OK 18:04:58 INFO [http_cmd]: http status code OK 18:04:58 DEBUG --- status code --- 18:04:58 INFO [http_cmd]: http status code OK 18:04:58 DEBUG --- status code --- 18:04:58 INFO [http_cmd]: http status code OK 18:04:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458698" 18:04:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458698" 18:04:58 DEBUG --- status code --- 18:04:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458698" 18:04:58 DEBUG --- status code --- 18:04:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458698" 18:04:58 DEBUG --- status code --- 18:04:58 INFO [http_cmd]: http status code OK 18:04:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458698" 18:04:58 DEBUG --- status code --- 18:04:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:04:58 DEBUG --- status code --- 18:04:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:04:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458698" 18:04:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734458698" 18:04:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458698" 18:04:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:04:58 INFO [http_cmd]: http status code OK 18:04:58 DEBUG --- status code --- 18:04:58 INFO [http_cmd]: http status code OK 18:04:58 INFO [http_cmd]: http status code OK 18:04:58 INFO [http_cmd]: http status code OK 18:04:58 DEBUG --- status code --- 18:04:58 INFO [http_cmd]: http status code OK 18:04:58 DEBUG --- status code --- 18:04:58 INFO [http_cmd]: http status code OK 18:04:58 DEBUG --- status code --- 18:04:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:04:58 DEBUG --- http response --- 18:04:58 DEBUG --- status code --- 18:04:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:04:58 DEBUG --- status code --- 18:04:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:04:58 ERROR [http_cmd]: ERROR 18:04:58 ERROR --- status code --- 18:04:58 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:04:58 INFO [http_cmd]: http status code OK 18:04:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:04:58 DEBUG --- status code --- 18:04:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:04:58 DEBUG --- http response --- 18:04:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458698" 18:04:58 DEBUG --- http response --- 18:04:58 DEBUG --- http response --- 18:04:58 DEBUG --- status code --- 18:04:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:04:58 DEBUG --- status code --- 18:04:58 INFO [http_cmd]: http status code OK 18:04:58 DEBUG --- status code --- 18:04:58 DEBUG --- status code --- 18:04:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:04:58 DEBUG --- status code --- 18:04:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:04:58 INFO [http_cmd]: http status code OK 18:04:58 DEBUG --- status code --- 18:04:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:04:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:04:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:04:58 DEBUG --- http response --- 18:04:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:04:58 DEBUG --- http response --- 18:04:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:04:58 ERROR --- http response --- 18:04:58 INFO [http_cmd]: http status code OK 18:04:58 DEBUG --- status code --- 18:04:58 DEBUG --- http response --- 18:04:58 INFO [http_cmd]: http status code OK 18:04:58 DEBUG --- http response --- 18:04:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:04:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:04:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:04:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:04:58 DEBUG --- http response --- 18:04:58 INFO [http_cmd]: http status code OK 18:04:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:04:58 INFO [http_cmd]: http status code OK 18:04:58 DEBUG --- status code --- 18:04:58 INFO [http_cmd]: http status code OK 18:04:58 INFO [http_cmd]: http status code OK 18:04:58 INFO [http_cmd]: http status code OK 18:04:58 INFO [http_cmd]: http status code OK 18:04:58 INFO [http_cmd]: http status code OK 18:04:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:04:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:04:58 DEBUG --- http response --- 18:04:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:04:58 DEBUG --- http response --- 18:04:58 DEBUG --- status code --- 18:04:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:04:58 DEBUG --- http response --- 18:04:58 DEBUG --- http response --- 18:04:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:04:58 DEBUG --- http response --- 18:04:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:04:58 DEBUG --- http response --- 18:04:58 DEBUG --- status code --- 18:04:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:04:58 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:04:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:04:58 DEBUG --- status code --- 18:04:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:04:58 DEBUG --- http response --- 18:04:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:04:58 DEBUG --- status code --- 18:04:58 DEBUG --- http response --- 18:04:58 INFO [http_cmd]: http status code OK 18:04:58 DEBUG --- status code --- 18:04:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:04:58 DEBUG --- status code --- 18:04:58 DEBUG --- status code --- 18:04:58 DEBUG --- status code --- 18:04:58 DEBUG --- status code --- 18:04:58 DEBUG --- status code --- 18:04:58 DEBUG --- http response --- 18:04:58 DEBUG --- http response --- 18:04:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:04:58 DEBUG --- http response --- 18:04:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:04:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:04:58 DEBUG --- http response --- 18:04:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:04:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:04:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:04:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:04:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:04:58 DEBUG --- http response --- 18:04:58 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:04:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:04:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:04:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:04:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:04:58 DEBUG --- status code --- 18:04:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:04:58 DEBUG --- http response --- 18:04:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:04:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:04:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:04:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:04:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:04:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:04:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:04:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:04:58 DEBUG --- http response --- 18:04:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:04:58 DEBUG --- http response --- 18:04:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:04:58 DEBUG --- http response --- 18:04:58 DEBUG --- http response --- 18:04:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:04:58 DEBUG --- http response --- 18:04:58 DEBUG --- http response --- 18:04:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:04:58 DEBUG ----- output ----- 18:04:58 DEBUG --- http response --- 18:04:58 DEBUG --- http response --- 18:04:58 DEBUG --- http response --- 18:04:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:04:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:04:58 DEBUG ----- output ----- 18:04:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:04:58 DEBUG ----- output ----- 18:04:58 DEBUG { 18:04:58 DEBUG --- http response --- 18:04:58 DEBUG --- http response --- 18:04:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:04:58 DEBUG { 18:04:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:04:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:04:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:04:58 DEBUG ----- output ----- 18:04:58 DEBUG { 18:04:58 DEBUG "status": "success", 18:04:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:04:58 DEBUG "status": "success", 18:04:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:04:58 DEBUG ----- output ----- 18:04:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:04:58 DEBUG "status": "success", 18:04:58 DEBUG ----- output ----- 18:04:58 DEBUG ----- output ----- 18:04:58 DEBUG ----- output ----- 18:04:58 DEBUG { 18:04:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:04:58 DEBUG "data": { 18:04:58 DEBUG ----- output ----- 18:04:58 DEBUG "data": { 18:04:58 DEBUG "resultType": "vector", 18:04:58 DEBUG "result": [ 18:04:58 DEBUG { 18:04:58 DEBUG ----- output ----- 18:04:58 DEBUG { 18:04:58 DEBUG { 18:04:58 DEBUG { 18:04:58 DEBUG "status": "success", 18:04:58 DEBUG "resultType": "vector", 18:04:58 DEBUG { 18:04:58 DEBUG ----- output ----- 18:04:58 DEBUG { 18:04:58 DEBUG "data": { 18:04:58 DEBUG "metric": { 18:04:58 DEBUG "pod": "ds-cts-0" 18:04:58 DEBUG { 18:04:58 DEBUG "status": "success", 18:04:58 DEBUG "status": "success", 18:04:58 DEBUG "data": { 18:04:58 DEBUG "result": [ 18:04:58 DEBUG "status": "success", 18:04:58 DEBUG "data": { 18:04:58 DEBUG { 18:04:58 DEBUG "resultType": "vector", 18:04:58 DEBUG "status": "success", 18:04:58 DEBUG }, 18:04:58 DEBUG "value": [ 18:04:58 DEBUG "status": "success", 18:04:58 DEBUG "data": { 18:04:58 DEBUG "resultType": "vector", 18:04:58 DEBUG { 18:04:58 DEBUG "status": "success", 18:04:58 DEBUG "resultType": "vector", 18:04:58 DEBUG "result": [ 18:04:58 DEBUG "result": [ 18:04:58 DEBUG "data": { 18:04:58 DEBUG "data": { 18:04:58 DEBUG 18:04:58 DEBUG 18:04:58 DEBUG "data": { 18:04:58 DEBUG "result": [ 18:04:58 DEBUG "metric": { 18:04:58 DEBUG "instance": "10.142.0.11:9100" 18:04:58 DEBUG }, 18:04:58 DEBUG { 18:04:58 DEBUG { 18:04:58 DEBUG "resultType": "vector", 18:04:58 DEBUG "resultType": "vector", 18:04:58 DEBUG "resultType": "vector", 18:04:58 DEBUG 18:04:58 DEBUG [...] 18:04:58 DEBUG 18:04:58 DEBUG 18:04:58 DEBUG 18:04:58 DEBUG "value": [ 18:04:58 DEBUG "metric": { 18:04:58 DEBUG "metric": { 18:04:58 DEBUG "result": [ 18:04:58 DEBUG "result": [ 18:04:58 DEBUG "result": [ 18:04:58 DEBUG { 18:04:58 DEBUG "metric": { 18:04:58 DEBUG "pod": "ds-cts-0" 18:04:58 DEBUG }, 18:04:58 DEBUG "value": [ 18:04:58 DEBUG 18:04:58 DEBUG "pod": "ds-cts-0" 18:04:58 DEBUG "instance": "10.142.0.11:9100" 18:04:58 DEBUG { 18:04:58 DEBUG { 18:04:58 DEBUG "metric": { 18:04:58 DEBUG "pod": "ds-cts-0" 18:04:58 DEBUG { 18:04:58 DEBUG "status": "success", 18:04:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:04:58 DEBUG 18:04:58 DEBUG 18:04:58 DEBUG }, 18:04:58 DEBUG }, 18:04:58 DEBUG "metric": { 18:04:58 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:04:58 DEBUG "data": { 18:04:58 DEBUG }, 18:04:58 DEBUG "value": [ 18:04:58 DEBUG 18:04:58 DEBUG }, 18:04:58 DEBUG 18:04:58 DEBUG 18:04:58 DEBUG "value": [ 18:04:58 DEBUG "value": [ 18:04:58 DEBUG 18:04:58 DEBUG }, 18:04:58 DEBUG "resultType": "vector", 18:04:58 DEBUG "metric": { 18:04:58 DEBUG "data": { 18:04:58 DEBUG 18:04:58 DEBUG "value": [ 18:04:58 DEBUG 18:04:58 DEBUG [...] 18:04:58 DEBUG 18:04:58 DEBUG 18:04:58 DEBUG "resultType": "vector", 18:04:58 DEBUG "value": [ 18:04:58 DEBUG "result": [ 18:04:58 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:04:58 DEBUG "resultType": "vector", 18:04:58 DEBUG 18:04:58 DEBUG 1734458698, 18:04:58 DEBUG [...] 18:04:58 DEBUG 18:04:58 DEBUG 18:04:58 DEBUG 18:04:58 DEBUG [...] 18:04:58 DEBUG "result": [ 18:04:58 DEBUG { 18:04:58 DEBUG }, 18:04:58 DEBUG "result": [ 18:04:58 DEBUG [...] 18:04:58 DEBUG "1312.4714922451549" 18:04:58 DEBUG 18:04:58 DEBUG 18:04:58 DEBUG 18:04:58 DEBUG 18:04:58 DEBUG 18:04:58 DEBUG 18:04:58 DEBUG "metric": { 18:04:58 DEBUG "value": [ 18:04:58 DEBUG { 18:04:58 DEBUG 18:04:58 DEBUG ] 18:04:58 DEBUG 18:04:58 DEBUG 18:04:58 DEBUG [...] 18:04:58 DEBUG 18:04:58 DEBUG 18:04:58 DEBUG [...] 18:04:58 DEBUG 18:04:58 DEBUG 18:04:58 DEBUG 18:04:58 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:04:58 DEBUG } 18:04:58 DEBUG 18:04:58 DEBUG "instance": "10.142.0.51:9100" 18:04:58 DEBUG 18:04:58 DEBUG 18:04:58 DEBUG 18:04:58 DEBUG "instance": "10.142.0.11:9100" 18:04:58 DEBUG 18:04:58 DEBUG "metric": { 18:04:58 DEBUG 18:04:58 DEBUG }, 18:04:58 DEBUG ] 18:04:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:04:58 DEBUG }, 18:04:58 DEBUG "value": [ 18:04:58 DEBUG 18:04:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:04:58 DEBUG }, 18:04:58 DEBUG 18:04:58 DEBUG "instance": "10.142.0.11:9100" 18:04:58 DEBUG 18:04:58 DEBUG "value": [ 18:04:58 DEBUG } 18:04:58 DEBUG }, 18:04:58 DEBUG "value": [ 18:04:58 DEBUG 1734458698, 18:04:58 DEBUG "instance": "10.142.0.51:9100" 18:04:58 DEBUG }, 18:04:58 DEBUG "value": [ 18:04:58 DEBUG 18:04:58 DEBUG }, 18:04:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:04:58 DEBUG 1734458698, 18:04:58 DEBUG } 18:04:58 DEBUG { 18:04:58 DEBUG 1734458698, 18:04:58 DEBUG "125595.93333333335" 18:04:58 DEBUG }, 18:04:58 DEBUG "value": [ 18:04:58 DEBUG 18:04:58 DEBUG [...] 18:04:58 DEBUG "value": [ 18:04:58 DEBUG }, 18:04:58 DEBUG "1.0702962683681867" 18:04:58 DEBUG "metric": { 18:04:58 DEBUG "698892288" 18:04:58 DEBUG ] 18:04:58 DEBUG "value": [ 18:04:58 DEBUG 1734458698, 18:04:58 DEBUG 18:04:58 DEBUG 18:04:58 DEBUG 18:04:58 DEBUG "value": [ 18:04:58 DEBUG ] 18:04:58 DEBUG "instance": "10.142.0.11:9100" 18:04:58 DEBUG ] 18:04:58 DEBUG } 18:04:58 DEBUG 1734458698, 18:04:58 DEBUG "0.010717783474799927" 18:04:58 DEBUG 18:04:58 DEBUG [...] 18:04:58 DEBUG 18:04:58 DEBUG 1734458698, 18:04:58 DEBUG } 18:04:58 DEBUG ] 18:04:58 DEBUG } 18:04:58 DEBUG ] 18:04:58 DEBUG "5870.9333333333325" 18:04:58 DEBUG ] 18:04:58 DEBUG } 18:04:58 DEBUG ] 18:04:58 DEBUG 18:04:58 DEBUG "0" 18:04:58 DEBUG ] 18:04:58 DEBUG } 18:04:58 DEBUG ] 18:04:58 DEBUG } 18:04:58 DEBUG ] 18:04:58 DEBUG } 18:04:58 DEBUG ] 18:04:58 DEBUG } 18:04:58 DEBUG 18:04:58 DEBUG "instance": "10.142.0.51:9100" 18:04:58 DEBUG } 18:04:58 DEBUG } 18:04:58 DEBUG } 18:04:58 DEBUG } 18:04:58 DEBUG 18:04:58 DEBUG 18:04:58 DEBUG } 18:04:58 DEBUG } 18:04:58 DEBUG }, 18:04:58 DEBUG }, 18:04:58 DEBUG ] 18:04:58 DEBUG } 18:04:58 DEBUG 18:04:58 DEBUG 18:04:58 DEBUG } 18:04:58 DEBUG "value": [ 18:04:58 DEBUG "value": [ 18:04:58 DEBUG } 18:04:58 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:04:58 DEBUG [...] 18:04:58 DEBUG 18:04:58 DEBUG 1734458698, 18:04:58 DEBUG } 18:04:58 DEBUG }, 18:04:58 DEBUG 18:04:58 DEBUG 18:04:58 DEBUG "0" 18:04:58 DEBUG "value": [ 18:04:58 DEBUG 18:04:58 DEBUG 18:04:58 DEBUG ] 18:04:58 DEBUG 1734458698, 18:04:58 DEBUG 18:04:58 DEBUG "instance": "10.142.0.51:9100" 18:04:58 DEBUG } 18:04:58 DEBUG "5875109888" 18:04:58 DEBUG [...] 18:04:58 DEBUG }, 18:04:58 DEBUG ] 18:04:58 DEBUG ] 18:04:58 DEBUG 18:04:58 DEBUG "value": [ 18:04:58 DEBUG } 18:04:58 DEBUG } 18:04:58 DEBUG 18:04:58 DEBUG 1734458698, 18:04:58 DEBUG } 18:04:58 DEBUG ] 18:04:58 DEBUG "129824.13333333335" 18:04:58 DEBUG 18:04:58 DEBUG } 18:04:58 DEBUG ] 18:04:58 DEBUG "instance": "10.142.0.51:9100" 18:04:58 DEBUG } 18:04:58 DEBUG } 18:04:58 DEBUG }, 18:04:58 DEBUG ] 18:04:58 DEBUG "value": [ 18:04:58 DEBUG } 18:04:58 DEBUG 1734458698, 18:04:58 DEBUG } 18:04:58 DEBUG "0.0003666666666665227" 18:04:58 DEBUG ] 18:04:58 DEBUG } 18:04:58 DEBUG ] 18:04:58 DEBUG } 18:04:58 DEBUG } 18:04:58 INFO 18:04:58 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:04:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:04:58 INFO 18:04:58 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:04:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:04:58 INFO [loop_until]: OK (rc = 0) 18:04:58 DEBUG --- stdout --- 18:04:58 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 7m 444Mi ds-idrepo-0 9m 642Mi rcs-5d696ff5b-7ttvr 6m 294Mi rcs-5d696ff5b-wwlbr 3m 332Mi 18:04:58 DEBUG --- stderr --- 18:04:58 DEBUG 18:04:59 INFO [loop_until]: OK (rc = 0) 18:04:59 DEBUG --- stdout --- 18:04:59 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 103m 0% 2280Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1139m 3% 8244Mi 28% 18:04:59 DEBUG --- stderr --- 18:04:59 DEBUG 18:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458758" 18:05:58 INFO [http_cmd]: http status code OK 18:05:58 DEBUG --- status code --- 18:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:05:58 DEBUG --- http response --- 18:05:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734458758" 18:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458758" 18:05:58 INFO [http_cmd]: http status code OK 18:05:58 INFO [http_cmd]: http status code OK 18:05:58 DEBUG --- status code --- 18:05:58 DEBUG --- status code --- 18:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:05:58 DEBUG --- http response --- 18:05:58 DEBUG --- http response --- 18:05:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:05:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458758" 18:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458758" 18:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458758" 18:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458758" 18:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458758" 18:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458758" 18:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458758" 18:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458758" 18:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458758" 18:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458758" 18:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458758" 18:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734458758" 18:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458758" 18:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458758" 18:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734458758" 18:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458758" 18:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458758" 18:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458758" 18:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458758" 18:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458758" 18:05:58 INFO [http_cmd]: http status code OK 18:05:58 DEBUG --- status code --- 18:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:05:58 DEBUG --- http response --- 18:05:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734458758" 18:05:58 INFO [http_cmd]: http status code OK 18:05:58 DEBUG --- status code --- 18:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:05:58 DEBUG --- http response --- 18:05:58 INFO [http_cmd]: http status code OK 18:05:58 INFO [http_cmd]: http status code OK 18:05:58 INFO [http_cmd]: http status code OK 18:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734458758" 18:05:58 INFO [http_cmd]: http status code OK 18:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458758" 18:05:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458758" 18:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458758" 18:05:58 INFO [http_cmd]: http status code OK 18:05:58 INFO [http_cmd]: http status code OK 18:05:58 DEBUG --- status code --- 18:05:58 DEBUG --- status code --- 18:05:58 ERROR [http_cmd]: ERROR 18:05:58 INFO [http_cmd]: http status code OK 18:05:58 DEBUG --- status code --- 18:05:58 INFO [http_cmd]: http status code OK 18:05:58 INFO [http_cmd]: http status code OK 18:05:58 INFO [http_cmd]: http status code OK 18:05:58 INFO [http_cmd]: http status code OK 18:05:58 INFO [http_cmd]: http status code OK 18:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458758" 18:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458758" 18:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458758" 18:05:58 INFO [http_cmd]: http status code OK 18:05:58 DEBUG --- status code --- 18:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458758" 18:05:58 INFO [http_cmd]: http status code OK 18:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458758" 18:05:58 INFO [http_cmd]: http status code OK 18:05:58 INFO [http_cmd]: http status code OK 18:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734458758" 18:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458758" 18:05:58 INFO [http_cmd]: http status code OK 18:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458758" 18:05:58 INFO [http_cmd]: http status code OK 18:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458758" 18:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734458758" 18:05:58 DEBUG --- status code --- 18:05:58 DEBUG --- status code --- 18:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:05:58 ERROR --- status code --- 18:05:58 DEBUG --- status code --- 18:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:05:58 DEBUG --- status code --- 18:05:58 DEBUG --- status code --- 18:05:58 DEBUG --- status code --- 18:05:58 INFO [http_cmd]: http status code OK 18:05:58 DEBUG --- status code --- 18:05:58 DEBUG --- status code --- 18:05:58 DEBUG --- status code --- 18:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:05:58 DEBUG --- status code --- 18:05:58 INFO [http_cmd]: http status code OK 18:05:58 INFO [http_cmd]: http status code OK 18:05:58 DEBUG --- status code --- 18:05:58 DEBUG --- status code --- 18:05:58 DEBUG --- status code --- 18:05:58 INFO [http_cmd]: http status code OK 18:05:58 DEBUG --- status code --- 18:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:05:58 INFO [http_cmd]: http status code OK 18:05:58 INFO [http_cmd]: http status code OK 18:05:58 DEBUG --- status code --- 18:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:05:58 DEBUG --- http response --- 18:05:58 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:05:58 INFO [http_cmd]: http status code OK 18:05:58 INFO [http_cmd]: http status code OK 18:05:58 INFO [http_cmd]: http status code OK 18:05:58 DEBUG --- http response --- 18:05:58 INFO [http_cmd]: http status code OK 18:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:05:58 DEBUG --- http response --- 18:05:58 DEBUG --- status code --- 18:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:05:58 INFO [http_cmd]: http status code OK 18:05:58 INFO [http_cmd]: http status code OK 18:05:58 DEBUG --- status code --- 18:05:58 DEBUG --- http response --- 18:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:05:58 DEBUG --- status code --- 18:05:58 DEBUG --- status code --- 18:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:05:58 DEBUG --- status code --- 18:05:58 INFO [http_cmd]: http status code OK 18:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:05:58 DEBUG --- http response --- 18:05:58 DEBUG --- status code --- 18:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:05:58 DEBUG --- http response --- 18:05:58 DEBUG --- http response --- 18:05:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:05:58 INFO [http_cmd]: http status code OK 18:05:58 ERROR --- http response --- 18:05:58 DEBUG --- http response --- 18:05:58 DEBUG --- status code --- 18:05:58 DEBUG --- status code --- 18:05:58 DEBUG --- status code --- 18:05:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:05:58 DEBUG --- status code --- 18:05:58 DEBUG --- http response --- 18:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:05:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:05:58 DEBUG --- http response --- 18:05:58 DEBUG --- http response --- 18:05:58 DEBUG --- status code --- 18:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:05:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:05:58 DEBUG --- http response --- 18:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:05:58 DEBUG --- http response --- 18:05:58 DEBUG --- http response --- 18:05:58 DEBUG --- http response --- 18:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:05:58 DEBUG --- status code --- 18:05:58 DEBUG --- http response --- 18:05:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:05:58 DEBUG --- http response --- 18:05:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:05:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:05:58 DEBUG --- status code --- 18:05:58 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:05:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:05:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:05:58 DEBUG --- http response --- 18:05:58 DEBUG --- http response --- 18:05:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:05:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:05:58 DEBUG --- http response --- 18:05:58 DEBUG --- http response --- 18:05:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:05:58 DEBUG --- http response --- 18:05:58 DEBUG --- http response --- 18:05:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:05:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:05:58 DEBUG --- http response --- 18:05:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:05:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:05:58 DEBUG --- http response --- 18:05:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:05:58 DEBUG ----- output ----- 18:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:05:58 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:05:58 DEBUG --- http response --- 18:05:58 DEBUG --- http response --- 18:05:58 DEBUG --- http response --- 18:05:58 DEBUG --- http response --- 18:05:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:05:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:05:58 DEBUG --- http response --- 18:05:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:05:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:05:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:05:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:05:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:05:58 DEBUG --- http response --- 18:05:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:05:58 DEBUG ----- output ----- 18:05:58 DEBUG { 18:05:58 DEBUG --- http response --- 18:05:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:05:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:05:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:05:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:05:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:05:58 DEBUG ----- output ----- 18:05:58 DEBUG ----- output ----- 18:05:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:05:58 DEBUG { 18:05:58 DEBUG "status": "success", 18:05:58 DEBUG "data": { 18:05:58 DEBUG "resultType": "vector", 18:05:58 DEBUG ----- output ----- 18:05:58 DEBUG ----- output ----- 18:05:58 DEBUG ----- output ----- 18:05:58 DEBUG ----- output ----- 18:05:58 DEBUG { 18:05:58 DEBUG { 18:05:58 DEBUG ----- output ----- 18:05:58 DEBUG "status": "success", 18:05:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:05:58 DEBUG ----- output ----- 18:05:58 DEBUG "result": [ 18:05:58 DEBUG { 18:05:58 DEBUG { 18:05:58 DEBUG { 18:05:58 DEBUG { 18:05:58 DEBUG "status": "success", 18:05:58 DEBUG "status": "success", 18:05:58 DEBUG { 18:05:58 DEBUG "data": { 18:05:58 DEBUG ----- output ----- 18:05:58 DEBUG { 18:05:58 DEBUG { 18:05:58 DEBUG "status": "success", 18:05:58 DEBUG "status": "success", 18:05:58 DEBUG "status": "success", 18:05:58 DEBUG "status": "success", 18:05:58 DEBUG "data": { 18:05:58 DEBUG "data": { 18:05:58 DEBUG "status": "success", 18:05:58 DEBUG "resultType": "vector", 18:05:58 DEBUG { 18:05:58 DEBUG "status": "success", 18:05:58 DEBUG "metric": { 18:05:58 DEBUG "data": { 18:05:58 DEBUG "data": { 18:05:58 DEBUG "data": { 18:05:58 DEBUG "data": { 18:05:58 DEBUG "resultType": "vector", 18:05:58 DEBUG "resultType": "vector", 18:05:58 DEBUG "data": { 18:05:58 DEBUG "result": [ 18:05:58 DEBUG "status": "success", 18:05:58 DEBUG "data": { 18:05:58 DEBUG "instance": "10.142.0.11:9100" 18:05:58 DEBUG "resultType": "vector", 18:05:58 DEBUG "resultType": "vector", 18:05:58 DEBUG "resultType": "vector", 18:05:58 DEBUG "resultType": "vector", 18:05:58 DEBUG "result": [ 18:05:58 DEBUG "result": [ 18:05:58 DEBUG "resultType": "vector", 18:05:58 DEBUG { 18:05:58 DEBUG "data": { 18:05:58 DEBUG "resultType": "vector", 18:05:58 DEBUG }, 18:05:58 DEBUG "result": [ 18:05:58 DEBUG "result": [ 18:05:58 DEBUG "result": [ 18:05:58 DEBUG "result": [ 18:05:58 DEBUG { 18:05:58 DEBUG { 18:05:58 DEBUG "result": [ 18:05:58 DEBUG "metric": { 18:05:58 DEBUG "resultType": "vector", 18:05:58 DEBUG "result": [ 18:05:58 DEBUG "value": [ 18:05:58 DEBUG { 18:05:58 DEBUG { 18:05:58 DEBUG { 18:05:58 DEBUG "metric": { 18:05:58 DEBUG "metric": { 18:05:58 DEBUG "metric": { 18:05:58 DEBUG { 18:05:58 DEBUG "pod": "ds-cts-0" 18:05:58 DEBUG "result": [ 18:05:58 DEBUG { 18:05:58 DEBUG 18:05:58 DEBUG "metric": { 18:05:58 DEBUG "metric": { 18:05:58 DEBUG { 18:05:58 DEBUG "instance": "10.142.0.11:9100" 18:05:58 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:05:58 DEBUG "pod": "ds-cts-0" 18:05:58 DEBUG "metric": { 18:05:58 DEBUG }, 18:05:58 DEBUG { 18:05:58 DEBUG "metric": { 18:05:58 DEBUG 18:05:58 DEBUG "pod": "ds-cts-0" 18:05:58 DEBUG "instance": "10.142.0.11:9100" 18:05:58 DEBUG "metric": { 18:05:58 DEBUG }, 18:05:58 DEBUG }, 18:05:58 DEBUG }, 18:05:58 DEBUG "instance": "10.142.0.11:9100" 18:05:58 DEBUG "value": [ 18:05:58 DEBUG "metric": { 18:05:58 DEBUG "pod": "ds-cts-0" 18:05:58 DEBUG 18:05:58 DEBUG }, 18:05:58 DEBUG }, 18:05:58 DEBUG "instance": "10.142.0.11:9100" 18:05:58 DEBUG "value": [ 18:05:58 DEBUG "value": [ 18:05:58 DEBUG "value": [ 18:05:58 DEBUG }, 18:05:58 DEBUG 18:05:58 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:05:58 DEBUG }, 18:05:58 DEBUG [...] 18:05:58 DEBUG "value": [ 18:05:58 DEBUG "value": [ 18:05:58 DEBUG }, 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG "value": [ 18:05:58 DEBUG 18:05:58 DEBUG }, 18:05:58 DEBUG "value": [ 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG "value": [ 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG "value": [ 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG [...] 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG [...] 18:05:58 DEBUG [...] 18:05:58 DEBUG [...] 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG "instance": "10.142.0.51:9100" 18:05:58 DEBUG [...] 18:05:58 DEBUG [...] 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG [...] 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG [...] 18:05:58 DEBUG }, 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG [...] 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG [...] 18:05:58 DEBUG 18:05:58 DEBUG "value": [ 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG 1734458758, 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG "instance": "10.142.0.51:9100" 18:05:58 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:05:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:05:58 DEBUG 18:05:58 DEBUG }, 18:05:58 DEBUG 18:05:58 DEBUG 18:05:58 DEBUG "120401.09999999996" 18:05:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:05:58 DEBUG "instance": "10.142.0.51:9100" 18:05:58 DEBUG 18:05:58 DEBUG }, 18:05:58 DEBUG }, 18:05:58 DEBUG }, 18:05:58 DEBUG "instance": "10.142.0.51:9100" 18:05:58 DEBUG "value": [ 18:05:58 DEBUG 18:05:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:05:58 DEBUG ] 18:05:58 DEBUG }, 18:05:58 DEBUG }, 18:05:58 DEBUG "instance": "10.142.0.51:9100" 18:05:58 DEBUG "value": [ 18:05:58 DEBUG "value": [ 18:05:58 DEBUG "value": [ 18:05:58 DEBUG }, 18:05:58 DEBUG 1734458758, 18:05:58 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:05:58 DEBUG }, 18:05:58 DEBUG } 18:05:58 DEBUG "value": [ 18:05:58 DEBUG "value": [ 18:05:58 DEBUG }, 18:05:58 DEBUG 1734458758, 18:05:58 DEBUG 1734458758, 18:05:58 DEBUG 1734458758, 18:05:58 DEBUG "value": [ 18:05:58 DEBUG "699351040" 18:05:58 DEBUG }, 18:05:58 DEBUG "value": [ 18:05:58 DEBUG ] 18:05:58 DEBUG 1734458758, 18:05:58 DEBUG 1734458758, 18:05:58 DEBUG "value": [ 18:05:58 DEBUG "25531.733333333334" 18:05:58 DEBUG "1.0619549294442743" 18:05:58 DEBUG ] 18:05:58 DEBUG 1734458758, 18:05:58 DEBUG ] 18:05:58 DEBUG "value": [ 18:05:58 DEBUG 1734458758, 18:05:58 DEBUG } 18:05:58 DEBUG "0" 18:05:58 DEBUG "0.000333333333333267" 18:05:58 DEBUG 1734458758, 18:05:58 DEBUG ] 18:05:58 DEBUG "1568.393883771527" 18:05:58 DEBUG } 18:05:58 DEBUG "124242.1" 18:05:58 DEBUG } 18:05:58 DEBUG 1734458758, 18:05:58 DEBUG "0.00496262390909184" 18:05:58 DEBUG } 18:05:58 DEBUG ] 18:05:58 DEBUG ] 18:05:58 DEBUG "0.00046666666666463167" 18:05:58 DEBUG } 18:05:58 DEBUG ] 18:05:58 DEBUG ] 18:05:58 DEBUG ] 18:05:58 DEBUG ] 18:05:58 DEBUG "10236461056" 18:05:58 DEBUG ] 18:05:58 DEBUG } 18:05:58 DEBUG } 18:05:58 DEBUG ] 18:05:58 DEBUG ] 18:05:58 DEBUG } 18:05:58 DEBUG } 18:05:58 DEBUG } 18:05:58 DEBUG } 18:05:58 DEBUG ] 18:05:58 DEBUG } 18:05:58 DEBUG ] 18:05:58 DEBUG ] 18:05:58 DEBUG } 18:05:58 DEBUG } 18:05:58 DEBUG ] 18:05:58 DEBUG } 18:05:58 DEBUG ] 18:05:58 DEBUG } 18:05:58 DEBUG } 18:05:58 DEBUG ] 18:05:58 DEBUG } 18:05:58 DEBUG } 18:05:58 DEBUG ] 18:05:58 DEBUG } 18:05:58 DEBUG } 18:05:58 DEBUG } 18:05:58 DEBUG ] 18:05:58 DEBUG } 18:05:58 DEBUG } 18:05:58 DEBUG } 18:05:58 DEBUG } 18:05:58 DEBUG } 18:05:58 DEBUG } 18:05:58 DEBUG } 18:05:58 DEBUG } 18:05:58 DEBUG } 18:05:58 DEBUG } 18:05:58 INFO 18:05:58 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:05:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:05:59 INFO 18:05:59 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:05:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:05:59 INFO [loop_until]: OK (rc = 0) 18:05:59 DEBUG --- stdout --- 18:05:59 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 9m 448Mi ds-idrepo-0 14m 648Mi rcs-5d696ff5b-7ttvr 7m 294Mi rcs-5d696ff5b-wwlbr 3m 333Mi 18:05:59 DEBUG --- stderr --- 18:05:59 DEBUG 18:05:59 INFO [loop_until]: OK (rc = 0) 18:05:59 DEBUG --- stdout --- 18:05:59 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 98m 0% 2260Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1152m 3% 3649Mi 12% 18:05:59 DEBUG --- stderr --- 18:05:59 DEBUG 18:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458818" 18:06:58 INFO [http_cmd]: http status code OK 18:06:58 DEBUG --- status code --- 18:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:06:58 DEBUG --- http response --- 18:06:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734458818" 18:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458818" 18:06:58 INFO [http_cmd]: http status code OK 18:06:58 DEBUG --- status code --- 18:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:06:58 DEBUG --- http response --- 18:06:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:06:58 INFO [http_cmd]: http status code OK 18:06:58 DEBUG --- status code --- 18:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:06:58 DEBUG --- http response --- 18:06:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458818" 18:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458818" 18:06:58 INFO [http_cmd]: http status code OK 18:06:58 DEBUG --- status code --- 18:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:06:58 DEBUG --- http response --- 18:06:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:06:58 INFO [http_cmd]: http status code OK 18:06:58 DEBUG --- status code --- 18:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:06:58 DEBUG --- http response --- 18:06:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458818" 18:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458818" 18:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458818" 18:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458818" 18:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458818" 18:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458818" 18:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458818" 18:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458818" 18:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458818" 18:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458818" 18:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458818" 18:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734458818" 18:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458818" 18:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458818" 18:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734458818" 18:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458818" 18:06:58 INFO [http_cmd]: http status code OK 18:06:58 DEBUG --- status code --- 18:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:06:58 DEBUG --- http response --- 18:06:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458818" 18:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734458818" 18:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458818" 18:06:58 INFO [http_cmd]: http status code OK 18:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458818" 18:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734458818" 18:06:58 INFO [http_cmd]: http status code OK 18:06:58 INFO [http_cmd]: http status code OK 18:06:58 INFO [http_cmd]: http status code OK 18:06:58 INFO [http_cmd]: http status code OK 18:06:58 DEBUG --- status code --- 18:06:58 INFO [http_cmd]: http status code OK 18:06:58 INFO [http_cmd]: http status code OK 18:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458818" 18:06:58 INFO [http_cmd]: http status code OK 18:06:58 INFO [http_cmd]: http status code OK 18:06:58 ERROR [http_cmd]: ERROR 18:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458818" 18:06:58 INFO [http_cmd]: http status code OK 18:06:58 INFO [http_cmd]: http status code OK 18:06:58 INFO [http_cmd]: http status code OK 18:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734458818" 18:06:58 DEBUG --- status code --- 18:06:58 INFO [http_cmd]: http status code OK 18:06:58 INFO [http_cmd]: http status code OK 18:06:58 INFO [http_cmd]: http status code OK 18:06:58 INFO [http_cmd]: http status code OK 18:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458818" 18:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458818" 18:06:58 DEBUG --- status code --- 18:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458818" 18:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458818" 18:06:58 DEBUG --- status code --- 18:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458818" 18:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734458818" 18:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458818" 18:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458818" 18:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458818" 18:06:58 DEBUG --- status code --- 18:06:58 INFO [http_cmd]: http status code OK 18:06:58 INFO [http_cmd]: http status code OK 18:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:06:58 DEBUG --- status code --- 18:06:58 DEBUG --- status code --- 18:06:58 DEBUG --- status code --- 18:06:58 INFO [http_cmd]: http status code OK 18:06:58 DEBUG --- status code --- 18:06:58 ERROR --- status code --- 18:06:58 DEBUG --- status code --- 18:06:58 DEBUG --- status code --- 18:06:58 DEBUG --- status code --- 18:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:06:58 DEBUG --- status code --- 18:06:58 DEBUG --- status code --- 18:06:58 DEBUG --- status code --- 18:06:58 DEBUG --- status code --- 18:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:06:58 INFO [http_cmd]: http status code OK 18:06:58 DEBUG --- status code --- 18:06:58 DEBUG --- status code --- 18:06:58 DEBUG --- status code --- 18:06:58 INFO [http_cmd]: http status code OK 18:06:58 INFO [http_cmd]: http status code OK 18:06:58 INFO [http_cmd]: http status code OK 18:06:58 INFO [http_cmd]: http status code OK 18:06:58 DEBUG --- http response --- 18:06:58 INFO [http_cmd]: http status code OK 18:06:58 DEBUG --- status code --- 18:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:06:58 INFO [http_cmd]: http status code OK 18:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:06:58 DEBUG --- http response --- 18:06:58 INFO [http_cmd]: http status code OK 18:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:06:58 INFO [http_cmd]: http status code OK 18:06:58 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:06:58 DEBUG --- http response --- 18:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:06:58 INFO [http_cmd]: http status code OK 18:06:58 INFO [http_cmd]: http status code OK 18:06:58 DEBUG --- status code --- 18:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:06:58 DEBUG --- http response --- 18:06:58 DEBUG --- http response --- 18:06:58 DEBUG --- http response --- 18:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:06:58 DEBUG --- status code --- 18:06:58 DEBUG --- status code --- 18:06:58 DEBUG --- status code --- 18:06:58 DEBUG --- status code --- 18:06:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:06:58 DEBUG --- http response --- 18:06:58 DEBUG --- status code --- 18:06:58 DEBUG --- status code --- 18:06:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:06:58 DEBUG --- status code --- 18:06:58 DEBUG --- http response --- 18:06:58 DEBUG --- status code --- 18:06:58 ERROR --- http response --- 18:06:58 DEBUG --- http response --- 18:06:58 DEBUG --- http response --- 18:06:58 DEBUG --- http response --- 18:06:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:06:58 DEBUG --- http response --- 18:06:58 DEBUG --- status code --- 18:06:58 INFO [http_cmd]: http status code OK 18:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:06:58 DEBUG --- http response --- 18:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:06:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:06:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:06:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:06:58 DEBUG --- http response --- 18:06:58 DEBUG --- http response --- 18:06:58 DEBUG --- http response --- 18:06:58 DEBUG --- http response --- 18:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:06:58 DEBUG --- http response --- 18:06:58 DEBUG --- http response --- 18:06:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:06:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:06:58 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:06:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:06:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:06:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:06:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:06:58 DEBUG --- status code --- 18:06:58 DEBUG --- http response --- 18:06:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:06:58 DEBUG --- http response --- 18:06:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:06:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:06:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:06:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:06:58 DEBUG --- http response --- 18:06:58 DEBUG --- http response --- 18:06:58 DEBUG --- http response --- 18:06:58 DEBUG --- http response --- 18:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:06:58 DEBUG --- http response --- 18:06:58 DEBUG --- http response --- 18:06:58 DEBUG --- http response --- 18:06:58 DEBUG --- http response --- 18:06:58 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:06:58 DEBUG --- http response --- 18:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:06:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:06:58 DEBUG ----- output ----- 18:06:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:06:58 DEBUG ----- output ----- 18:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:06:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:06:58 DEBUG --- http response --- 18:06:58 DEBUG ----- output ----- 18:06:58 DEBUG ----- output ----- 18:06:58 DEBUG ----- output ----- 18:06:58 DEBUG ----- output ----- 18:06:58 DEBUG { 18:06:58 DEBUG { 18:06:58 DEBUG ----- output ----- 18:06:58 DEBUG ----- output ----- 18:06:58 DEBUG ----- output ----- 18:06:58 DEBUG ----- output ----- 18:06:58 DEBUG { 18:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:06:58 DEBUG { 18:06:58 DEBUG { 18:06:58 DEBUG { 18:06:58 DEBUG "status": "success", 18:06:58 DEBUG "status": "success", 18:06:58 DEBUG { 18:06:58 DEBUG { 18:06:58 DEBUG { 18:06:58 DEBUG { 18:06:58 DEBUG "status": "success", 18:06:58 DEBUG ----- output ----- 18:06:58 DEBUG "status": "success", 18:06:58 DEBUG "status": "success", 18:06:58 DEBUG "status": "success", 18:06:58 DEBUG "data": { 18:06:58 DEBUG "data": { 18:06:58 DEBUG "status": "success", 18:06:58 DEBUG "status": "success", 18:06:58 DEBUG "status": "success", 18:06:58 DEBUG "status": "success", 18:06:58 DEBUG "data": { 18:06:58 DEBUG { 18:06:58 DEBUG "data": { 18:06:58 DEBUG "data": { 18:06:58 DEBUG "data": { 18:06:58 DEBUG "resultType": "vector", 18:06:58 DEBUG "resultType": "vector", 18:06:58 DEBUG "data": { 18:06:58 DEBUG "data": { 18:06:58 DEBUG "data": { 18:06:58 DEBUG "data": { 18:06:58 DEBUG "resultType": "vector", 18:06:58 DEBUG "status": "success", 18:06:58 DEBUG "resultType": "vector", 18:06:58 DEBUG "resultType": "vector", 18:06:58 DEBUG "resultType": "vector", 18:06:58 DEBUG "result": [ 18:06:58 DEBUG "result": [ 18:06:58 DEBUG "resultType": "vector", 18:06:58 DEBUG "resultType": "vector", 18:06:58 DEBUG "resultType": "vector", 18:06:58 DEBUG "resultType": "vector", 18:06:58 DEBUG "result": [ 18:06:58 DEBUG "data": { 18:06:58 DEBUG "result": [ 18:06:58 DEBUG "result": [ 18:06:58 DEBUG "result": [ 18:06:58 DEBUG { 18:06:58 DEBUG { 18:06:58 DEBUG "result": [ 18:06:58 DEBUG "result": [ 18:06:58 DEBUG "result": [ 18:06:58 DEBUG "result": [ 18:06:58 DEBUG { 18:06:58 DEBUG "resultType": "vector", 18:06:58 DEBUG { 18:06:58 DEBUG { 18:06:58 DEBUG { 18:06:58 DEBUG "metric": { 18:06:58 DEBUG "metric": { 18:06:58 DEBUG { 18:06:58 DEBUG { 18:06:58 DEBUG { 18:06:58 DEBUG { 18:06:58 DEBUG "metric": { 18:06:58 DEBUG "result": [ 18:06:58 DEBUG "metric": { 18:06:58 DEBUG "metric": { 18:06:58 DEBUG "metric": { 18:06:58 DEBUG "instance": "10.142.0.11:9100" 18:06:58 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:06:58 DEBUG "metric": { 18:06:58 DEBUG "metric": { 18:06:58 DEBUG "metric": { 18:06:58 DEBUG "metric": { 18:06:58 DEBUG "instance": "10.142.0.11:9100" 18:06:58 DEBUG { 18:06:58 DEBUG "pod": "ds-cts-0" 18:06:58 DEBUG "pod": "ds-cts-0" 18:06:58 DEBUG "instance": "10.142.0.11:9100" 18:06:58 DEBUG }, 18:06:58 DEBUG }, 18:06:58 DEBUG "instance": "10.142.0.11:9100" 18:06:58 DEBUG "instance": "10.142.0.11:9100" 18:06:58 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:06:58 DEBUG "pod": "ds-cts-0" 18:06:58 DEBUG }, 18:06:58 DEBUG "metric": { 18:06:58 DEBUG }, 18:06:58 DEBUG }, 18:06:58 DEBUG }, 18:06:58 DEBUG "value": [ 18:06:58 DEBUG "value": [ 18:06:58 DEBUG }, 18:06:58 DEBUG }, 18:06:58 DEBUG }, 18:06:58 DEBUG }, 18:06:58 DEBUG "value": [ 18:06:58 DEBUG "pod": "ds-cts-0" 18:06:58 DEBUG "value": [ 18:06:58 DEBUG "value": [ 18:06:58 DEBUG "value": [ 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG "value": [ 18:06:58 DEBUG "value": [ 18:06:58 DEBUG "value": [ 18:06:58 DEBUG "value": [ 18:06:58 DEBUG 18:06:58 DEBUG }, 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG "value": [ 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG [...] 18:06:58 DEBUG [...] 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG [...] 18:06:58 DEBUG 18:06:58 DEBUG [...] 18:06:58 DEBUG [...] 18:06:58 DEBUG [...] 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG [...] 18:06:58 DEBUG [...] 18:06:58 DEBUG [...] 18:06:58 DEBUG [...] 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:06:58 DEBUG }, 18:06:58 DEBUG "value": [ 18:06:58 DEBUG 1734458818, 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG [...] 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG "2738593792" 18:06:58 DEBUG "instance": "10.142.0.51:9100" 18:06:58 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG 18:06:58 DEBUG ] 18:06:58 DEBUG }, 18:06:58 DEBUG }, 18:06:58 DEBUG "instance": "10.142.0.51:9100" 18:06:58 DEBUG "instance": "10.142.0.51:9100" 18:06:58 DEBUG 18:06:58 DEBUG "instance": "10.142.0.51:9100" 18:06:58 DEBUG 18:06:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:06:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:06:58 DEBUG "instance": "10.142.0.51:9100" 18:06:58 DEBUG } 18:06:58 DEBUG "value": [ 18:06:58 DEBUG "value": [ 18:06:58 DEBUG }, 18:06:58 DEBUG }, 18:06:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:06:58 DEBUG }, 18:06:58 DEBUG 18:06:58 DEBUG }, 18:06:58 DEBUG }, 18:06:58 DEBUG }, 18:06:58 DEBUG ] 18:06:58 DEBUG 1734458818, 18:06:58 DEBUG 1734458818, 18:06:58 DEBUG "value": [ 18:06:58 DEBUG "value": [ 18:06:58 DEBUG }, 18:06:58 DEBUG "value": [ 18:06:58 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:06:58 DEBUG "value": [ 18:06:58 DEBUG "value": [ 18:06:58 DEBUG "value": [ 18:06:58 DEBUG } 18:06:58 DEBUG "24029.86666666667" 18:06:58 DEBUG "1.063564361083398" 18:06:58 DEBUG 1734458818, 18:06:58 DEBUG 1734458818, 18:06:58 DEBUG "value": [ 18:06:58 DEBUG 1734458818, 18:06:58 DEBUG }, 18:06:58 DEBUG 1734458818, 18:06:58 DEBUG 1734458818, 18:06:58 DEBUG 1734458818, 18:06:58 DEBUG } 18:06:58 DEBUG ] 18:06:58 DEBUG ] 18:06:58 DEBUG "0.0003333333333333854" 18:06:58 DEBUG "121826.93333333333" 18:06:58 DEBUG 1734458818, 18:06:58 DEBUG "0.0003333333333306617" 18:06:58 DEBUG "value": [ 18:06:58 DEBUG "699809792" 18:06:58 DEBUG "931.3015464233613" 18:06:58 DEBUG "117426.63333333332" 18:06:58 DEBUG } 18:06:58 DEBUG } 18:06:58 DEBUG ] 18:06:58 DEBUG ] 18:06:58 DEBUG "0.008299782445455779" 18:06:58 DEBUG ] 18:06:58 DEBUG 1734458818, 18:06:58 DEBUG ] 18:06:58 DEBUG ] 18:06:58 DEBUG ] 18:06:58 DEBUG ] 18:06:58 DEBUG ] 18:06:58 DEBUG } 18:06:58 DEBUG } 18:06:58 DEBUG ] 18:06:58 DEBUG } 18:06:58 DEBUG "0" 18:06:58 DEBUG } 18:06:58 DEBUG } 18:06:58 DEBUG } 18:06:58 DEBUG } 18:06:58 DEBUG } 18:06:58 DEBUG ] 18:06:58 DEBUG ] 18:06:58 DEBUG } 18:06:58 DEBUG ] 18:06:58 DEBUG ] 18:06:58 DEBUG ] 18:06:58 DEBUG ] 18:06:58 DEBUG ] 18:06:58 DEBUG } 18:06:58 DEBUG } 18:06:58 DEBUG } 18:06:58 DEBUG } 18:06:58 DEBUG ] 18:06:58 DEBUG } 18:06:58 DEBUG } 18:06:58 DEBUG } 18:06:58 DEBUG } 18:06:58 DEBUG } 18:06:58 DEBUG } 18:06:58 DEBUG } 18:06:58 DEBUG } 18:06:58 DEBUG } 18:06:58 DEBUG ] 18:06:58 DEBUG } 18:06:58 DEBUG } 18:06:58 DEBUG } 18:06:58 DEBUG } 18:06:58 DEBUG } 18:06:58 DEBUG } 18:06:59 INFO 18:06:59 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:06:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:06:59 INFO 18:06:59 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:06:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:06:59 INFO [loop_until]: OK (rc = 0) 18:06:59 DEBUG --- stdout --- 18:06:59 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 7m 450Mi ds-idrepo-0 10m 653Mi rcs-5d696ff5b-7ttvr 5m 294Mi rcs-5d696ff5b-wwlbr 5m 333Mi 18:06:59 DEBUG --- stderr --- 18:06:59 DEBUG 18:06:59 INFO [loop_until]: OK (rc = 0) 18:06:59 DEBUG --- stdout --- 18:06:59 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 100m 0% 2267Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1132m 3% 3658Mi 12% 18:06:59 DEBUG --- stderr --- 18:06:59 DEBUG 18:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458878" 18:07:58 INFO [http_cmd]: http status code OK 18:07:58 DEBUG --- status code --- 18:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:07:58 DEBUG --- http response --- 18:07:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734458878" 18:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458878" 18:07:58 INFO [http_cmd]: http status code OK 18:07:58 DEBUG --- status code --- 18:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:07:58 DEBUG --- http response --- 18:07:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:07:58 INFO [http_cmd]: http status code OK 18:07:58 DEBUG --- status code --- 18:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:07:58 DEBUG --- http response --- 18:07:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458878" 18:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458878" 18:07:58 INFO [http_cmd]: http status code OK 18:07:58 DEBUG --- status code --- 18:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:07:58 DEBUG --- http response --- 18:07:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:07:58 INFO [http_cmd]: http status code OK 18:07:58 DEBUG --- status code --- 18:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:07:58 DEBUG --- http response --- 18:07:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458878" 18:07:58 INFO [http_cmd]: http status code OK 18:07:58 DEBUG --- status code --- 18:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:07:58 DEBUG --- http response --- 18:07:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458878" 18:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458878" 18:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458878" 18:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458878" 18:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458878" 18:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458878" 18:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458878" 18:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458878" 18:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458878" 18:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458878" 18:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734458878" 18:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458878" 18:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734458878" 18:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734458878" 18:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458878" 18:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458878" 18:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458878" 18:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458878" 18:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458878" 18:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458878" 18:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734458878" 18:07:58 INFO [http_cmd]: http status code OK 18:07:58 INFO [http_cmd]: http status code OK 18:07:58 INFO [http_cmd]: http status code OK 18:07:58 INFO [http_cmd]: http status code OK 18:07:58 INFO [http_cmd]: http status code OK 18:07:58 INFO [http_cmd]: http status code OK 18:07:58 INFO [http_cmd]: http status code OK 18:07:58 DEBUG --- status code --- 18:07:58 INFO [http_cmd]: http status code OK 18:07:58 DEBUG --- status code --- 18:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734458878" 18:07:58 DEBUG --- status code --- 18:07:58 DEBUG --- status code --- 18:07:58 DEBUG --- status code --- 18:07:58 DEBUG --- status code --- 18:07:58 DEBUG --- status code --- 18:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:07:58 DEBUG --- status code --- 18:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:07:58 DEBUG --- http response --- 18:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:07:58 DEBUG --- http response --- 18:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:07:58 DEBUG --- http response --- 18:07:58 DEBUG --- http response --- 18:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:07:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:07:58 DEBUG --- http response --- 18:07:58 DEBUG --- http response --- 18:07:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458878" 18:07:58 DEBUG --- http response --- 18:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734458878" 18:07:58 INFO [http_cmd]: http status code OK 18:07:58 INFO [http_cmd]: http status code OK 18:07:58 INFO [http_cmd]: http status code OK 18:07:58 ERROR [http_cmd]: ERROR 18:07:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:07:58 INFO [http_cmd]: http status code OK 18:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458878" 18:07:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:07:58 INFO [http_cmd]: http status code OK 18:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458878" 18:07:58 DEBUG --- http response --- 18:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458878" 18:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458878" 18:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458878" 18:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458878" 18:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458878" 18:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458878" 18:07:58 INFO [http_cmd]: http status code OK 18:07:58 INFO [http_cmd]: http status code OK 18:07:58 INFO [http_cmd]: http status code OK 18:07:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:07:59 INFO [http_cmd]: http status code OK 18:07:59 INFO [http_cmd]: http status code OK 18:07:59 INFO [http_cmd]: http status code OK 18:07:59 DEBUG --- status code --- 18:07:59 INFO [http_cmd]: http status code OK 18:07:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:07:59 INFO [http_cmd]: http status code OK 18:07:59 DEBUG --- status code --- 18:07:59 DEBUG --- status code --- 18:07:59 DEBUG --- status code --- 18:07:59 ERROR --- status code --- 18:07:59 DEBUG --- status code --- 18:07:59 DEBUG --- status code --- 18:07:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:07:59 DEBUG --- status code --- 18:07:59 INFO [http_cmd]: http status code OK 18:07:59 DEBUG --- status code --- 18:07:59 INFO [http_cmd]: http status code OK 18:07:59 DEBUG --- status code --- 18:07:59 DEBUG --- status code --- 18:07:59 DEBUG --- status code --- 18:07:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:07:59 INFO [http_cmd]: http status code OK 18:07:59 INFO [http_cmd]: http status code OK 18:07:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:07:59 INFO [http_cmd]: http status code OK 18:07:59 DEBUG --- status code --- 18:07:59 DEBUG --- status code --- 18:07:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:07:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:07:59 INFO [http_cmd]: http status code OK 18:07:59 INFO [http_cmd]: http status code OK 18:07:59 INFO [http_cmd]: http status code OK 18:07:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:07:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:07:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:07:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:07:59 DEBUG --- http response --- 18:07:59 INFO [http_cmd]: http status code OK 18:07:59 DEBUG --- status code --- 18:07:59 INFO [http_cmd]: http status code OK 18:07:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:07:59 DEBUG --- status code --- 18:07:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:07:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:07:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:07:59 DEBUG --- status code --- 18:07:59 DEBUG --- status code --- 18:07:59 DEBUG --- http response --- 18:07:59 DEBUG --- status code --- 18:07:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:07:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:07:59 DEBUG --- http response --- 18:07:59 DEBUG --- http response --- 18:07:59 DEBUG --- status code --- 18:07:59 DEBUG --- status code --- 18:07:59 DEBUG --- status code --- 18:07:59 DEBUG --- http response --- 18:07:59 ERROR --- http response --- 18:07:59 DEBUG --- http response --- 18:07:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:07:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:07:59 DEBUG --- status code --- 18:07:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:07:59 DEBUG --- status code --- 18:07:59 DEBUG --- http response --- 18:07:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:07:59 DEBUG --- http response --- 18:07:59 DEBUG --- http response --- 18:07:59 DEBUG --- http response --- 18:07:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:07:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:07:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:07:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:07:59 DEBUG --- http response --- 18:07:59 DEBUG --- http response --- 18:07:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:07:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:07:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:07:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:07:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:07:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:07:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:07:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:07:59 DEBUG --- http response --- 18:07:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:07:59 DEBUG --- http response --- 18:07:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:07:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:07:59 DEBUG --- http response --- 18:07:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:07:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:07:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:07:59 DEBUG --- http response --- 18:07:59 DEBUG --- http response --- 18:07:59 DEBUG --- http response --- 18:07:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:07:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:07:59 DEBUG --- http response --- 18:07:59 DEBUG --- http response --- 18:07:59 DEBUG --- http response --- 18:07:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:07:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:07:59 DEBUG --- http response --- 18:07:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:07:59 DEBUG --- http response --- 18:07:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:07:59 DEBUG ----- output ----- 18:07:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:07:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:07:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:07:59 DEBUG ----- output ----- 18:07:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:07:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:07:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:07:59 DEBUG ----- output ----- 18:07:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:07:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:07:59 DEBUG { 18:07:59 DEBUG ----- output ----- 18:07:59 DEBUG ----- output ----- 18:07:59 DEBUG ----- output ----- 18:07:59 DEBUG { 18:07:59 DEBUG ----- output ----- 18:07:59 DEBUG ----- output ----- 18:07:59 DEBUG ----- output ----- 18:07:59 DEBUG { 18:07:59 DEBUG ----- output ----- 18:07:59 DEBUG ----- output ----- 18:07:59 DEBUG "status": "success", 18:07:59 DEBUG { 18:07:59 DEBUG { 18:07:59 DEBUG { 18:07:59 DEBUG "status": "success", 18:07:59 DEBUG "data": { 18:07:59 DEBUG "resultType": "vector", 18:07:59 DEBUG "result": [ 18:07:59 DEBUG "status": "success", 18:07:59 DEBUG { 18:07:59 DEBUG { 18:07:59 DEBUG "data": { 18:07:59 DEBUG "status": "success", 18:07:59 DEBUG "status": "success", 18:07:59 DEBUG "data": { 18:07:59 DEBUG "resultType": "vector", 18:07:59 DEBUG "result": [ 18:07:59 DEBUG { 18:07:59 DEBUG "metric": { 18:07:59 DEBUG "instance": "10.142.0.11:9100" 18:07:59 DEBUG "data": { 18:07:59 DEBUG "status": "success", 18:07:59 DEBUG "resultType": "vector", 18:07:59 DEBUG "data": { 18:07:59 DEBUG "resultType": "vector", 18:07:59 DEBUG { 18:07:59 DEBUG { 18:07:59 DEBUG { 18:07:59 DEBUG { 18:07:59 DEBUG }, 18:07:59 DEBUG "status": "success", 18:07:59 DEBUG "resultType": "vector", 18:07:59 DEBUG "data": { 18:07:59 DEBUG "result": [ 18:07:59 DEBUG "status": "success", 18:07:59 DEBUG "result": [ 18:07:59 DEBUG "status": "success", 18:07:59 DEBUG "status": "success", 18:07:59 DEBUG "status": "success", 18:07:59 DEBUG "metric": { 18:07:59 DEBUG "instance": "10.142.0.11:9100" 18:07:59 DEBUG }, 18:07:59 DEBUG "value": [ 18:07:59 DEBUG "result": [ 18:07:59 DEBUG { 18:07:59 DEBUG "data": { 18:07:59 DEBUG { 18:07:59 DEBUG "data": { 18:07:59 DEBUG "data": { 18:07:59 DEBUG "data": { 18:07:59 DEBUG "data": { 18:07:59 DEBUG "value": [ 18:07:59 DEBUG "resultType": "vector", 18:07:59 DEBUG 18:07:59 DEBUG 18:07:59 DEBUG "metric": { 18:07:59 DEBUG "resultType": "vector", 18:07:59 DEBUG "metric": { 18:07:59 DEBUG "resultType": "vector", 18:07:59 DEBUG "result": [ 18:07:59 DEBUG "resultType": "vector", 18:07:59 DEBUG "resultType": "vector", 18:07:59 DEBUG 18:07:59 DEBUG "result": [ 18:07:59 DEBUG { 18:07:59 DEBUG 18:07:59 DEBUG "instance": "10.142.0.11:9100" 18:07:59 DEBUG "result": [ 18:07:59 DEBUG "instance": "10.142.0.11:9100" 18:07:59 DEBUG }, 18:07:59 DEBUG "value": [ 18:07:59 DEBUG 18:07:59 DEBUG 18:07:59 DEBUG 18:07:59 DEBUG 18:07:59 DEBUG "metric": { 18:07:59 DEBUG [...] 18:07:59 DEBUG }, 18:07:59 DEBUG { 18:07:59 DEBUG "resultType": "vector", 18:07:59 DEBUG { 18:07:59 DEBUG "result": [ 18:07:59 DEBUG "result": [ 18:07:59 DEBUG { 18:07:59 DEBUG [...] 18:07:59 DEBUG 18:07:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:07:59 DEBUG 18:07:59 DEBUG "value": [ 18:07:59 DEBUG "metric": { 18:07:59 DEBUG "result": [ 18:07:59 DEBUG "metric": { 18:07:59 DEBUG { 18:07:59 DEBUG { 18:07:59 DEBUG "metric": { 18:07:59 DEBUG 18:07:59 DEBUG 18:07:59 DEBUG }, 18:07:59 DEBUG 18:07:59 DEBUG 18:07:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:07:59 DEBUG }, 18:07:59 DEBUG "value": [ 18:07:59 DEBUG 18:07:59 DEBUG 18:07:59 DEBUG 18:07:59 DEBUG [...] 18:07:59 DEBUG [...] 18:07:59 DEBUG "value": [ 18:07:59 DEBUG 18:07:59 DEBUG 18:07:59 DEBUG { 18:07:59 DEBUG "pod": "ds-cts-0" 18:07:59 DEBUG "metric": { 18:07:59 DEBUG "metric": { 18:07:59 DEBUG "pod": "ds-cts-0" 18:07:59 DEBUG 18:07:59 DEBUG 18:07:59 DEBUG 18:07:59 DEBUG 18:07:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:07:59 DEBUG }, 18:07:59 DEBUG "value": [ 18:07:59 DEBUG 1734458878, 18:07:59 DEBUG "2745556992" 18:07:59 DEBUG ] 18:07:59 DEBUG "instance": "10.142.0.11:9100" 18:07:59 DEBUG "instance": "10.142.0.51:9100" 18:07:59 DEBUG 18:07:59 DEBUG 18:07:59 DEBUG "instance": "10.142.0.51:9100" 18:07:59 DEBUG 18:07:59 DEBUG "metric": { 18:07:59 DEBUG }, 18:07:59 DEBUG "pod": "ds-cts-0" 18:07:59 DEBUG } 18:07:59 DEBUG }, 18:07:59 DEBUG }, 18:07:59 DEBUG "value": [ 18:07:59 DEBUG 18:07:59 DEBUG 18:07:59 DEBUG 18:07:59 DEBUG [...] 18:07:59 DEBUG 18:07:59 DEBUG 18:07:59 DEBUG }, 18:07:59 DEBUG "value": [ 18:07:59 DEBUG 18:07:59 DEBUG 18:07:59 DEBUG }, 18:07:59 DEBUG 18:07:59 DEBUG }, 18:07:59 DEBUG [...] 18:07:59 DEBUG "pod": "ds-cts-0" 18:07:59 DEBUG "value": [ 18:07:59 DEBUG 18:07:59 DEBUG ] 18:07:59 DEBUG "value": [ 18:07:59 DEBUG 18:07:59 DEBUG [...] 18:07:59 DEBUG 18:07:59 DEBUG "value": [ 18:07:59 DEBUG "value": [ 18:07:59 DEBUG 18:07:59 DEBUG }, 18:07:59 DEBUG 18:07:59 DEBUG "instance": "10.142.0.51:9100" 18:07:59 DEBUG } 18:07:59 DEBUG 18:07:59 DEBUG 18:07:59 DEBUG 18:07:59 DEBUG 18:07:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:07:59 DEBUG }, 18:07:59 DEBUG "value": [ 18:07:59 DEBUG 18:07:59 DEBUG 18:07:59 DEBUG }, 18:07:59 DEBUG } 18:07:59 DEBUG 18:07:59 DEBUG 18:07:59 DEBUG 18:07:59 DEBUG 1734458878, 18:07:59 DEBUG 1734458878, 18:07:59 DEBUG 1734458878, 18:07:59 DEBUG "value": [ 18:07:59 DEBUG 18:07:59 DEBUG 18:07:59 DEBUG [...] 18:07:59 DEBUG 18:07:59 DEBUG "instance": "10.142.0.51:9100" 18:07:59 DEBUG [...] 18:07:59 DEBUG "117841.2" 18:07:59 DEBUG "122889.70000000001" 18:07:59 DEBUG "0.010092526788894503" 18:07:59 DEBUG ] 18:07:59 DEBUG } 18:07:59 DEBUG ] 18:07:59 DEBUG } 18:07:59 DEBUG } 18:07:59 DEBUG }, 18:07:59 DEBUG 18:07:59 DEBUG ] 18:07:59 DEBUG ] 18:07:59 DEBUG 18:07:59 DEBUG "instance": "10.142.0.51:9100" 18:07:59 DEBUG "value": [ 18:07:59 DEBUG 18:07:59 DEBUG [...] 18:07:59 DEBUG "value": [ 18:07:59 DEBUG 18:07:59 DEBUG } 18:07:59 DEBUG } 18:07:59 DEBUG 18:07:59 DEBUG }, 18:07:59 DEBUG 1734458878, 18:07:59 DEBUG 18:07:59 DEBUG 18:07:59 DEBUG 1734458878, 18:07:59 DEBUG 18:07:59 DEBUG ] 18:07:59 DEBUG ] 18:07:59 DEBUG 18:07:59 DEBUG "value": [ 18:07:59 DEBUG "0.0003666666666669963" 18:07:59 DEBUG 18:07:59 DEBUG 18:07:59 DEBUG "0.0006666666666666524" 18:07:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:07:59 DEBUG }, 18:07:59 DEBUG "value": [ 18:07:59 DEBUG 1734458878, 18:07:59 DEBUG "1.0584707100908344" 18:07:59 DEBUG ] 18:07:59 DEBUG ] 18:07:59 DEBUG 18:07:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:07:59 DEBUG }, 18:07:59 DEBUG } 18:07:59 DEBUG [...] 18:07:59 DEBUG 1734458878, 18:07:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:07:59 DEBUG } 18:07:59 DEBUG ] 18:07:59 DEBUG } 18:07:59 DEBUG ] 18:07:59 DEBUG "value": [ 18:07:59 DEBUG } 18:07:59 DEBUG 18:07:59 DEBUG "24712.533333333333" 18:07:59 DEBUG }, 18:07:59 DEBUG } 18:07:59 DEBUG } 18:07:59 DEBUG } 18:07:59 DEBUG } 18:07:59 DEBUG ] 18:07:59 DEBUG 18:07:59 DEBUG ] 18:07:59 DEBUG "value": [ 18:07:59 DEBUG 1734458878, 18:07:59 DEBUG "1817.6355039443274" 18:07:59 DEBUG ] 18:07:59 DEBUG 1734458878, 18:07:59 DEBUG 18:07:59 DEBUG } 18:07:59 DEBUG ] 18:07:59 DEBUG } 18:07:59 DEBUG } 18:07:59 DEBUG } 18:07:59 DEBUG "0" 18:07:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:07:59 DEBUG ] 18:07:59 DEBUG } 18:07:59 DEBUG ] 18:07:59 DEBUG } 18:07:59 DEBUG ] 18:07:59 DEBUG }, 18:07:59 DEBUG } 18:07:59 DEBUG } 18:07:59 DEBUG } 18:07:59 DEBUG } 18:07:59 DEBUG "value": [ 18:07:59 DEBUG } 18:07:59 DEBUG } 18:07:59 DEBUG 1734458878, 18:07:59 DEBUG ] 18:07:59 DEBUG "700256256" 18:07:59 DEBUG } 18:07:59 DEBUG ] 18:07:59 DEBUG } 18:07:59 DEBUG } 18:07:59 DEBUG ] 18:07:59 DEBUG } 18:07:59 DEBUG } 18:07:59 INFO 18:07:59 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:07:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:07:59 INFO 18:07:59 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:07:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:07:59 INFO [loop_until]: OK (rc = 0) 18:07:59 DEBUG --- stdout --- 18:07:59 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 8m 455Mi ds-idrepo-0 7m 659Mi rcs-5d696ff5b-7ttvr 6m 294Mi rcs-5d696ff5b-wwlbr 9m 333Mi 18:07:59 DEBUG --- stderr --- 18:07:59 DEBUG 18:07:59 INFO [loop_until]: OK (rc = 0) 18:07:59 DEBUG --- stdout --- 18:07:59 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 104m 0% 2246Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1147m 3% 3668Mi 12% 18:07:59 DEBUG --- stderr --- 18:07:59 DEBUG 18:08:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458938" 18:08:58 INFO [http_cmd]: http status code OK 18:08:58 DEBUG --- status code --- 18:08:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:08:58 DEBUG --- http response --- 18:08:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:08:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734458938" 18:08:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458938" 18:08:58 INFO [http_cmd]: http status code OK 18:08:58 DEBUG --- status code --- 18:08:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:08:58 DEBUG --- http response --- 18:08:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:08:58 INFO [http_cmd]: http status code OK 18:08:58 DEBUG --- status code --- 18:08:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:08:58 DEBUG --- http response --- 18:08:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:08:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458938" 18:08:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458938" 18:08:58 INFO [http_cmd]: http status code OK 18:08:58 DEBUG --- status code --- 18:08:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:08:58 DEBUG --- http response --- 18:08:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:08:58 INFO [http_cmd]: http status code OK 18:08:58 DEBUG --- status code --- 18:08:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:08:58 DEBUG --- http response --- 18:08:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:08:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458938" 18:08:58 INFO [http_cmd]: http status code OK 18:08:58 DEBUG --- status code --- 18:08:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:08:58 DEBUG --- http response --- 18:08:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458939" 18:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458939" 18:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458939" 18:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458939" 18:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458939" 18:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458939" 18:08:59 INFO [http_cmd]: http status code OK 18:08:59 DEBUG --- status code --- 18:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:08:59 DEBUG --- http response --- 18:08:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734458939" 18:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458939" 18:08:59 INFO [http_cmd]: http status code OK 18:08:59 DEBUG --- status code --- 18:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:08:59 DEBUG --- http response --- 18:08:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:08:59 INFO [http_cmd]: http status code OK 18:08:59 INFO [http_cmd]: http status code OK 18:08:59 DEBUG --- status code --- 18:08:59 DEBUG --- status code --- 18:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734458939" 18:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:08:59 INFO [http_cmd]: http status code OK 18:08:59 INFO [http_cmd]: http status code OK 18:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:08:59 DEBUG --- http response --- 18:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458939" 18:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458939" 18:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458939" 18:08:59 DEBUG --- status code --- 18:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458939" 18:08:59 DEBUG --- status code --- 18:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458939" 18:08:59 DEBUG --- http response --- 18:08:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458939" 18:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458939" 18:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458939" 18:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734458939" 18:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458939" 18:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458939" 18:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734458939" 18:08:59 INFO [http_cmd]: http status code OK 18:08:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:08:59 INFO [http_cmd]: http status code OK 18:08:59 DEBUG --- http response --- 18:08:59 INFO [http_cmd]: http status code OK 18:08:59 INFO [http_cmd]: http status code OK 18:08:59 INFO [http_cmd]: http status code OK 18:08:59 INFO [http_cmd]: http status code OK 18:08:59 DEBUG --- http response --- 18:08:59 DEBUG --- status code --- 18:08:59 DEBUG --- status code --- 18:08:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:08:59 DEBUG --- status code --- 18:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734458939" 18:08:59 DEBUG --- status code --- 18:08:59 DEBUG --- status code --- 18:08:59 DEBUG --- status code --- 18:08:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458939" 18:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:08:59 DEBUG --- http response --- 18:08:59 DEBUG --- http response --- 18:08:59 DEBUG --- http response --- 18:08:59 DEBUG --- http response --- 18:08:59 DEBUG --- http response --- 18:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:08:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:08:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:08:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:08:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:08:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:08:59 DEBUG --- http response --- 18:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458939" 18:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734458939" 18:08:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458939" 18:08:59 INFO [http_cmd]: http status code OK 18:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458939" 18:08:59 INFO [http_cmd]: http status code OK 18:08:59 INFO [http_cmd]: http status code OK 18:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458939" 18:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458939" 18:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458939" 18:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458939" 18:08:59 INFO [http_cmd]: http status code OK 18:08:59 INFO [http_cmd]: http status code OK 18:08:59 INFO [http_cmd]: http status code OK 18:08:59 INFO [http_cmd]: http status code OK 18:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458939" 18:08:59 INFO [http_cmd]: http status code OK 18:08:59 INFO [http_cmd]: http status code OK 18:08:59 DEBUG --- status code --- 18:08:59 ERROR [http_cmd]: ERROR 18:08:59 DEBUG --- status code --- 18:08:59 DEBUG --- status code --- 18:08:59 INFO [http_cmd]: http status code OK 18:08:59 INFO [http_cmd]: http status code OK 18:08:59 INFO [http_cmd]: http status code OK 18:08:59 DEBUG --- status code --- 18:08:59 DEBUG --- status code --- 18:08:59 DEBUG --- status code --- 18:08:59 DEBUG --- status code --- 18:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:08:59 DEBUG --- http response --- 18:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:08:59 ERROR --- status code --- 18:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:08:59 INFO [http_cmd]: http status code OK 18:08:59 INFO [http_cmd]: http status code OK 18:08:59 INFO [http_cmd]: http status code OK 18:08:59 DEBUG --- status code --- 18:08:59 DEBUG --- status code --- 18:08:59 DEBUG --- status code --- 18:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:08:59 DEBUG --- status code --- 18:08:59 DEBUG --- status code --- 18:08:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:08:59 DEBUG --- http response --- 18:08:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:08:59 DEBUG --- http response --- 18:08:59 DEBUG --- http response --- 18:08:59 DEBUG --- status code --- 18:08:59 DEBUG --- status code --- 18:08:59 DEBUG --- status code --- 18:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:08:59 INFO [http_cmd]: http status code OK 18:08:59 INFO [http_cmd]: http status code OK 18:08:59 INFO [http_cmd]: http status code OK 18:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:08:59 DEBUG --- http response --- 18:08:59 DEBUG --- http response --- 18:08:59 DEBUG --- http response --- 18:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:08:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:08:59 ERROR --- http response --- 18:08:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:08:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:08:59 DEBUG --- http response --- 18:08:59 DEBUG --- http response --- 18:08:59 DEBUG --- status code --- 18:08:59 DEBUG --- status code --- 18:08:59 DEBUG --- status code --- 18:08:59 DEBUG --- http response --- 18:08:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:08:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:08:59 INFO [http_cmd]: http status code OK 18:08:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:08:59 DEBUG --- http response --- 18:08:59 DEBUG --- http response --- 18:08:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:08:59 DEBUG --- http response --- 18:08:59 DEBUG --- http response --- 18:08:59 DEBUG --- http response --- 18:08:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:08:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:08:59 DEBUG --- http response --- 18:08:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:08:59 DEBUG --- status code --- 18:08:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:08:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:08:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:08:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:08:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:08:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:08:59 DEBUG ----- output ----- 18:08:59 DEBUG ----- output ----- 18:08:59 DEBUG --- http response --- 18:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:08:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:08:59 DEBUG ----- output ----- 18:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:08:59 DEBUG ----- output ----- 18:08:59 DEBUG ----- output ----- 18:08:59 DEBUG ----- output ----- 18:08:59 DEBUG ----- output ----- 18:08:59 DEBUG { 18:08:59 DEBUG { 18:08:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:08:59 DEBUG --- http response --- 18:08:59 DEBUG ----- output ----- 18:08:59 DEBUG { 18:08:59 DEBUG --- http response --- 18:08:59 DEBUG { 18:08:59 DEBUG { 18:08:59 DEBUG { 18:08:59 DEBUG { 18:08:59 DEBUG "status": "success", 18:08:59 DEBUG "status": "success", 18:08:59 DEBUG ----- output ----- 18:08:59 DEBUG { 18:08:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:08:59 DEBUG "status": "success", 18:08:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:08:59 DEBUG "status": "success", 18:08:59 DEBUG "status": "success", 18:08:59 DEBUG "status": "success", 18:08:59 DEBUG "status": "success", 18:08:59 DEBUG "data": { 18:08:59 DEBUG "data": { 18:08:59 DEBUG { 18:08:59 DEBUG "status": "success", 18:08:59 DEBUG ----- output ----- 18:08:59 DEBUG "data": { 18:08:59 DEBUG ----- output ----- 18:08:59 DEBUG "data": { 18:08:59 DEBUG "data": { 18:08:59 DEBUG "data": { 18:08:59 DEBUG "data": { 18:08:59 DEBUG "resultType": "vector", 18:08:59 DEBUG "resultType": "vector", 18:08:59 DEBUG "status": "success", 18:08:59 DEBUG "data": { 18:08:59 DEBUG { 18:08:59 DEBUG "resultType": "vector", 18:08:59 DEBUG { 18:08:59 DEBUG "resultType": "vector", 18:08:59 DEBUG "resultType": "vector", 18:08:59 DEBUG "resultType": "vector", 18:08:59 DEBUG "resultType": "vector", 18:08:59 DEBUG "result": [ 18:08:59 DEBUG "result": [ 18:08:59 DEBUG "data": { 18:08:59 DEBUG "resultType": "vector", 18:08:59 DEBUG "status": "success", 18:08:59 DEBUG "result": [ 18:08:59 DEBUG "status": "success", 18:08:59 DEBUG "result": [ 18:08:59 DEBUG "result": [ 18:08:59 DEBUG "result": [ 18:08:59 DEBUG "result": [ 18:08:59 DEBUG { 18:08:59 DEBUG { 18:08:59 DEBUG "resultType": "vector", 18:08:59 DEBUG "result": [ 18:08:59 DEBUG "data": { 18:08:59 DEBUG { 18:08:59 DEBUG "data": { 18:08:59 DEBUG { 18:08:59 DEBUG { 18:08:59 DEBUG { 18:08:59 DEBUG { 18:08:59 DEBUG "metric": { 18:08:59 DEBUG "metric": { 18:08:59 DEBUG "result": [ 18:08:59 DEBUG { 18:08:59 DEBUG "resultType": "vector", 18:08:59 DEBUG "metric": { 18:08:59 DEBUG "resultType": "vector", 18:08:59 DEBUG "metric": { 18:08:59 DEBUG "metric": { 18:08:59 DEBUG "metric": { 18:08:59 DEBUG "metric": { 18:08:59 DEBUG "instance": "10.142.0.11:9100" 18:08:59 DEBUG "instance": "10.142.0.11:9100" 18:08:59 DEBUG { 18:08:59 DEBUG "metric": { 18:08:59 DEBUG "result": [ 18:08:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:08:59 DEBUG "result": [ 18:08:59 DEBUG "pod": "ds-cts-0" 18:08:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:08:59 DEBUG "instance": "10.142.0.11:9100" 18:08:59 DEBUG "instance": "10.142.0.11:9100" 18:08:59 DEBUG }, 18:08:59 DEBUG }, 18:08:59 DEBUG "metric": { 18:08:59 DEBUG "pod": "ds-cts-0" 18:08:59 DEBUG { 18:08:59 DEBUG }, 18:08:59 DEBUG { 18:08:59 DEBUG }, 18:08:59 DEBUG }, 18:08:59 DEBUG }, 18:08:59 DEBUG }, 18:08:59 DEBUG "value": [ 18:08:59 DEBUG "value": [ 18:08:59 DEBUG "instance": "10.142.0.11:9100" 18:08:59 DEBUG }, 18:08:59 DEBUG "metric": { 18:08:59 DEBUG "value": [ 18:08:59 DEBUG "metric": { 18:08:59 DEBUG "value": [ 18:08:59 DEBUG "value": [ 18:08:59 DEBUG "value": [ 18:08:59 DEBUG "value": [ 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG }, 18:08:59 DEBUG "value": [ 18:08:59 DEBUG "pod": "ds-cts-0" 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG "value": [ 18:08:59 DEBUG 18:08:59 DEBUG }, 18:08:59 DEBUG "pod": "ds-cts-0" 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG "value": [ 18:08:59 DEBUG }, 18:08:59 DEBUG [...] 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG [...] 18:08:59 DEBUG [...] 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG "value": [ 18:08:59 DEBUG 18:08:59 DEBUG [...] 18:08:59 DEBUG [...] 18:08:59 DEBUG [...] 18:08:59 DEBUG [...] 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG [...] 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG [...] 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG [...] 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG "instance": "10.142.0.51:9100" 18:08:59 DEBUG "instance": "10.142.0.51:9100" 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG 18:08:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:08:59 DEBUG 18:08:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:08:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:08:59 DEBUG "instance": "10.142.0.51:9100" 18:08:59 DEBUG "instance": "10.142.0.51:9100" 18:08:59 DEBUG }, 18:08:59 DEBUG }, 18:08:59 DEBUG 18:08:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:08:59 DEBUG [...] 18:08:59 DEBUG }, 18:08:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:08:59 DEBUG }, 18:08:59 DEBUG "value": [ 18:08:59 DEBUG 1734458939, 18:08:59 DEBUG "0.006332037422876158" 18:08:59 DEBUG ] 18:08:59 DEBUG "value": [ 18:08:59 DEBUG "instance": "10.142.0.51:9100" 18:08:59 DEBUG }, 18:08:59 DEBUG 18:08:59 DEBUG "value": [ 18:08:59 DEBUG }, 18:08:59 DEBUG }, 18:08:59 DEBUG }, 18:08:59 DEBUG }, 18:08:59 DEBUG "value": [ 18:08:59 DEBUG } 18:08:59 DEBUG 1734458939, 18:08:59 DEBUG }, 18:08:59 DEBUG "value": [ 18:08:59 DEBUG 18:08:59 DEBUG 1734458939, 18:08:59 DEBUG "value": [ 18:08:59 DEBUG "value": [ 18:08:59 DEBUG "value": [ 18:08:59 DEBUG "value": [ 18:08:59 DEBUG 1734458939, 18:08:59 DEBUG ] 18:08:59 DEBUG "132869.76666666663" 18:08:59 DEBUG "value": [ 18:08:59 DEBUG 1734458939, 18:08:59 DEBUG 18:08:59 DEBUG "1.057894526858251" 18:08:59 DEBUG 1734458939, 18:08:59 DEBUG 1734458939, 18:08:59 DEBUG 1734458939, 18:08:59 DEBUG 1734458939, 18:08:59 DEBUG "128617.76666666665" 18:08:59 DEBUG } 18:08:59 DEBUG ] 18:08:59 DEBUG 1734458939, 18:08:59 DEBUG "1109.4620183598504" 18:08:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:08:59 DEBUG ] 18:08:59 DEBUG "700649472" 18:08:59 DEBUG "2757500928" 18:08:59 DEBUG "0.000400000000003331" 18:08:59 DEBUG "0.000333333333333267" 18:08:59 DEBUG ] 18:08:59 DEBUG } 18:08:59 DEBUG } 18:08:59 DEBUG "26350.933333333334" 18:08:59 DEBUG ] 18:08:59 DEBUG }, 18:08:59 DEBUG } 18:08:59 DEBUG ] 18:08:59 DEBUG ] 18:08:59 DEBUG ] 18:08:59 DEBUG ] 18:08:59 DEBUG } 18:08:59 DEBUG ] 18:08:59 DEBUG ] 18:08:59 DEBUG } 18:08:59 DEBUG "value": [ 18:08:59 DEBUG ] 18:08:59 DEBUG } 18:08:59 DEBUG } 18:08:59 DEBUG } 18:08:59 DEBUG } 18:08:59 DEBUG ] 18:08:59 DEBUG } 18:08:59 DEBUG } 18:08:59 DEBUG ] 18:08:59 DEBUG 1734458939, 18:08:59 DEBUG } 18:08:59 DEBUG ] 18:08:59 DEBUG ] 18:08:59 DEBUG ] 18:08:59 DEBUG ] 18:08:59 DEBUG } 18:08:59 DEBUG } 18:08:59 DEBUG ] 18:08:59 DEBUG } 18:08:59 DEBUG "0" 18:08:59 DEBUG } 18:08:59 DEBUG } 18:08:59 DEBUG } 18:08:59 DEBUG } 18:08:59 DEBUG } 18:08:59 DEBUG } 18:08:59 DEBUG } 18:08:59 DEBUG } 18:08:59 DEBUG ] 18:08:59 DEBUG } 18:08:59 DEBUG } 18:08:59 DEBUG } 18:08:59 DEBUG } 18:08:59 DEBUG } 18:08:59 DEBUG } 18:08:59 DEBUG ] 18:08:59 DEBUG } 18:08:59 DEBUG } 18:08:59 INFO 18:08:59 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:08:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:08:59 INFO 18:08:59 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:08:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:08:59 INFO [loop_until]: OK (rc = 0) 18:08:59 DEBUG --- stdout --- 18:08:59 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 8m 460Mi ds-idrepo-0 7m 666Mi rcs-5d696ff5b-7ttvr 6m 294Mi rcs-5d696ff5b-wwlbr 2m 333Mi 18:08:59 DEBUG --- stderr --- 18:08:59 DEBUG 18:08:59 INFO [loop_until]: OK (rc = 0) 18:08:59 DEBUG --- stdout --- 18:08:59 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 105m 0% 2261Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1138m 3% 3681Mi 12% 18:08:59 DEBUG --- stderr --- 18:08:59 DEBUG 18:09:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458998" 18:09:58 INFO [http_cmd]: http status code OK 18:09:58 DEBUG --- status code --- 18:09:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:09:58 DEBUG --- http response --- 18:09:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:09:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734458998" 18:09:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458998" 18:09:58 INFO [http_cmd]: http status code OK 18:09:58 DEBUG --- status code --- 18:09:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:09:58 DEBUG --- http response --- 18:09:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:09:58 INFO [http_cmd]: http status code OK 18:09:58 DEBUG --- status code --- 18:09:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:09:58 DEBUG --- http response --- 18:09:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:09:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458998" 18:09:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458998" 18:09:58 INFO [http_cmd]: http status code OK 18:09:58 DEBUG --- status code --- 18:09:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:09:58 DEBUG --- http response --- 18:09:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:09:58 INFO [http_cmd]: http status code OK 18:09:58 DEBUG --- status code --- 18:09:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:09:58 DEBUG --- http response --- 18:09:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:09:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458998" 18:09:58 INFO [http_cmd]: http status code OK 18:09:58 DEBUG --- status code --- 18:09:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:09:58 DEBUG --- http response --- 18:09:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458999" 18:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458999" 18:09:59 INFO [http_cmd]: http status code OK 18:09:59 INFO [http_cmd]: http status code OK 18:09:59 DEBUG --- status code --- 18:09:59 DEBUG --- status code --- 18:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:09:59 DEBUG --- http response --- 18:09:59 DEBUG --- http response --- 18:09:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:09:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458999" 18:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458999" 18:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458999" 18:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458999" 18:09:59 INFO [http_cmd]: http status code OK 18:09:59 DEBUG --- status code --- 18:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:09:59 DEBUG --- http response --- 18:09:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734458999" 18:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458999" 18:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458999" 18:09:59 INFO [http_cmd]: http status code OK 18:09:59 DEBUG --- status code --- 18:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:09:59 DEBUG --- http response --- 18:09:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458999" 18:09:59 INFO [http_cmd]: http status code OK 18:09:59 INFO [http_cmd]: http status code OK 18:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734458999" 18:09:59 DEBUG --- status code --- 18:09:59 DEBUG --- status code --- 18:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:09:59 DEBUG --- http response --- 18:09:59 DEBUG --- http response --- 18:09:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:09:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:09:59 INFO [http_cmd]: http status code OK 18:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458999" 18:09:59 DEBUG --- status code --- 18:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:09:59 DEBUG --- http response --- 18:09:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:09:59 INFO [http_cmd]: http status code OK 18:09:59 DEBUG --- status code --- 18:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:09:59 DEBUG --- http response --- 18:09:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:09:59 INFO [http_cmd]: http status code OK 18:09:59 DEBUG --- status code --- 18:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:09:59 DEBUG --- http response --- 18:09:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:09:59 INFO [http_cmd]: http status code OK 18:09:59 DEBUG --- status code --- 18:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:09:59 DEBUG --- http response --- 18:09:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:09:59 INFO [http_cmd]: http status code OK 18:09:59 DEBUG --- status code --- 18:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:09:59 DEBUG --- http response --- 18:09:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:09:59 INFO [http_cmd]: http status code OK 18:09:59 DEBUG --- status code --- 18:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:09:59 DEBUG --- http response --- 18:09:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458999" 18:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458999" 18:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458999" 18:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458999" 18:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458999" 18:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458999" 18:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458999" 18:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734458999" 18:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734458999" 18:09:59 INFO [http_cmd]: http status code OK 18:09:59 DEBUG --- status code --- 18:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:09:59 DEBUG --- http response --- 18:09:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:09:59 INFO [http_cmd]: http status code OK 18:09:59 DEBUG --- status code --- 18:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:09:59 DEBUG --- http response --- 18:09:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:09:59 INFO [http_cmd]: http status code OK 18:09:59 DEBUG --- status code --- 18:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:09:59 DEBUG --- http response --- 18:09:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:09:59 ERROR [http_cmd]: ERROR 18:09:59 ERROR --- status code --- 18:09:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:09:59 ERROR --- http response --- 18:09:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:09:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:09:59 INFO [http_cmd]: http status code OK 18:09:59 DEBUG --- status code --- 18:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:09:59 DEBUG --- http response --- 18:09:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:09:59 INFO [http_cmd]: http status code OK 18:09:59 DEBUG --- status code --- 18:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:09:59 DEBUG --- http response --- 18:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458999" 18:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458999" 18:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734458999" 18:09:59 INFO [http_cmd]: http status code OK 18:09:59 DEBUG --- status code --- 18:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:09:59 DEBUG --- http response --- 18:09:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734458999" 18:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458999" 18:09:59 INFO [http_cmd]: http status code OK 18:09:59 DEBUG --- status code --- 18:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:09:59 DEBUG --- http response --- 18:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458999" 18:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734458999" 18:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458999" 18:09:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458999" 18:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734458999" 18:09:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:09:59 INFO [http_cmd]: http status code OK 18:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734458999" 18:09:59 INFO [http_cmd]: http status code OK 18:09:59 INFO [http_cmd]: http status code OK 18:09:59 INFO [http_cmd]: http status code OK 18:09:59 INFO [http_cmd]: http status code OK 18:09:59 INFO [http_cmd]: http status code OK 18:09:59 INFO [http_cmd]: http status code OK 18:09:59 DEBUG --- status code --- 18:09:59 DEBUG --- status code --- 18:09:59 INFO [http_cmd]: http status code OK 18:09:59 DEBUG --- status code --- 18:09:59 DEBUG --- status code --- 18:09:59 DEBUG --- status code --- 18:09:59 INFO [http_cmd]: http status code OK 18:09:59 DEBUG --- status code --- 18:09:59 DEBUG --- status code --- 18:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:09:59 DEBUG --- status code --- 18:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:09:59 DEBUG --- status code --- 18:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:09:59 DEBUG --- http response --- 18:09:59 DEBUG --- http response --- 18:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:09:59 DEBUG --- http response --- 18:09:59 DEBUG --- http response --- 18:09:59 DEBUG --- http response --- 18:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:09:59 DEBUG --- http response --- 18:09:59 DEBUG --- http response --- 18:09:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:09:59 DEBUG --- http response --- 18:09:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:09:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:09:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:09:59 INFO [http_cmd]: http status code OK 18:09:59 INFO [http_cmd]: http status code OK 18:09:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:09:59 DEBUG --- http response --- 18:09:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:09:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:09:59 DEBUG ----- output ----- 18:09:59 DEBUG ----- output ----- 18:09:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:09:59 DEBUG ----- output ----- 18:09:59 INFO [http_cmd]: http status code OK 18:09:59 DEBUG --- status code --- 18:09:59 DEBUG --- status code --- 18:09:59 DEBUG ----- output ----- 18:09:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:09:59 DEBUG ----- output ----- 18:09:59 DEBUG ----- output ----- 18:09:59 DEBUG { 18:09:59 DEBUG { 18:09:59 DEBUG ----- output ----- 18:09:59 DEBUG { 18:09:59 DEBUG --- status code --- 18:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:09:59 DEBUG { 18:09:59 DEBUG ----- output ----- 18:09:59 DEBUG { 18:09:59 DEBUG { 18:09:59 DEBUG "status": "success", 18:09:59 DEBUG "status": "success", 18:09:59 DEBUG { 18:09:59 DEBUG "status": "success", 18:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:09:59 DEBUG --- http response --- 18:09:59 DEBUG --- http response --- 18:09:59 DEBUG "status": "success", 18:09:59 DEBUG { 18:09:59 DEBUG "status": "success", 18:09:59 DEBUG "status": "success", 18:09:59 DEBUG "data": { 18:09:59 DEBUG "data": { 18:09:59 DEBUG "status": "success", 18:09:59 DEBUG "data": { 18:09:59 DEBUG --- http response --- 18:09:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:09:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:09:59 DEBUG "data": { 18:09:59 DEBUG "status": "success", 18:09:59 DEBUG "data": { 18:09:59 DEBUG "data": { 18:09:59 DEBUG "resultType": "vector", 18:09:59 DEBUG "resultType": "vector", 18:09:59 DEBUG "data": { 18:09:59 DEBUG "resultType": "vector", 18:09:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:09:59 DEBUG ----- output ----- 18:09:59 DEBUG ----- output ----- 18:09:59 DEBUG "resultType": "vector", 18:09:59 DEBUG "data": { 18:09:59 DEBUG "resultType": "vector", 18:09:59 DEBUG "resultType": "vector", 18:09:59 DEBUG "result": [ 18:09:59 DEBUG "result": [ 18:09:59 DEBUG "resultType": "vector", 18:09:59 DEBUG "result": [ 18:09:59 DEBUG ----- output ----- 18:09:59 DEBUG { 18:09:59 DEBUG { 18:09:59 DEBUG "result": [ 18:09:59 DEBUG "resultType": "vector", 18:09:59 DEBUG "result": [ 18:09:59 DEBUG "result": [ 18:09:59 DEBUG { 18:09:59 DEBUG { 18:09:59 DEBUG "result": [ 18:09:59 DEBUG { 18:09:59 DEBUG { 18:09:59 DEBUG "status": "success", 18:09:59 DEBUG "status": "success", 18:09:59 DEBUG { 18:09:59 DEBUG "result": [ 18:09:59 DEBUG { 18:09:59 DEBUG { 18:09:59 DEBUG "metric": { 18:09:59 DEBUG "metric": { 18:09:59 DEBUG { 18:09:59 DEBUG "metric": { 18:09:59 DEBUG "status": "success", 18:09:59 DEBUG "data": { 18:09:59 DEBUG "data": { 18:09:59 DEBUG "metric": { 18:09:59 DEBUG { 18:09:59 DEBUG "metric": { 18:09:59 DEBUG "metric": { 18:09:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:09:59 DEBUG "instance": "10.142.0.11:9100" 18:09:59 DEBUG "metric": { 18:09:59 DEBUG "pod": "ds-cts-0" 18:09:59 DEBUG "data": { 18:09:59 DEBUG "resultType": "vector", 18:09:59 DEBUG "resultType": "vector", 18:09:59 DEBUG "pod": "ds-cts-0" 18:09:59 DEBUG "metric": { 18:09:59 DEBUG "pod": "ds-cts-0" 18:09:59 DEBUG "instance": "10.142.0.11:9100" 18:09:59 DEBUG }, 18:09:59 DEBUG }, 18:09:59 DEBUG "pod": "ds-cts-0" 18:09:59 DEBUG }, 18:09:59 DEBUG "resultType": "vector", 18:09:59 DEBUG "result": [ 18:09:59 DEBUG "result": [ 18:09:59 DEBUG }, 18:09:59 DEBUG "instance": "10.142.0.11:9100" 18:09:59 DEBUG }, 18:09:59 DEBUG }, 18:09:59 DEBUG "value": [ 18:09:59 DEBUG "value": [ 18:09:59 DEBUG }, 18:09:59 DEBUG "value": [ 18:09:59 DEBUG "result": [ 18:09:59 DEBUG { 18:09:59 DEBUG { 18:09:59 DEBUG "metric": { 18:09:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:09:59 DEBUG }, 18:09:59 DEBUG "value": [ 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG "value": [ 18:09:59 DEBUG 18:09:59 DEBUG { 18:09:59 DEBUG "metric": { 18:09:59 DEBUG "instance": "10.142.0.11:9100" 18:09:59 DEBUG }, 18:09:59 DEBUG "value": [ 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG [...] 18:09:59 DEBUG 18:09:59 DEBUG "metric": { 18:09:59 DEBUG "instance": "10.142.0.11:9100" 18:09:59 DEBUG }, 18:09:59 DEBUG "value": [ 18:09:59 DEBUG "value": [ 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG "value": [ 18:09:59 DEBUG }, 18:09:59 DEBUG "value": [ 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG [...] 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG "instance": "10.142.0.51:9100" 18:09:59 DEBUG }, 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG [...] 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG "value": [ 18:09:59 DEBUG "value": [ 18:09:59 DEBUG 1734458999, 18:09:59 DEBUG "0.0004999999999981241" 18:09:59 DEBUG ] 18:09:59 DEBUG } 18:09:59 DEBUG ] 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG [...] 18:09:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG [...] 18:09:59 DEBUG [...] 18:09:59 DEBUG [...] 18:09:59 DEBUG 18:09:59 DEBUG }, 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG [...] 18:09:59 DEBUG 18:09:59 DEBUG } 18:09:59 DEBUG } 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG "value": [ 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG "instance": "10.142.0.51:9100" 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG 1734458999, 18:09:59 DEBUG 18:09:59 DEBUG [...] 18:09:59 DEBUG [...] 18:09:59 DEBUG 18:09:59 DEBUG }, 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:09:59 DEBUG "2774925312" 18:09:59 DEBUG [...] 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG "instance": "10.142.0.51:9100" 18:09:59 DEBUG "value": [ 18:09:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:09:59 DEBUG "instance": "10.142.0.51:9100" 18:09:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:09:59 DEBUG }, 18:09:59 DEBUG ] 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG }, 18:09:59 DEBUG 1734458999, 18:09:59 DEBUG }, 18:09:59 DEBUG }, 18:09:59 DEBUG }, 18:09:59 DEBUG "value": [ 18:09:59 DEBUG } 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG 18:09:59 DEBUG "value": [ 18:09:59 DEBUG "123025.53333333334" 18:09:59 DEBUG "value": [ 18:09:59 DEBUG "value": [ 18:09:59 DEBUG "value": [ 18:09:59 DEBUG 1734458999, 18:09:59 DEBUG "700829696" 18:09:59 DEBUG ] 18:09:59 DEBUG } 18:09:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:09:59 DEBUG 1734458999, 18:09:59 DEBUG ] 18:09:59 DEBUG 1734458999, 18:09:59 DEBUG 1734458999, 18:09:59 DEBUG 1734458999, 18:09:59 DEBUG "0" 18:09:59 DEBUG ] 18:09:59 DEBUG } 18:09:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:09:59 DEBUG }, 18:09:59 DEBUG "0" 18:09:59 DEBUG } 18:09:59 DEBUG "1.0575324527055534" 18:09:59 DEBUG "126690.73333333334" 18:09:59 DEBUG ] 18:09:59 DEBUG 18:09:59 DEBUG ] 18:09:59 DEBUG ] 18:09:59 DEBUG }, 18:09:59 DEBUG "value": [ 18:09:59 DEBUG ] 18:09:59 DEBUG ] 18:09:59 DEBUG ] 18:09:59 DEBUG ] 18:09:59 DEBUG } 18:09:59 DEBUG ] 18:09:59 DEBUG "instance": "10.142.0.51:9100" 18:09:59 DEBUG } 18:09:59 DEBUG "value": [ 18:09:59 DEBUG 1734458999, 18:09:59 DEBUG } 18:09:59 DEBUG } 18:09:59 DEBUG } 18:09:59 DEBUG } 18:09:59 DEBUG } 18:09:59 DEBUG } 18:09:59 DEBUG }, 18:09:59 DEBUG } 18:09:59 DEBUG 1734458999, 18:09:59 DEBUG "0.004947525830177344" 18:09:59 DEBUG ] 18:09:59 DEBUG } 18:09:59 DEBUG ] 18:09:59 DEBUG } 18:09:59 DEBUG } 18:09:59 DEBUG } 18:09:59 DEBUG "value": [ 18:09:59 DEBUG "1497.8528626125567" 18:09:59 DEBUG ] 18:09:59 DEBUG } 18:09:59 DEBUG } 18:09:59 DEBUG 1734458999, 18:09:59 DEBUG ] 18:09:59 DEBUG } 18:09:59 DEBUG } 18:09:59 DEBUG } 18:09:59 DEBUG "24849.066666666666" 18:09:59 DEBUG } 18:09:59 DEBUG ] 18:09:59 DEBUG ] 18:09:59 DEBUG ] 18:09:59 DEBUG } 18:09:59 DEBUG } 18:09:59 DEBUG } 18:09:59 DEBUG } 18:09:59 DEBUG ] 18:09:59 DEBUG } 18:09:59 DEBUG } 18:09:59 DEBUG } 18:09:59 INFO 18:09:59 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:09:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:09:59 INFO 18:09:59 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:09:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:09:59 INFO [loop_until]: OK (rc = 0) 18:09:59 DEBUG --- stdout --- 18:09:59 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 7m 462Mi ds-idrepo-0 7m 671Mi rcs-5d696ff5b-7ttvr 4m 295Mi rcs-5d696ff5b-wwlbr 3m 333Mi 18:09:59 DEBUG --- stderr --- 18:09:59 DEBUG 18:09:59 INFO [loop_until]: OK (rc = 0) 18:09:59 DEBUG --- stdout --- 18:09:59 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 106m 0% 2275Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1130m 3% 3699Mi 13% 18:09:59 DEBUG --- stderr --- 18:09:59 DEBUG 18:10:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459058" 18:10:58 INFO [http_cmd]: http status code OK 18:10:58 DEBUG --- status code --- 18:10:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:10:58 DEBUG --- http response --- 18:10:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:10:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734459058" 18:10:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459058" 18:10:58 INFO [http_cmd]: http status code OK 18:10:58 DEBUG --- status code --- 18:10:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:10:58 DEBUG --- http response --- 18:10:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:10:58 INFO [http_cmd]: http status code OK 18:10:58 DEBUG --- status code --- 18:10:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:10:58 DEBUG --- http response --- 18:10:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:10:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459058" 18:10:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459058" 18:10:58 INFO [http_cmd]: http status code OK 18:10:58 INFO [http_cmd]: http status code OK 18:10:58 DEBUG --- status code --- 18:10:58 DEBUG --- status code --- 18:10:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:10:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:10:58 DEBUG --- http response --- 18:10:58 DEBUG --- http response --- 18:10:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:10:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:10:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459058" 18:10:58 INFO [http_cmd]: http status code OK 18:10:58 DEBUG --- status code --- 18:10:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:10:58 DEBUG --- http response --- 18:10:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459059" 18:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459059" 18:10:59 INFO [http_cmd]: http status code OK 18:10:59 DEBUG --- status code --- 18:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:10:59 DEBUG --- http response --- 18:10:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459059" 18:10:59 INFO [http_cmd]: http status code OK 18:10:59 DEBUG --- status code --- 18:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:10:59 DEBUG --- http response --- 18:10:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:10:59 INFO [http_cmd]: http status code OK 18:10:59 DEBUG --- status code --- 18:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:10:59 DEBUG --- http response --- 18:10:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459059" 18:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459059" 18:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459059" 18:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734459059" 18:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459059" 18:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459059" 18:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459059" 18:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734459059" 18:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459059" 18:10:59 INFO [http_cmd]: http status code OK 18:10:59 DEBUG --- status code --- 18:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:10:59 DEBUG --- http response --- 18:10:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:10:59 INFO [http_cmd]: http status code OK 18:10:59 INFO [http_cmd]: http status code OK 18:10:59 DEBUG --- status code --- 18:10:59 INFO [http_cmd]: http status code OK 18:10:59 DEBUG --- status code --- 18:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:10:59 DEBUG --- status code --- 18:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:10:59 DEBUG --- http response --- 18:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:10:59 DEBUG --- http response --- 18:10:59 DEBUG --- http response --- 18:10:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:10:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:10:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:10:59 INFO [http_cmd]: http status code OK 18:10:59 DEBUG --- status code --- 18:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:10:59 DEBUG --- http response --- 18:10:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:10:59 INFO [http_cmd]: http status code OK 18:10:59 DEBUG --- status code --- 18:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:10:59 DEBUG --- http response --- 18:10:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:10:59 INFO [http_cmd]: http status code OK 18:10:59 INFO [http_cmd]: http status code OK 18:10:59 DEBUG --- status code --- 18:10:59 DEBUG --- status code --- 18:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:10:59 DEBUG --- http response --- 18:10:59 DEBUG --- http response --- 18:10:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:10:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:10:59 INFO [http_cmd]: http status code OK 18:10:59 DEBUG --- status code --- 18:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:10:59 DEBUG --- http response --- 18:10:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459059" 18:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459059" 18:10:59 INFO [http_cmd]: http status code OK 18:10:59 DEBUG --- status code --- 18:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:10:59 DEBUG --- http response --- 18:10:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:10:59 INFO [http_cmd]: http status code OK 18:10:59 DEBUG --- status code --- 18:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:10:59 DEBUG --- http response --- 18:10:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734459059" 18:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459059" 18:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459059" 18:10:59 ERROR [http_cmd]: ERROR 18:10:59 ERROR --- status code --- 18:10:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:10:59 ERROR --- http response --- 18:10:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:10:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:10:59 INFO [http_cmd]: http status code OK 18:10:59 DEBUG --- status code --- 18:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:10:59 DEBUG --- http response --- 18:10:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459059" 18:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459059" 18:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459059" 18:10:59 INFO [http_cmd]: http status code OK 18:10:59 DEBUG --- status code --- 18:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:10:59 DEBUG --- http response --- 18:10:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:10:59 INFO [http_cmd]: http status code OK 18:10:59 DEBUG --- status code --- 18:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:10:59 DEBUG --- http response --- 18:10:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:10:59 INFO [http_cmd]: http status code OK 18:10:59 DEBUG --- status code --- 18:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:10:59 DEBUG --- http response --- 18:10:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734459059" 18:10:59 INFO [http_cmd]: http status code OK 18:10:59 DEBUG --- status code --- 18:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:10:59 DEBUG --- http response --- 18:10:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:10:59 INFO [http_cmd]: http status code OK 18:10:59 DEBUG --- status code --- 18:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:10:59 DEBUG --- http response --- 18:10:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459059" 18:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459059" 18:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459059" 18:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734459059" 18:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459059" 18:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459059" 18:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734459059" 18:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459059" 18:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459059" 18:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459059" 18:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459059" 18:10:59 INFO [http_cmd]: http status code OK 18:10:59 DEBUG --- status code --- 18:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:10:59 DEBUG --- http response --- 18:10:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:10:59 INFO [http_cmd]: http status code OK 18:10:59 DEBUG --- status code --- 18:10:59 INFO [http_cmd]: http status code OK 18:10:59 INFO [http_cmd]: http status code OK 18:10:59 INFO [http_cmd]: http status code OK 18:10:59 DEBUG ----- output ----- 18:10:59 INFO [http_cmd]: http status code OK 18:10:59 INFO [http_cmd]: http status code OK 18:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:10:59 DEBUG --- status code --- 18:10:59 DEBUG --- status code --- 18:10:59 DEBUG --- status code --- 18:10:59 DEBUG { 18:10:59 DEBUG --- status code --- 18:10:59 DEBUG --- status code --- 18:10:59 DEBUG --- http response --- 18:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:10:59 INFO [http_cmd]: http status code OK 18:10:59 INFO [http_cmd]: http status code OK 18:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:10:59 DEBUG "status": "success", 18:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:10:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:10:59 DEBUG --- http response --- 18:10:59 DEBUG --- status code --- 18:10:59 DEBUG --- status code --- 18:10:59 DEBUG --- http response --- 18:10:59 INFO [http_cmd]: http status code OK 18:10:59 INFO [http_cmd]: http status code OK 18:10:59 DEBUG --- http response --- 18:10:59 DEBUG "data": { 18:10:59 DEBUG --- http response --- 18:10:59 DEBUG --- http response --- 18:10:59 DEBUG ----- output ----- 18:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:10:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:10:59 DEBUG --- status code --- 18:10:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:10:59 DEBUG --- status code --- 18:10:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:10:59 DEBUG "resultType": "vector", 18:10:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:10:59 DEBUG { 18:10:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:10:59 DEBUG --- http response --- 18:10:59 DEBUG ----- output ----- 18:10:59 DEBUG --- http response --- 18:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:10:59 DEBUG ----- output ----- 18:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:10:59 DEBUG ----- output ----- 18:10:59 DEBUG "result": [ 18:10:59 DEBUG ----- output ----- 18:10:59 DEBUG "status": "success", 18:10:59 DEBUG ----- output ----- 18:10:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:10:59 DEBUG { 18:10:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:10:59 DEBUG --- http response --- 18:10:59 DEBUG { 18:10:59 DEBUG --- http response --- 18:10:59 DEBUG { 18:10:59 DEBUG { 18:10:59 DEBUG { 18:10:59 DEBUG "data": { 18:10:59 DEBUG { 18:10:59 DEBUG ----- output ----- 18:10:59 DEBUG "status": "success", 18:10:59 DEBUG ----- output ----- 18:10:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:10:59 DEBUG "status": "success", 18:10:59 DEBUG "status": "success", 18:10:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:10:59 DEBUG "metric": { 18:10:59 DEBUG "status": "success", 18:10:59 DEBUG "resultType": "vector", 18:10:59 DEBUG "status": "success", 18:10:59 DEBUG { 18:10:59 DEBUG "data": { 18:10:59 DEBUG { 18:10:59 DEBUG ----- output ----- 18:10:59 DEBUG "data": { 18:10:59 DEBUG "data": { 18:10:59 DEBUG ----- output ----- 18:10:59 DEBUG "instance": "10.142.0.11:9100" 18:10:59 DEBUG "data": { 18:10:59 DEBUG "result": [ 18:10:59 DEBUG "data": { 18:10:59 DEBUG "status": "success", 18:10:59 DEBUG "resultType": "vector", 18:10:59 DEBUG "status": "success", 18:10:59 DEBUG { 18:10:59 DEBUG "resultType": "vector", 18:10:59 DEBUG "resultType": "vector", 18:10:59 DEBUG { 18:10:59 DEBUG }, 18:10:59 DEBUG "resultType": "vector", 18:10:59 DEBUG { 18:10:59 DEBUG "resultType": "vector", 18:10:59 DEBUG "data": { 18:10:59 DEBUG "result": [ 18:10:59 DEBUG "data": { 18:10:59 DEBUG "status": "success", 18:10:59 DEBUG "result": [ 18:10:59 DEBUG "result": [ 18:10:59 DEBUG "status": "success", 18:10:59 DEBUG "value": [ 18:10:59 DEBUG "result": [ 18:10:59 DEBUG "metric": { 18:10:59 DEBUG "result": [ 18:10:59 DEBUG "resultType": "vector", 18:10:59 DEBUG { 18:10:59 DEBUG "resultType": "vector", 18:10:59 DEBUG "data": { 18:10:59 DEBUG { 18:10:59 DEBUG { 18:10:59 DEBUG "data": { 18:10:59 DEBUG 18:10:59 DEBUG { 18:10:59 DEBUG "pod": "ds-cts-0" 18:10:59 DEBUG { 18:10:59 DEBUG "result": [ 18:10:59 DEBUG "metric": { 18:10:59 DEBUG "result": [ 18:10:59 DEBUG "resultType": "vector", 18:10:59 DEBUG "metric": { 18:10:59 DEBUG "metric": { 18:10:59 DEBUG "resultType": "vector", 18:10:59 DEBUG 18:10:59 DEBUG "metric": { 18:10:59 DEBUG }, 18:10:59 DEBUG "metric": { 18:10:59 DEBUG { 18:10:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:10:59 DEBUG { 18:10:59 DEBUG "result": [ 18:10:59 DEBUG "pod": "ds-cts-0" 18:10:59 DEBUG "instance": "10.142.0.11:9100" 18:10:59 DEBUG "result": [ 18:10:59 DEBUG 18:10:59 DEBUG "pod": "ds-cts-0" 18:10:59 DEBUG "value": [ 18:10:59 DEBUG "instance": "10.142.0.11:9100" 18:10:59 DEBUG "metric": { 18:10:59 DEBUG }, 18:10:59 DEBUG "metric": { 18:10:59 DEBUG { 18:10:59 DEBUG }, 18:10:59 DEBUG }, 18:10:59 DEBUG { 18:10:59 DEBUG [...] 18:10:59 DEBUG }, 18:10:59 DEBUG 18:10:59 DEBUG }, 18:10:59 DEBUG "instance": "10.142.0.11:9100" 18:10:59 DEBUG "value": [ 18:10:59 DEBUG "pod": "ds-cts-0" 18:10:59 DEBUG "metric": { 18:10:59 DEBUG "value": [ 18:10:59 DEBUG "value": [ 18:10:59 DEBUG "metric": { 18:10:59 DEBUG 18:10:59 DEBUG "value": [ 18:10:59 DEBUG 18:10:59 DEBUG "value": [ 18:10:59 DEBUG }, 18:10:59 DEBUG 18:10:59 DEBUG }, 18:10:59 DEBUG "instance": "10.142.0.11:9100" 18:10:59 DEBUG 18:10:59 DEBUG 18:10:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:10:59 DEBUG 18:10:59 DEBUG 18:10:59 DEBUG 18:10:59 DEBUG 18:10:59 DEBUG "value": [ 18:10:59 DEBUG 18:10:59 DEBUG "value": [ 18:10:59 DEBUG }, 18:10:59 DEBUG 18:10:59 DEBUG 18:10:59 DEBUG }, 18:10:59 DEBUG 18:10:59 DEBUG 18:10:59 DEBUG [...] 18:10:59 DEBUG 18:10:59 DEBUG 18:10:59 DEBUG 18:10:59 DEBUG 18:10:59 DEBUG "value": [ 18:10:59 DEBUG 18:10:59 DEBUG 18:10:59 DEBUG "value": [ 18:10:59 DEBUG "instance": "10.142.0.51:9100" 18:10:59 DEBUG 18:10:59 DEBUG 18:10:59 DEBUG 18:10:59 DEBUG 18:10:59 DEBUG [...] 18:10:59 DEBUG 18:10:59 DEBUG 18:10:59 DEBUG [...] 18:10:59 DEBUG [...] 18:10:59 DEBUG 18:10:59 DEBUG }, 18:10:59 DEBUG [...] 18:10:59 DEBUG 18:10:59 DEBUG 18:10:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:10:59 DEBUG 18:10:59 DEBUG 18:10:59 DEBUG 18:10:59 DEBUG 18:10:59 DEBUG 18:10:59 DEBUG 18:10:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:10:59 DEBUG }, 18:10:59 DEBUG "value": [ 18:10:59 DEBUG [...] 18:10:59 DEBUG }, 18:10:59 DEBUG 18:10:59 DEBUG 18:10:59 DEBUG 18:10:59 DEBUG 18:10:59 DEBUG 18:10:59 DEBUG "value": [ 18:10:59 DEBUG 18:10:59 DEBUG 18:10:59 DEBUG [...] 18:10:59 DEBUG 18:10:59 DEBUG 18:10:59 DEBUG [...] 18:10:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:10:59 DEBUG [...] 18:10:59 DEBUG 18:10:59 DEBUG 18:10:59 DEBUG 1734459059, 18:10:59 DEBUG 18:10:59 DEBUG 1734459059, 18:10:59 DEBUG 18:10:59 DEBUG "value": [ 18:10:59 DEBUG 1734459059, 18:10:59 DEBUG "0" 18:10:59 DEBUG ] 18:10:59 DEBUG 18:10:59 DEBUG 18:10:59 DEBUG [...] 18:10:59 DEBUG "115798.19999999998" 18:10:59 DEBUG 18:10:59 DEBUG "0.00909446614161536" 18:10:59 DEBUG 18:10:59 DEBUG 18:10:59 DEBUG 18:10:59 DEBUG }, 18:10:59 DEBUG } 18:10:59 DEBUG 18:10:59 DEBUG "instance": "10.142.0.51:9100" 18:10:59 DEBUG 18:10:59 DEBUG ] 18:10:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:10:59 DEBUG ] 18:10:59 DEBUG "instance": "10.142.0.51:9100" 18:10:59 DEBUG 18:10:59 DEBUG 18:10:59 DEBUG "value": [ 18:10:59 DEBUG ] 18:10:59 DEBUG 18:10:59 DEBUG }, 18:10:59 DEBUG 18:10:59 DEBUG } 18:10:59 DEBUG }, 18:10:59 DEBUG } 18:10:59 DEBUG }, 18:10:59 DEBUG "instance": "10.142.0.51:9100" 18:10:59 DEBUG 18:10:59 DEBUG 1734459059, 18:10:59 DEBUG } 18:10:59 DEBUG "instance": "10.142.0.51:9100" 18:10:59 DEBUG "value": [ 18:10:59 DEBUG 18:10:59 DEBUG ] 18:10:59 DEBUG "value": [ 18:10:59 DEBUG ] 18:10:59 DEBUG "value": [ 18:10:59 DEBUG }, 18:10:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:10:59 DEBUG "2785980416" 18:10:59 DEBUG } 18:10:59 DEBUG }, 18:10:59 DEBUG 1734459059, 18:10:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:10:59 DEBUG } 18:10:59 DEBUG 1734459059, 18:10:59 DEBUG } 18:10:59 DEBUG 1734459059, 18:10:59 DEBUG "value": [ 18:10:59 DEBUG }, 18:10:59 DEBUG ] 18:10:59 DEBUG "value": [ 18:10:59 DEBUG "0.00046666666666557904" 18:10:59 DEBUG }, 18:10:59 DEBUG } 18:10:59 DEBUG "705851392" 18:10:59 DEBUG } 18:10:59 DEBUG "27716.266666666663" 18:10:59 DEBUG ] 18:10:59 DEBUG } 18:10:59 DEBUG ] 18:10:59 DEBUG } 18:10:59 DEBUG ] 18:10:59 DEBUG "value": [ 18:10:59 DEBUG ] 18:10:59 DEBUG 1734459059, 18:10:59 DEBUG "value": [ 18:10:59 DEBUG } 18:10:59 DEBUG 1734459059, 18:10:59 DEBUG } 18:10:59 DEBUG } 18:10:59 DEBUG 1734459059, 18:10:59 DEBUG } 18:10:59 DEBUG "119717.56666666665" 18:10:59 DEBUG 1734459059, 18:10:59 DEBUG ] 18:10:59 DEBUG "0.0003333333333333854" 18:10:59 DEBUG ] 18:10:59 DEBUG "1.06129787482848" 18:10:59 DEBUG ] 18:10:59 DEBUG ] 18:10:59 DEBUG "1877.8009253735975" 18:10:59 DEBUG } 18:10:59 DEBUG ] 18:10:59 DEBUG } 18:10:59 DEBUG ] 18:10:59 DEBUG } 18:10:59 DEBUG } 18:10:59 DEBUG ] 18:10:59 DEBUG } 18:10:59 DEBUG } 18:10:59 DEBUG ] 18:10:59 DEBUG } 18:10:59 DEBUG } 18:10:59 DEBUG } 18:10:59 DEBUG } 18:10:59 DEBUG } 18:10:59 DEBUG } 18:10:59 DEBUG ] 18:10:59 DEBUG ] 18:10:59 DEBUG ] 18:10:59 DEBUG } 18:10:59 DEBUG } 18:10:59 DEBUG } 18:10:59 DEBUG } 18:10:59 DEBUG } 18:10:59 DEBUG } 18:10:59 INFO 18:10:59 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:10:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:10:59 INFO 18:10:59 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:10:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:10:59 INFO [loop_until]: OK (rc = 0) 18:10:59 DEBUG --- stdout --- 18:10:59 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 7m 466Mi ds-idrepo-0 8m 675Mi rcs-5d696ff5b-7ttvr 5m 295Mi rcs-5d696ff5b-wwlbr 5m 336Mi 18:10:59 DEBUG --- stderr --- 18:10:59 DEBUG 18:10:59 INFO [loop_until]: OK (rc = 0) 18:10:59 DEBUG --- stdout --- 18:10:59 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 107m 0% 2230Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1133m 3% 3695Mi 12% 18:10:59 DEBUG --- stderr --- 18:10:59 DEBUG 18:11:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459118" 18:11:58 INFO [http_cmd]: http status code OK 18:11:58 DEBUG --- status code --- 18:11:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:11:58 DEBUG --- http response --- 18:11:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:11:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734459118" 18:11:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459118" 18:11:58 INFO [http_cmd]: http status code OK 18:11:58 INFO [http_cmd]: http status code OK 18:11:58 DEBUG --- status code --- 18:11:58 DEBUG --- status code --- 18:11:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:11:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:11:58 DEBUG --- http response --- 18:11:58 DEBUG --- http response --- 18:11:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:11:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:11:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459118" 18:11:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459118" 18:11:58 INFO [http_cmd]: http status code OK 18:11:58 INFO [http_cmd]: http status code OK 18:11:58 DEBUG --- status code --- 18:11:58 DEBUG --- status code --- 18:11:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:11:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:11:58 DEBUG --- http response --- 18:11:58 DEBUG --- http response --- 18:11:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:11:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:11:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459118" 18:11:58 INFO [http_cmd]: http status code OK 18:11:58 DEBUG --- status code --- 18:11:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:11:58 DEBUG --- http response --- 18:11:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459119" 18:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459119" 18:11:59 INFO [http_cmd]: http status code OK 18:11:59 DEBUG --- status code --- 18:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:11:59 DEBUG --- http response --- 18:11:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459119" 18:11:59 INFO [http_cmd]: http status code OK 18:11:59 DEBUG --- status code --- 18:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:11:59 DEBUG --- http response --- 18:11:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459119" 18:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734459119" 18:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459119" 18:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459119" 18:11:59 INFO [http_cmd]: http status code OK 18:11:59 DEBUG --- status code --- 18:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:11:59 DEBUG --- http response --- 18:11:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459119" 18:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734459119" 18:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459119" 18:11:59 INFO [http_cmd]: http status code OK 18:11:59 DEBUG --- status code --- 18:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:11:59 DEBUG --- http response --- 18:11:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459119" 18:11:59 INFO [http_cmd]: http status code OK 18:11:59 DEBUG --- status code --- 18:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:11:59 DEBUG --- http response --- 18:11:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459119" 18:11:59 INFO [http_cmd]: http status code OK 18:11:59 DEBUG --- status code --- 18:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:11:59 DEBUG --- http response --- 18:11:59 INFO [http_cmd]: http status code OK 18:11:59 DEBUG --- status code --- 18:11:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:11:59 DEBUG --- http response --- 18:11:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:11:59 INFO [http_cmd]: http status code OK 18:11:59 DEBUG --- status code --- 18:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:11:59 DEBUG --- http response --- 18:11:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:11:59 INFO [http_cmd]: http status code OK 18:11:59 DEBUG --- status code --- 18:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:11:59 DEBUG --- http response --- 18:11:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:11:59 INFO [http_cmd]: http status code OK 18:11:59 DEBUG --- status code --- 18:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:11:59 DEBUG --- http response --- 18:11:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:11:59 INFO [http_cmd]: http status code OK 18:11:59 DEBUG --- status code --- 18:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:11:59 DEBUG --- http response --- 18:11:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:11:59 INFO [http_cmd]: http status code OK 18:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459119" 18:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459119" 18:11:59 DEBUG --- status code --- 18:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:11:59 DEBUG --- http response --- 18:11:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:11:59 INFO [http_cmd]: http status code OK 18:11:59 DEBUG --- status code --- 18:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:11:59 DEBUG --- http response --- 18:11:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:11:59 INFO [http_cmd]: http status code OK 18:11:59 DEBUG --- status code --- 18:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:11:59 DEBUG --- http response --- 18:11:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734459119" 18:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459119" 18:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459119" 18:11:59 ERROR [http_cmd]: ERROR 18:11:59 ERROR --- status code --- 18:11:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:11:59 ERROR --- http response --- 18:11:59 INFO [http_cmd]: http status code OK 18:11:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:11:59 DEBUG --- status code --- 18:11:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:11:59 DEBUG --- http response --- 18:11:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459119" 18:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459119" 18:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459119" 18:11:59 INFO [http_cmd]: http status code OK 18:11:59 DEBUG --- status code --- 18:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:11:59 DEBUG --- http response --- 18:11:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:11:59 INFO [http_cmd]: http status code OK 18:11:59 DEBUG --- status code --- 18:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:11:59 DEBUG --- http response --- 18:11:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734459119" 18:11:59 INFO [http_cmd]: http status code OK 18:11:59 DEBUG --- status code --- 18:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:11:59 INFO [http_cmd]: http status code OK 18:11:59 DEBUG --- http response --- 18:11:59 DEBUG --- status code --- 18:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:11:59 DEBUG --- http response --- 18:11:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:11:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:11:59 INFO [http_cmd]: http status code OK 18:11:59 DEBUG --- status code --- 18:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:11:59 DEBUG --- http response --- 18:11:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459119" 18:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459119" 18:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459119" 18:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459119" 18:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459119" 18:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734459119" 18:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459119" 18:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459119" 18:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734459119" 18:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459119" 18:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459119" 18:11:59 INFO [http_cmd]: http status code OK 18:11:59 DEBUG --- status code --- 18:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:11:59 DEBUG --- http response --- 18:11:59 INFO [http_cmd]: http status code OK 18:11:59 DEBUG --- status code --- 18:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:11:59 DEBUG --- http response --- 18:11:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:11:59 DEBUG ----- output ----- 18:11:59 DEBUG { 18:11:59 DEBUG "status": "success", 18:11:59 DEBUG "data": { 18:11:59 DEBUG "resultType": "vector", 18:11:59 DEBUG "result": [ 18:11:59 DEBUG { 18:11:59 DEBUG "metric": { 18:11:59 DEBUG "pod": "ds-cts-0" 18:11:59 DEBUG }, 18:11:59 DEBUG "value": [ 18:11:59 DEBUG 18:11:59 INFO [http_cmd]: http status code OK 18:11:59 DEBUG --- status code --- 18:11:59 DEBUG 18:11:59 DEBUG 18:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:11:59 DEBUG --- http response --- 18:11:59 INFO [http_cmd]: http status code OK 18:11:59 DEBUG [...] 18:11:59 DEBUG 18:11:59 DEBUG 18:11:59 DEBUG --- status code --- 18:11:59 INFO [http_cmd]: http status code OK 18:11:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:11:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:11:59 DEBUG 18:11:59 INFO [http_cmd]: http status code OK 18:11:59 INFO [http_cmd]: http status code OK 18:11:59 INFO [http_cmd]: http status code OK 18:11:59 DEBUG --- status code --- 18:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:11:59 DEBUG --- status code --- 18:11:59 DEBUG ----- output ----- 18:11:59 DEBUG ----- output ----- 18:11:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:11:59 DEBUG --- status code --- 18:11:59 DEBUG --- status code --- 18:11:59 INFO [http_cmd]: http status code OK 18:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:11:59 DEBUG --- http response --- 18:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:11:59 DEBUG { 18:11:59 INFO [http_cmd]: http status code OK 18:11:59 DEBUG { 18:11:59 DEBUG }, 18:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:11:59 DEBUG --- status code --- 18:11:59 DEBUG --- http response --- 18:11:59 DEBUG --- http response --- 18:11:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:11:59 DEBUG "status": "success", 18:11:59 DEBUG --- status code --- 18:11:59 DEBUG "status": "success", 18:11:59 DEBUG "value": [ 18:11:59 DEBUG --- http response --- 18:11:59 DEBUG --- http response --- 18:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:11:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:11:59 DEBUG ----- output ----- 18:11:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:11:59 DEBUG "data": { 18:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:11:59 DEBUG "data": { 18:11:59 DEBUG 1734459119, 18:11:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:11:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:11:59 DEBUG --- http response --- 18:11:59 DEBUG ----- output ----- 18:11:59 DEBUG { 18:11:59 DEBUG "status": "success", 18:11:59 DEBUG ----- output ----- 18:11:59 DEBUG "resultType": "vector", 18:11:59 DEBUG --- http response --- 18:11:59 DEBUG "resultType": "vector", 18:11:59 DEBUG "0" 18:11:59 DEBUG ----- output ----- 18:11:59 DEBUG ----- output ----- 18:11:59 DEBUG { 18:11:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:11:59 DEBUG ----- output ----- 18:11:59 DEBUG { 18:11:59 DEBUG { 18:11:59 DEBUG "result": [ 18:11:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:11:59 DEBUG "result": [ 18:11:59 DEBUG ] 18:11:59 DEBUG { 18:11:59 DEBUG { 18:11:59 DEBUG "status": "success", 18:11:59 INFO [http_cmd]: http status code OK 18:11:59 DEBUG "data": { 18:11:59 DEBUG "status": "success", 18:11:59 DEBUG "status": "success", 18:11:59 DEBUG { 18:11:59 DEBUG ----- output ----- 18:11:59 DEBUG { 18:11:59 DEBUG } 18:11:59 DEBUG "status": "success", 18:11:59 DEBUG "status": "success", 18:11:59 DEBUG "data": { 18:11:59 DEBUG "resultType": "vector", 18:11:59 DEBUG --- status code --- 18:11:59 DEBUG "data": { 18:11:59 DEBUG "data": { 18:11:59 DEBUG "metric": { 18:11:59 DEBUG { 18:11:59 DEBUG "metric": { 18:11:59 DEBUG ] 18:11:59 DEBUG "data": { 18:11:59 DEBUG "data": { 18:11:59 DEBUG "resultType": "vector", 18:11:59 DEBUG "result": [ 18:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:11:59 DEBUG "resultType": "vector", 18:11:59 DEBUG "resultType": "vector", 18:11:59 DEBUG "instance": "10.142.0.11:9100" 18:11:59 DEBUG "status": "success", 18:11:59 DEBUG "pod": "ds-cts-0" 18:11:59 DEBUG } 18:11:59 DEBUG "resultType": "vector", 18:11:59 DEBUG "resultType": "vector", 18:11:59 DEBUG "result": [ 18:11:59 DEBUG { 18:11:59 DEBUG --- http response --- 18:11:59 DEBUG "result": [ 18:11:59 DEBUG "result": [ 18:11:59 DEBUG }, 18:11:59 DEBUG "data": { 18:11:59 DEBUG }, 18:11:59 DEBUG } 18:11:59 DEBUG "result": [ 18:11:59 DEBUG "result": [ 18:11:59 DEBUG { 18:11:59 DEBUG "metric": { 18:11:59 DEBUG { 18:11:59 DEBUG "metric": { 18:11:59 DEBUG "instance": "10.142.0.11:9100" 18:11:59 DEBUG }, 18:11:59 DEBUG "resultType": "vector", 18:11:59 DEBUG "value": [ 18:11:59 DEBUG { 18:11:59 DEBUG { 18:11:59 DEBUG "metric": { 18:11:59 DEBUG "pod": "ds-cts-0" 18:11:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:11:59 DEBUG { 18:11:59 DEBUG "value": [ 18:11:59 DEBUG "value": [ 18:11:59 DEBUG "result": [ 18:11:59 DEBUG 18:11:59 DEBUG "metric": { 18:11:59 DEBUG "metric": { 18:11:59 DEBUG "instance": "10.142.0.11:9100" 18:11:59 DEBUG }, 18:11:59 DEBUG ----- output ----- 18:11:59 DEBUG "metric": { 18:11:59 DEBUG 18:11:59 DEBUG 18:11:59 DEBUG { 18:11:59 DEBUG 18:11:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:11:59 DEBUG "pod": "ds-cts-0" 18:11:59 DEBUG }, 18:11:59 DEBUG "value": [ 18:11:59 DEBUG { 18:11:59 DEBUG "instance": "10.142.0.11:9100" 18:11:59 DEBUG 18:11:59 DEBUG 18:11:59 DEBUG "metric": { 18:11:59 DEBUG 18:11:59 DEBUG }, 18:11:59 DEBUG }, 18:11:59 DEBUG "value": [ 18:11:59 DEBUG 18:11:59 DEBUG "status": "success", 18:11:59 DEBUG }, 18:11:59 DEBUG 18:11:59 DEBUG 18:11:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:11:59 DEBUG [...] 18:11:59 DEBUG "value": [ 18:11:59 DEBUG "value": [ 18:11:59 DEBUG 18:11:59 DEBUG 18:11:59 DEBUG "data": { 18:11:59 DEBUG "value": [ 18:11:59 DEBUG [...] 18:11:59 DEBUG [...] 18:11:59 DEBUG }, 18:11:59 DEBUG 18:11:59 DEBUG 18:11:59 DEBUG 18:11:59 DEBUG 18:11:59 DEBUG 18:11:59 DEBUG "resultType": "vector", 18:11:59 DEBUG 18:11:59 DEBUG 18:11:59 DEBUG 18:11:59 DEBUG "value": [ 18:11:59 DEBUG 18:11:59 DEBUG 18:11:59 DEBUG 18:11:59 DEBUG 18:11:59 DEBUG [...] 18:11:59 DEBUG "result": [ 18:11:59 DEBUG 18:11:59 DEBUG 18:11:59 DEBUG 18:11:59 DEBUG 18:11:59 DEBUG 18:11:59 DEBUG 18:11:59 DEBUG [...] 18:11:59 DEBUG 18:11:59 DEBUG 18:11:59 DEBUG 18:11:59 DEBUG { 18:11:59 DEBUG 18:11:59 DEBUG "instance": "10.142.0.51:9100" 18:11:59 DEBUG 18:11:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:11:59 DEBUG 18:11:59 DEBUG [...] 18:11:59 DEBUG 18:11:59 DEBUG 18:11:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:11:59 DEBUG }, 18:11:59 DEBUG "value": [ 18:11:59 DEBUG 1734459119, 18:11:59 DEBUG "2798198784" 18:11:59 DEBUG ] 18:11:59 DEBUG } 18:11:59 DEBUG ] 18:11:59 DEBUG } 18:11:59 DEBUG [...] 18:11:59 DEBUG 18:11:59 DEBUG 18:11:59 DEBUG 18:11:59 DEBUG "instance": "10.142.0.51:9100" 18:11:59 DEBUG }, 18:11:59 DEBUG [...] 18:11:59 DEBUG 18:11:59 DEBUG 18:11:59 DEBUG } 18:11:59 DEBUG "metric": { 18:11:59 DEBUG 18:11:59 DEBUG }, 18:11:59 DEBUG 18:11:59 DEBUG }, 18:11:59 DEBUG "value": [ 18:11:59 DEBUG 18:11:59 DEBUG 18:11:59 DEBUG 18:11:59 DEBUG "instance": "10.142.0.11:9100" 18:11:59 DEBUG "instance": "10.142.0.51:9100" 18:11:59 DEBUG "value": [ 18:11:59 DEBUG [...] 18:11:59 DEBUG "value": [ 18:11:59 DEBUG 1734459119, 18:11:59 DEBUG "0.0037022532870022292" 18:11:59 DEBUG ] 18:11:59 DEBUG } 18:11:59 DEBUG ] 18:11:59 DEBUG }, 18:11:59 DEBUG 1734459119, 18:11:59 DEBUG 18:11:59 DEBUG 1734459119, 18:11:59 DEBUG 18:11:59 DEBUG 18:11:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:11:59 DEBUG } 18:11:59 DEBUG }, 18:11:59 DEBUG "value": [ 18:11:59 DEBUG "121518.06666666667" 18:11:59 DEBUG ] 18:11:59 DEBUG } 18:11:59 DEBUG 18:11:59 DEBUG "instance": "10.142.0.51:9100" 18:11:59 DEBUG }, 18:11:59 DEBUG } 18:11:59 DEBUG "value": [ 18:11:59 DEBUG 1734459119, 18:11:59 DEBUG 18:11:59 DEBUG "125533.90000000001" 18:11:59 DEBUG ] 18:11:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:11:59 DEBUG }, 18:11:59 DEBUG "value": [ 18:11:59 DEBUG 1734459119, 18:11:59 DEBUG "25122.13333333333" 18:11:59 DEBUG 18:11:59 DEBUG ] 18:11:59 DEBUG } 18:11:59 DEBUG }, 18:11:59 DEBUG "value": [ 18:11:59 DEBUG 1734459119, 18:11:59 DEBUG "706105344" 18:11:59 DEBUG ] 18:11:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:11:59 DEBUG } 18:11:59 DEBUG } 18:11:59 DEBUG "value": [ 18:11:59 DEBUG 1734459119, 18:11:59 DEBUG "839.2276770904969" 18:11:59 DEBUG ] 18:11:59 DEBUG } 18:11:59 DEBUG }, 18:11:59 DEBUG ] 18:11:59 DEBUG } 18:11:59 DEBUG "0" 18:11:59 DEBUG ] 18:11:59 DEBUG } 18:11:59 DEBUG ] 18:11:59 DEBUG "value": [ 18:11:59 DEBUG 1734459119, 18:11:59 DEBUG "1.0614899005783638" 18:11:59 DEBUG ] 18:11:59 DEBUG } 18:11:59 DEBUG ] 18:11:59 DEBUG } 18:11:59 DEBUG } 18:11:59 DEBUG } 18:11:59 DEBUG ] 18:11:59 DEBUG } 18:11:59 DEBUG ] 18:11:59 DEBUG } 18:11:59 DEBUG } 18:11:59 DEBUG 18:11:59 DEBUG ] 18:11:59 DEBUG } 18:11:59 DEBUG } 18:11:59 DEBUG ] 18:11:59 DEBUG 18:11:59 DEBUG } 18:11:59 DEBUG } 18:11:59 DEBUG } 18:11:59 DEBUG } 18:11:59 DEBUG } 18:11:59 DEBUG 18:11:59 DEBUG [...] 18:11:59 DEBUG 18:11:59 DEBUG 18:11:59 DEBUG 18:11:59 DEBUG "instance": "10.142.0.51:9100" 18:11:59 DEBUG }, 18:11:59 DEBUG "value": [ 18:11:59 DEBUG 1734459119, 18:11:59 DEBUG "0.0004333333333358761" 18:11:59 DEBUG ] 18:11:59 DEBUG } 18:11:59 DEBUG ] 18:11:59 DEBUG } 18:11:59 DEBUG } 18:11:59 INFO 18:11:59 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:11:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:11:59 INFO 18:11:59 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:11:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:11:59 INFO [loop_until]: OK (rc = 0) 18:11:59 DEBUG --- stdout --- 18:11:59 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 7m 470Mi ds-idrepo-0 7m 680Mi rcs-5d696ff5b-7ttvr 4m 295Mi rcs-5d696ff5b-wwlbr 3m 336Mi 18:11:59 DEBUG --- stderr --- 18:11:59 DEBUG 18:11:59 INFO [loop_until]: OK (rc = 0) 18:11:59 DEBUG --- stdout --- 18:11:59 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 102m 0% 2240Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1128m 3% 3706Mi 13% 18:11:59 DEBUG --- stderr --- 18:11:59 DEBUG 18:12:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459178" 18:12:58 INFO [http_cmd]: http status code OK 18:12:58 DEBUG --- status code --- 18:12:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:12:58 DEBUG --- http response --- 18:12:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:12:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459178" 18:12:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734459178" 18:12:58 INFO [http_cmd]: http status code OK 18:12:58 DEBUG --- status code --- 18:12:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:12:58 DEBUG --- http response --- 18:12:58 INFO [http_cmd]: http status code OK 18:12:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:12:58 DEBUG --- status code --- 18:12:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:12:58 DEBUG --- http response --- 18:12:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:12:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459178" 18:12:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459178" 18:12:58 INFO [http_cmd]: http status code OK 18:12:58 DEBUG --- status code --- 18:12:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:12:58 DEBUG --- http response --- 18:12:58 INFO [http_cmd]: http status code OK 18:12:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:12:58 DEBUG --- status code --- 18:12:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:12:58 DEBUG --- http response --- 18:12:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:12:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459178" 18:12:59 INFO [http_cmd]: http status code OK 18:12:59 DEBUG --- status code --- 18:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:12:59 DEBUG --- http response --- 18:12:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459179" 18:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459179" 18:12:59 INFO [http_cmd]: http status code OK 18:12:59 DEBUG --- status code --- 18:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:12:59 DEBUG --- http response --- 18:12:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:12:59 INFO [http_cmd]: http status code OK 18:12:59 DEBUG --- status code --- 18:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:12:59 DEBUG --- http response --- 18:12:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459179" 18:12:59 INFO [http_cmd]: http status code OK 18:12:59 DEBUG --- status code --- 18:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:12:59 DEBUG --- http response --- 18:12:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459179" 18:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734459179" 18:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459179" 18:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459179" 18:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734459179" 18:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459179" 18:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459179" 18:12:59 INFO [http_cmd]: http status code OK 18:12:59 DEBUG --- status code --- 18:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:12:59 DEBUG --- http response --- 18:12:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:12:59 INFO [http_cmd]: http status code OK 18:12:59 DEBUG --- status code --- 18:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:12:59 DEBUG --- http response --- 18:12:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:12:59 INFO [http_cmd]: http status code OK 18:12:59 DEBUG --- status code --- 18:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:12:59 DEBUG --- http response --- 18:12:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:12:59 INFO [http_cmd]: http status code OK 18:12:59 DEBUG --- status code --- 18:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:12:59 DEBUG --- http response --- 18:12:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459179" 18:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459179" 18:12:59 INFO [http_cmd]: http status code OK 18:12:59 DEBUG --- status code --- 18:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:12:59 INFO [http_cmd]: http status code OK 18:12:59 DEBUG --- http response --- 18:12:59 DEBUG --- status code --- 18:12:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:12:59 DEBUG --- http response --- 18:12:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:12:59 INFO [http_cmd]: http status code OK 18:12:59 DEBUG --- status code --- 18:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:12:59 DEBUG --- http response --- 18:12:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:12:59 INFO [http_cmd]: http status code OK 18:12:59 DEBUG --- status code --- 18:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:12:59 DEBUG --- http response --- 18:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459179" 18:12:59 INFO [http_cmd]: http status code OK 18:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459179" 18:12:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:12:59 DEBUG --- status code --- 18:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:12:59 DEBUG --- http response --- 18:12:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:12:59 INFO [http_cmd]: http status code OK 18:12:59 DEBUG --- status code --- 18:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:12:59 DEBUG --- http response --- 18:12:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:12:59 INFO [http_cmd]: http status code OK 18:12:59 DEBUG --- status code --- 18:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:12:59 DEBUG --- http response --- 18:12:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734459179" 18:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459179" 18:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459179" 18:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459179" 18:12:59 INFO [http_cmd]: http status code OK 18:12:59 DEBUG --- status code --- 18:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:12:59 DEBUG --- http response --- 18:12:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:12:59 ERROR [http_cmd]: ERROR 18:12:59 ERROR --- status code --- 18:12:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:12:59 ERROR --- http response --- 18:12:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:12:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459179" 18:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459179" 18:12:59 INFO [http_cmd]: http status code OK 18:12:59 DEBUG --- status code --- 18:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:12:59 DEBUG --- http response --- 18:12:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:12:59 INFO [http_cmd]: http status code OK 18:12:59 DEBUG --- status code --- 18:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:12:59 DEBUG --- http response --- 18:12:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734459179" 18:12:59 INFO [http_cmd]: http status code OK 18:12:59 DEBUG --- status code --- 18:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:12:59 INFO [http_cmd]: http status code OK 18:12:59 DEBUG --- http response --- 18:12:59 DEBUG --- status code --- 18:12:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:12:59 DEBUG --- http response --- 18:12:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:12:59 INFO [http_cmd]: http status code OK 18:12:59 DEBUG --- status code --- 18:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:12:59 DEBUG --- http response --- 18:12:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459179" 18:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734459179" 18:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459179" 18:12:59 INFO [http_cmd]: http status code OK 18:12:59 DEBUG --- status code --- 18:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:12:59 DEBUG --- http response --- 18:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:12:59 DEBUG ----- output ----- 18:12:59 DEBUG { 18:12:59 DEBUG "status": "success", 18:12:59 DEBUG "data": { 18:12:59 DEBUG "resultType": "vector", 18:12:59 DEBUG "result": [ 18:12:59 DEBUG { 18:12:59 DEBUG "metric": { 18:12:59 DEBUG "pod": "ds-cts-0" 18:12:59 DEBUG }, 18:12:59 DEBUG "value": [ 18:12:59 DEBUG 18:12:59 DEBUG 18:12:59 DEBUG 18:12:59 DEBUG [...] 18:12:59 DEBUG 18:12:59 DEBUG 18:12:59 DEBUG 18:12:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459179" 18:12:59 DEBUG }, 18:12:59 DEBUG "value": [ 18:12:59 DEBUG 1734459179, 18:12:59 DEBUG "0" 18:12:59 DEBUG ] 18:12:59 DEBUG } 18:12:59 DEBUG ] 18:12:59 DEBUG } 18:12:59 DEBUG } 18:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459179" 18:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459179" 18:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459179" 18:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734459179" 18:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459179" 18:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459179" 18:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459179" 18:12:59 INFO [http_cmd]: http status code OK 18:12:59 DEBUG --- status code --- 18:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:12:59 DEBUG --- http response --- 18:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:12:59 DEBUG ----- output ----- 18:12:59 DEBUG { 18:12:59 DEBUG "status": "success", 18:12:59 DEBUG "data": { 18:12:59 DEBUG "resultType": "vector", 18:12:59 DEBUG "result": [ 18:12:59 DEBUG { 18:12:59 DEBUG "metric": { 18:12:59 DEBUG "pod": "ds-cts-0" 18:12:59 DEBUG }, 18:12:59 DEBUG "value": [ 18:12:59 DEBUG 18:12:59 DEBUG 18:12:59 DEBUG 18:12:59 INFO [http_cmd]: http status code OK 18:12:59 DEBUG [...] 18:12:59 INFO [http_cmd]: http status code OK 18:12:59 DEBUG --- status code --- 18:12:59 DEBUG 18:12:59 DEBUG --- status code --- 18:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:12:59 INFO [http_cmd]: http status code OK 18:12:59 DEBUG 18:12:59 INFO [http_cmd]: http status code OK 18:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:12:59 DEBUG --- http response --- 18:12:59 DEBUG --- status code --- 18:12:59 DEBUG 18:12:59 DEBUG --- status code --- 18:12:59 DEBUG --- http response --- 18:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:12:59 INFO [http_cmd]: http status code OK 18:12:59 INFO [http_cmd]: http status code OK 18:12:59 INFO [http_cmd]: http status code OK 18:12:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:12:59 INFO [http_cmd]: http status code OK 18:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:12:59 DEBUG ----- output ----- 18:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:12:59 DEBUG --- http response --- 18:12:59 DEBUG --- status code --- 18:12:59 DEBUG --- status code --- 18:12:59 DEBUG --- status code --- 18:12:59 DEBUG }, 18:12:59 DEBUG --- status code --- 18:12:59 DEBUG --- http response --- 18:12:59 DEBUG { 18:12:59 DEBUG ----- output ----- 18:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:12:59 INFO [http_cmd]: http status code OK 18:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:12:59 DEBUG "value": [ 18:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:12:59 DEBUG "status": "success", 18:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:12:59 DEBUG { 18:12:59 DEBUG ----- output ----- 18:12:59 DEBUG --- status code --- 18:12:59 DEBUG --- http response --- 18:12:59 DEBUG --- http response --- 18:12:59 DEBUG --- http response --- 18:12:59 DEBUG 1734459179, 18:12:59 DEBUG --- http response --- 18:12:59 DEBUG "data": { 18:12:59 DEBUG ----- output ----- 18:12:59 DEBUG "status": "success", 18:12:59 DEBUG { 18:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:12:59 DEBUG "0.006469949781932603" 18:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:12:59 DEBUG "resultType": "vector", 18:12:59 DEBUG { 18:12:59 DEBUG "data": { 18:12:59 DEBUG "status": "success", 18:12:59 DEBUG --- http response --- 18:12:59 DEBUG ----- output ----- 18:12:59 DEBUG ----- output ----- 18:12:59 DEBUG ----- output ----- 18:12:59 DEBUG ] 18:12:59 DEBUG ----- output ----- 18:12:59 DEBUG "result": [ 18:12:59 DEBUG "status": "success", 18:12:59 DEBUG "resultType": "vector", 18:12:59 DEBUG "data": { 18:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:12:59 DEBUG { 18:12:59 DEBUG { 18:12:59 DEBUG { 18:12:59 DEBUG } 18:12:59 DEBUG { 18:12:59 DEBUG { 18:12:59 DEBUG "data": { 18:12:59 DEBUG "result": [ 18:12:59 DEBUG "resultType": "vector", 18:12:59 DEBUG ----- output ----- 18:12:59 DEBUG "status": "success", 18:12:59 DEBUG "status": "success", 18:12:59 DEBUG "status": "success", 18:12:59 DEBUG ] 18:12:59 DEBUG "status": "success", 18:12:59 DEBUG "metric": { 18:12:59 DEBUG "resultType": "vector", 18:12:59 DEBUG { 18:12:59 DEBUG "result": [ 18:12:59 DEBUG { 18:12:59 DEBUG "data": { 18:12:59 DEBUG "data": { 18:12:59 DEBUG "data": { 18:12:59 DEBUG } 18:12:59 DEBUG "data": { 18:12:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:12:59 DEBUG "result": [ 18:12:59 DEBUG "metric": { 18:12:59 DEBUG { 18:12:59 DEBUG "status": "success", 18:12:59 DEBUG "resultType": "vector", 18:12:59 DEBUG "resultType": "vector", 18:12:59 DEBUG "resultType": "vector", 18:12:59 DEBUG } 18:12:59 DEBUG "resultType": "vector", 18:12:59 DEBUG }, 18:12:59 DEBUG { 18:12:59 DEBUG "instance": "10.142.0.11:9100" 18:12:59 DEBUG "metric": { 18:12:59 DEBUG "data": { 18:12:59 DEBUG "result": [ 18:12:59 DEBUG "result": [ 18:12:59 DEBUG "result": [ 18:12:59 DEBUG "result": [ 18:12:59 DEBUG "value": [ 18:12:59 DEBUG "metric": { 18:12:59 DEBUG }, 18:12:59 DEBUG "instance": "10.142.0.11:9100" 18:12:59 DEBUG "resultType": "vector", 18:12:59 DEBUG { 18:12:59 DEBUG { 18:12:59 DEBUG { 18:12:59 DEBUG { 18:12:59 DEBUG 18:12:59 DEBUG "pod": "ds-cts-0" 18:12:59 DEBUG "value": [ 18:12:59 DEBUG }, 18:12:59 DEBUG "result": [ 18:12:59 DEBUG "metric": { 18:12:59 DEBUG "metric": { 18:12:59 DEBUG "metric": { 18:12:59 DEBUG "metric": { 18:12:59 DEBUG 18:12:59 DEBUG }, 18:12:59 DEBUG 18:12:59 DEBUG "value": [ 18:12:59 DEBUG { 18:12:59 DEBUG "pod": "ds-cts-0" 18:12:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:12:59 DEBUG "instance": "10.142.0.11:9100" 18:12:59 DEBUG "instance": "10.142.0.11:9100" 18:12:59 DEBUG 18:12:59 DEBUG "value": [ 18:12:59 DEBUG 18:12:59 DEBUG 18:12:59 DEBUG "metric": { 18:12:59 DEBUG }, 18:12:59 DEBUG }, 18:12:59 DEBUG }, 18:12:59 DEBUG }, 18:12:59 DEBUG [...] 18:12:59 DEBUG 18:12:59 DEBUG 18:12:59 DEBUG 18:12:59 DEBUG "instance": "10.142.0.11:9100" 18:12:59 DEBUG "value": [ 18:12:59 DEBUG "value": [ 18:12:59 DEBUG "value": [ 18:12:59 DEBUG "value": [ 18:12:59 DEBUG 18:12:59 DEBUG 18:12:59 DEBUG [...] 18:12:59 DEBUG 18:12:59 DEBUG }, 18:12:59 DEBUG 18:12:59 DEBUG 18:12:59 DEBUG 18:12:59 DEBUG 18:12:59 DEBUG 18:12:59 DEBUG 18:12:59 DEBUG 18:12:59 DEBUG [...] 18:12:59 DEBUG "value": [ 18:12:59 DEBUG 18:12:59 DEBUG 18:12:59 DEBUG 18:12:59 DEBUG 18:12:59 DEBUG 18:12:59 DEBUG [...] 18:12:59 DEBUG 18:12:59 DEBUG 18:12:59 DEBUG 18:12:59 DEBUG 18:12:59 DEBUG 18:12:59 DEBUG 18:12:59 DEBUG 18:12:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:12:59 DEBUG 18:12:59 DEBUG 18:12:59 DEBUG 18:12:59 DEBUG 18:12:59 DEBUG [...] 18:12:59 DEBUG [...] 18:12:59 DEBUG [...] 18:12:59 DEBUG [...] 18:12:59 DEBUG }, 18:12:59 DEBUG 18:12:59 DEBUG "instance": "10.142.0.51:9100" 18:12:59 DEBUG 18:12:59 DEBUG 18:12:59 DEBUG 18:12:59 DEBUG 18:12:59 DEBUG 18:12:59 DEBUG 18:12:59 DEBUG "value": [ 18:12:59 DEBUG 18:12:59 DEBUG }, 18:12:59 DEBUG "instance": "10.142.0.51:9100" 18:12:59 DEBUG [...] 18:12:59 DEBUG 18:12:59 DEBUG 18:12:59 DEBUG 18:12:59 DEBUG 18:12:59 DEBUG 1734459179, 18:12:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:12:59 DEBUG "value": [ 18:12:59 DEBUG }, 18:12:59 DEBUG 18:12:59 DEBUG 18:12:59 DEBUG 18:12:59 DEBUG 18:12:59 DEBUG 18:12:59 DEBUG "2808565760" 18:12:59 DEBUG }, 18:12:59 DEBUG 1734459179, 18:12:59 DEBUG "value": [ 18:12:59 DEBUG 18:12:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:12:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:12:59 DEBUG "instance": "10.142.0.51:9100" 18:12:59 DEBUG "instance": "10.142.0.51:9100" 18:12:59 DEBUG ] 18:12:59 DEBUG "value": [ 18:12:59 DEBUG "123174.83333333333" 18:12:59 DEBUG 1734459179, 18:12:59 DEBUG 18:12:59 DEBUG }, 18:12:59 DEBUG }, 18:12:59 DEBUG }, 18:12:59 DEBUG }, 18:12:59 DEBUG } 18:12:59 DEBUG 1734459179, 18:12:59 DEBUG ] 18:12:59 DEBUG "126965.56666666667" 18:12:59 DEBUG "instance": "10.142.0.51:9100" 18:12:59 DEBUG "value": [ 18:12:59 DEBUG "value": [ 18:12:59 DEBUG "value": [ 18:12:59 DEBUG "value": [ 18:12:59 DEBUG ] 18:12:59 DEBUG "706506752" 18:12:59 DEBUG } 18:12:59 DEBUG ] 18:12:59 DEBUG }, 18:12:59 DEBUG 1734459179, 18:12:59 DEBUG 1734459179, 18:12:59 DEBUG 1734459179, 18:12:59 DEBUG 1734459179, 18:12:59 DEBUG } 18:12:59 DEBUG ] 18:12:59 DEBUG ] 18:12:59 DEBUG } 18:12:59 DEBUG "value": [ 18:12:59 DEBUG "541.0117553823801" 18:12:59 DEBUG "1.0587943051637982" 18:12:59 DEBUG "24576" 18:12:59 DEBUG "0.00040000000000001514" 18:12:59 DEBUG } 18:12:59 DEBUG } 18:12:59 DEBUG } 18:12:59 DEBUG ] 18:12:59 DEBUG 1734459179, 18:12:59 DEBUG ] 18:12:59 DEBUG ] 18:12:59 DEBUG ] 18:12:59 DEBUG ] 18:12:59 DEBUG ] 18:12:59 DEBUG } 18:12:59 DEBUG } 18:12:59 DEBUG "0" 18:12:59 DEBUG } 18:12:59 DEBUG } 18:12:59 DEBUG } 18:12:59 DEBUG } 18:12:59 DEBUG } 18:12:59 DEBUG } 18:12:59 DEBUG ] 18:12:59 DEBUG ] 18:12:59 DEBUG ] 18:12:59 DEBUG ] 18:12:59 DEBUG ] 18:12:59 DEBUG } 18:12:59 DEBUG } 18:12:59 DEBUG } 18:12:59 DEBUG } 18:12:59 DEBUG } 18:12:59 DEBUG } 18:12:59 DEBUG ] 18:12:59 DEBUG } 18:12:59 DEBUG } 18:12:59 DEBUG } 18:12:59 DEBUG } 18:12:59 DEBUG } 18:12:59 DEBUG } 18:12:59 INFO 18:12:59 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:12:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:12:59 INFO 18:12:59 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:12:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:12:59 INFO [loop_until]: OK (rc = 0) 18:12:59 DEBUG --- stdout --- 18:12:59 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 473Mi ds-idrepo-0 7m 688Mi rcs-5d696ff5b-7ttvr 4m 295Mi rcs-5d696ff5b-wwlbr 3m 336Mi 18:12:59 DEBUG --- stderr --- 18:12:59 DEBUG 18:12:59 INFO [loop_until]: OK (rc = 0) 18:12:59 DEBUG --- stdout --- 18:12:59 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 103m 0% 2253Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1136m 3% 3711Mi 13% 18:12:59 DEBUG --- stderr --- 18:12:59 DEBUG 18:13:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459238" 18:13:58 INFO [http_cmd]: http status code OK 18:13:58 DEBUG --- status code --- 18:13:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:13:58 DEBUG --- http response --- 18:13:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:13:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459238" 18:13:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734459238" 18:13:58 INFO [http_cmd]: http status code OK 18:13:58 INFO [http_cmd]: http status code OK 18:13:58 DEBUG --- status code --- 18:13:58 DEBUG --- status code --- 18:13:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:13:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:13:58 DEBUG --- http response --- 18:13:58 DEBUG --- http response --- 18:13:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:13:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:13:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459238" 18:13:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459238" 18:13:58 INFO [http_cmd]: http status code OK 18:13:58 DEBUG --- status code --- 18:13:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:13:58 INFO [http_cmd]: http status code OK 18:13:58 DEBUG --- http response --- 18:13:58 DEBUG --- status code --- 18:13:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:13:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:13:58 DEBUG --- http response --- 18:13:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459239" 18:13:59 INFO [http_cmd]: http status code OK 18:13:59 DEBUG --- status code --- 18:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:13:59 DEBUG --- http response --- 18:13:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459239" 18:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459239" 18:13:59 INFO [http_cmd]: http status code OK 18:13:59 DEBUG --- status code --- 18:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:13:59 INFO [http_cmd]: http status code OK 18:13:59 DEBUG --- http response --- 18:13:59 DEBUG --- status code --- 18:13:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:13:59 DEBUG --- http response --- 18:13:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459239" 18:13:59 INFO [http_cmd]: http status code OK 18:13:59 DEBUG --- status code --- 18:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:13:59 DEBUG --- http response --- 18:13:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459239" 18:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734459239" 18:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459239" 18:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459239" 18:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459239" 18:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459239" 18:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734459239" 18:13:59 INFO [http_cmd]: http status code OK 18:13:59 DEBUG --- status code --- 18:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:13:59 DEBUG --- http response --- 18:13:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:13:59 INFO [http_cmd]: http status code OK 18:13:59 DEBUG --- status code --- 18:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:13:59 DEBUG --- http response --- 18:13:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459239" 18:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459239" 18:13:59 INFO [http_cmd]: http status code OK 18:13:59 DEBUG --- status code --- 18:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:13:59 DEBUG --- http response --- 18:13:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:13:59 INFO [http_cmd]: http status code OK 18:13:59 DEBUG --- status code --- 18:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:13:59 DEBUG --- http response --- 18:13:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:13:59 INFO [http_cmd]: http status code OK 18:13:59 DEBUG --- status code --- 18:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:13:59 DEBUG --- http response --- 18:13:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:13:59 INFO [http_cmd]: http status code OK 18:13:59 INFO [http_cmd]: http status code OK 18:13:59 DEBUG --- status code --- 18:13:59 DEBUG --- status code --- 18:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:13:59 DEBUG --- http response --- 18:13:59 DEBUG --- http response --- 18:13:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:13:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459239" 18:13:59 INFO [http_cmd]: http status code OK 18:13:59 INFO [http_cmd]: http status code OK 18:13:59 DEBUG --- status code --- 18:13:59 DEBUG --- status code --- 18:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:13:59 DEBUG --- http response --- 18:13:59 DEBUG --- http response --- 18:13:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:13:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459239" 18:13:59 INFO [http_cmd]: http status code OK 18:13:59 DEBUG --- status code --- 18:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:13:59 DEBUG --- http response --- 18:13:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:13:59 INFO [http_cmd]: http status code OK 18:13:59 DEBUG --- status code --- 18:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:13:59 DEBUG --- http response --- 18:13:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459239" 18:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734459239" 18:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459239" 18:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459239" 18:13:59 INFO [http_cmd]: http status code OK 18:13:59 DEBUG --- status code --- 18:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:13:59 DEBUG --- http response --- 18:13:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:13:59 ERROR [http_cmd]: ERROR 18:13:59 ERROR --- status code --- 18:13:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:13:59 ERROR --- http response --- 18:13:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:13:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459239" 18:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459239" 18:13:59 INFO [http_cmd]: http status code OK 18:13:59 DEBUG --- status code --- 18:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:13:59 DEBUG --- http response --- 18:13:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734459239" 18:13:59 INFO [http_cmd]: http status code OK 18:13:59 DEBUG --- status code --- 18:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:13:59 DEBUG --- http response --- 18:13:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:13:59 INFO [http_cmd]: http status code OK 18:13:59 DEBUG --- status code --- 18:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:13:59 DEBUG --- http response --- 18:13:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:13:59 INFO [http_cmd]: http status code OK 18:13:59 INFO [http_cmd]: http status code OK 18:13:59 DEBUG --- status code --- 18:13:59 DEBUG --- status code --- 18:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:13:59 DEBUG --- http response --- 18:13:59 DEBUG --- http response --- 18:13:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:13:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459239" 18:13:59 INFO [http_cmd]: http status code OK 18:13:59 DEBUG --- status code --- 18:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:13:59 DEBUG --- http response --- 18:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:13:59 DEBUG ----- output ----- 18:13:59 DEBUG { 18:13:59 DEBUG "status": "success", 18:13:59 DEBUG "data": { 18:13:59 DEBUG "resultType": "vector", 18:13:59 DEBUG "result": [ 18:13:59 DEBUG { 18:13:59 DEBUG "metric": { 18:13:59 DEBUG "pod": "ds-cts-0" 18:13:59 DEBUG }, 18:13:59 DEBUG "value": [ 18:13:59 DEBUG 18:13:59 DEBUG 18:13:59 DEBUG 18:13:59 DEBUG [...] 18:13:59 DEBUG 18:13:59 DEBUG 18:13:59 DEBUG 18:13:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:13:59 DEBUG }, 18:13:59 DEBUG "value": [ 18:13:59 DEBUG 1734459239, 18:13:59 DEBUG "0" 18:13:59 DEBUG ] 18:13:59 DEBUG } 18:13:59 DEBUG ] 18:13:59 DEBUG } 18:13:59 DEBUG } 18:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459239" 18:13:59 INFO [http_cmd]: http status code OK 18:13:59 DEBUG --- status code --- 18:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:13:59 DEBUG --- http response --- 18:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:13:59 DEBUG ----- output ----- 18:13:59 DEBUG { 18:13:59 DEBUG "status": "success", 18:13:59 DEBUG "data": { 18:13:59 DEBUG "resultType": "vector", 18:13:59 DEBUG "result": [ 18:13:59 DEBUG { 18:13:59 DEBUG "metric": { 18:13:59 DEBUG "pod": "ds-cts-0" 18:13:59 DEBUG }, 18:13:59 DEBUG "value": [ 18:13:59 DEBUG 18:13:59 DEBUG 18:13:59 DEBUG 18:13:59 DEBUG [...] 18:13:59 DEBUG 18:13:59 DEBUG 18:13:59 DEBUG 18:13:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:13:59 DEBUG }, 18:13:59 DEBUG "value": [ 18:13:59 DEBUG 1734459239, 18:13:59 DEBUG "0.004946235126643619" 18:13:59 DEBUG ] 18:13:59 DEBUG } 18:13:59 DEBUG ] 18:13:59 DEBUG } 18:13:59 DEBUG } 18:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734459239" 18:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459239" 18:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459239" 18:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459239" 18:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459239" 18:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459239" 18:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734459239" 18:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459239" 18:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459239" 18:13:59 INFO [http_cmd]: http status code OK 18:13:59 INFO [http_cmd]: http status code OK 18:13:59 INFO [http_cmd]: http status code OK 18:13:59 DEBUG --- status code --- 18:13:59 DEBUG --- status code --- 18:13:59 DEBUG --- status code --- 18:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:13:59 INFO [http_cmd]: http status code OK 18:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:13:59 DEBUG --- http response --- 18:13:59 DEBUG --- http response --- 18:13:59 DEBUG --- status code --- 18:13:59 DEBUG --- http response --- 18:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:13:59 DEBUG ----- output ----- 18:13:59 DEBUG ----- output ----- 18:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:13:59 DEBUG --- http response --- 18:13:59 DEBUG { 18:13:59 DEBUG { 18:13:59 DEBUG ----- output ----- 18:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:13:59 INFO [http_cmd]: http status code OK 18:13:59 INFO [http_cmd]: http status code OK 18:13:59 DEBUG "status": "success", 18:13:59 INFO [http_cmd]: http status code OK 18:13:59 DEBUG "status": "success", 18:13:59 INFO [http_cmd]: http status code OK 18:13:59 DEBUG { 18:13:59 DEBUG ----- output ----- 18:13:59 DEBUG --- status code --- 18:13:59 DEBUG --- status code --- 18:13:59 DEBUG "data": { 18:13:59 DEBUG --- status code --- 18:13:59 DEBUG "data": { 18:13:59 DEBUG --- status code --- 18:13:59 DEBUG "status": "success", 18:13:59 DEBUG { 18:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:13:59 DEBUG "resultType": "vector", 18:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:13:59 DEBUG "resultType": "vector", 18:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:13:59 DEBUG "data": { 18:13:59 DEBUG "status": "success", 18:13:59 DEBUG --- http response --- 18:13:59 DEBUG --- http response --- 18:13:59 DEBUG "result": [ 18:13:59 DEBUG --- http response --- 18:13:59 DEBUG "result": [ 18:13:59 DEBUG --- http response --- 18:13:59 DEBUG "resultType": "vector", 18:13:59 DEBUG "data": { 18:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:13:59 DEBUG { 18:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:13:59 DEBUG { 18:13:59 DEBUG "result": [ 18:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:13:59 DEBUG "resultType": "vector", 18:13:59 DEBUG ----- output ----- 18:13:59 DEBUG ----- output ----- 18:13:59 DEBUG "metric": { 18:13:59 DEBUG ----- output ----- 18:13:59 DEBUG "metric": { 18:13:59 DEBUG { 18:13:59 DEBUG ----- output ----- 18:13:59 DEBUG "result": [ 18:13:59 DEBUG { 18:13:59 DEBUG { 18:13:59 DEBUG "instance": "10.142.0.11:9100" 18:13:59 DEBUG { 18:13:59 DEBUG "instance": "10.142.0.11:9100" 18:13:59 DEBUG "metric": { 18:13:59 DEBUG { 18:13:59 DEBUG { 18:13:59 DEBUG "status": "success", 18:13:59 INFO [http_cmd]: http status code OK 18:13:59 DEBUG "status": "success", 18:13:59 DEBUG }, 18:13:59 DEBUG "status": "success", 18:13:59 DEBUG }, 18:13:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:13:59 DEBUG "status": "success", 18:13:59 DEBUG "metric": { 18:13:59 DEBUG "data": { 18:13:59 DEBUG --- status code --- 18:13:59 DEBUG "data": { 18:13:59 DEBUG "value": [ 18:13:59 DEBUG "data": { 18:13:59 DEBUG "value": [ 18:13:59 DEBUG }, 18:13:59 DEBUG "data": { 18:13:59 DEBUG "pod": "ds-cts-0" 18:13:59 DEBUG "resultType": "vector", 18:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:13:59 DEBUG "resultType": "vector", 18:13:59 DEBUG 18:13:59 DEBUG "resultType": "vector", 18:13:59 DEBUG 18:13:59 DEBUG "value": [ 18:13:59 DEBUG "resultType": "vector", 18:13:59 DEBUG }, 18:13:59 DEBUG "result": [ 18:13:59 DEBUG --- http response --- 18:13:59 DEBUG "result": [ 18:13:59 DEBUG 18:13:59 DEBUG "result": [ 18:13:59 DEBUG 18:13:59 DEBUG 18:13:59 DEBUG "result": [ 18:13:59 DEBUG "value": [ 18:13:59 DEBUG { 18:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:13:59 DEBUG { 18:13:59 DEBUG 18:13:59 DEBUG { 18:13:59 DEBUG 18:13:59 DEBUG 18:13:59 DEBUG { 18:13:59 DEBUG 18:13:59 DEBUG "metric": { 18:13:59 DEBUG ----- output ----- 18:13:59 DEBUG "metric": { 18:13:59 DEBUG [...] 18:13:59 DEBUG "metric": { 18:13:59 DEBUG [...] 18:13:59 DEBUG 18:13:59 DEBUG "metric": { 18:13:59 DEBUG 18:13:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:13:59 DEBUG { 18:13:59 DEBUG "instance": "10.142.0.11:9100" 18:13:59 DEBUG 18:13:59 DEBUG "instance": "10.142.0.11:9100" 18:13:59 DEBUG 18:13:59 DEBUG [...] 18:13:59 DEBUG "instance": "10.142.0.11:9100" 18:13:59 DEBUG 18:13:59 DEBUG }, 18:13:59 DEBUG "status": "success", 18:13:59 DEBUG }, 18:13:59 DEBUG 18:13:59 DEBUG }, 18:13:59 DEBUG 18:13:59 DEBUG 18:13:59 DEBUG }, 18:13:59 DEBUG [...] 18:13:59 DEBUG "value": [ 18:13:59 DEBUG "data": { 18:13:59 DEBUG "value": [ 18:13:59 DEBUG 18:13:59 DEBUG "value": [ 18:13:59 DEBUG 18:13:59 DEBUG 18:13:59 DEBUG "value": [ 18:13:59 DEBUG 18:13:59 DEBUG 18:13:59 DEBUG "resultType": "vector", 18:13:59 DEBUG 18:13:59 DEBUG "instance": "10.142.0.51:9100" 18:13:59 DEBUG 18:13:59 DEBUG "instance": "10.142.0.51:9100" 18:13:59 DEBUG 18:13:59 DEBUG 18:13:59 DEBUG 18:13:59 DEBUG 18:13:59 DEBUG "result": [ 18:13:59 DEBUG 18:13:59 DEBUG }, 18:13:59 DEBUG 18:13:59 DEBUG }, 18:13:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:13:59 DEBUG 18:13:59 DEBUG 18:13:59 DEBUG 18:13:59 DEBUG { 18:13:59 DEBUG 18:13:59 DEBUG "value": [ 18:13:59 DEBUG 18:13:59 DEBUG "value": [ 18:13:59 DEBUG }, 18:13:59 DEBUG 18:13:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:13:59 DEBUG [...] 18:13:59 DEBUG "metric": { 18:13:59 DEBUG [...] 18:13:59 DEBUG 1734459239, 18:13:59 DEBUG [...] 18:13:59 DEBUG 1734459239, 18:13:59 DEBUG "value": [ 18:13:59 DEBUG [...] 18:13:59 DEBUG }, 18:13:59 DEBUG 18:13:59 DEBUG "pod": "ds-cts-0" 18:13:59 DEBUG 18:13:59 DEBUG "129295.13333333335" 18:13:59 DEBUG 18:13:59 DEBUG "124774.5666666667" 18:13:59 DEBUG 1734459239, 18:13:59 DEBUG 18:13:59 DEBUG "value": [ 18:13:59 DEBUG 18:13:59 DEBUG }, 18:13:59 DEBUG 18:13:59 DEBUG ] 18:13:59 DEBUG 18:13:59 DEBUG ] 18:13:59 DEBUG "2822852608" 18:13:59 DEBUG 18:13:59 DEBUG 1734459239, 18:13:59 DEBUG 18:13:59 DEBUG "value": [ 18:13:59 DEBUG 18:13:59 DEBUG } 18:13:59 DEBUG 18:13:59 DEBUG } 18:13:59 DEBUG ] 18:13:59 DEBUG 18:13:59 DEBUG "707424256" 18:13:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:13:59 DEBUG 18:13:59 DEBUG "instance": "10.142.0.51:9100" 18:13:59 DEBUG ] 18:13:59 DEBUG "instance": "10.142.0.51:9100" 18:13:59 DEBUG ] 18:13:59 DEBUG } 18:13:59 DEBUG "instance": "10.142.0.51:9100" 18:13:59 DEBUG ] 18:13:59 DEBUG }, 18:13:59 DEBUG 18:13:59 DEBUG }, 18:13:59 DEBUG } 18:13:59 DEBUG }, 18:13:59 DEBUG } 18:13:59 DEBUG } 18:13:59 DEBUG }, 18:13:59 DEBUG } 18:13:59 DEBUG "value": [ 18:13:59 DEBUG 18:13:59 DEBUG "value": [ 18:13:59 DEBUG } 18:13:59 DEBUG "value": [ 18:13:59 DEBUG ] 18:13:59 DEBUG "value": [ 18:13:59 DEBUG ] 18:13:59 DEBUG 1734459239, 18:13:59 DEBUG [...] 18:13:59 DEBUG 1734459239, 18:13:59 DEBUG 1734459239, 18:13:59 DEBUG } 18:13:59 DEBUG 1734459239, 18:13:59 DEBUG } 18:13:59 DEBUG "1.055852695495779" 18:13:59 DEBUG 18:13:59 DEBUG "0.0003666666666665227" 18:13:59 DEBUG "0.000666666666666534" 18:13:59 DEBUG } 18:13:59 DEBUG "18568.533333333333" 18:13:59 DEBUG } 18:13:59 DEBUG ] 18:13:59 DEBUG 18:13:59 DEBUG ] 18:13:59 DEBUG ] 18:13:59 DEBUG ] 18:13:59 DEBUG } 18:13:59 DEBUG 18:13:59 DEBUG } 18:13:59 DEBUG } 18:13:59 DEBUG } 18:13:59 DEBUG ] 18:13:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:13:59 DEBUG ] 18:13:59 DEBUG ] 18:13:59 DEBUG ] 18:13:59 DEBUG } 18:13:59 DEBUG }, 18:13:59 DEBUG } 18:13:59 DEBUG } 18:13:59 DEBUG } 18:13:59 DEBUG } 18:13:59 DEBUG "value": [ 18:13:59 DEBUG } 18:13:59 DEBUG } 18:13:59 DEBUG } 18:13:59 DEBUG 1734459239, 18:13:59 DEBUG "884.1549856032623" 18:13:59 DEBUG ] 18:13:59 DEBUG } 18:13:59 DEBUG ] 18:13:59 DEBUG } 18:13:59 DEBUG } 18:13:59 INFO 18:13:59 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:13:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:13:59 INFO 18:13:59 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:13:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:13:59 INFO [loop_until]: OK (rc = 0) 18:13:59 DEBUG --- stdout --- 18:13:59 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 7m 477Mi ds-idrepo-0 7m 691Mi rcs-5d696ff5b-7ttvr 12m 296Mi rcs-5d696ff5b-wwlbr 4m 337Mi 18:13:59 DEBUG --- stderr --- 18:13:59 DEBUG 18:13:59 INFO [loop_until]: OK (rc = 0) 18:13:59 DEBUG --- stdout --- 18:13:59 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 98m 0% 2260Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1147m 3% 3722Mi 13% 18:13:59 DEBUG --- stderr --- 18:13:59 DEBUG 18:14:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459298" 18:14:58 INFO [http_cmd]: http status code OK 18:14:58 DEBUG --- status code --- 18:14:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:14:58 DEBUG --- http response --- 18:14:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:14:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459298" 18:14:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734459298" 18:14:58 INFO [http_cmd]: http status code OK 18:14:58 INFO [http_cmd]: http status code OK 18:14:58 DEBUG --- status code --- 18:14:58 DEBUG --- status code --- 18:14:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:14:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:14:58 DEBUG --- http response --- 18:14:58 DEBUG --- http response --- 18:14:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:14:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:14:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459298" 18:14:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459298" 18:14:59 INFO [http_cmd]: http status code OK 18:14:59 DEBUG --- status code --- 18:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:14:59 DEBUG --- http response --- 18:14:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:14:59 INFO [http_cmd]: http status code OK 18:14:59 DEBUG --- status code --- 18:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:14:59 DEBUG --- http response --- 18:14:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459299" 18:14:59 INFO [http_cmd]: http status code OK 18:14:59 DEBUG --- status code --- 18:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:14:59 DEBUG --- http response --- 18:14:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459299" 18:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459299" 18:14:59 INFO [http_cmd]: http status code OK 18:14:59 DEBUG --- status code --- 18:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:14:59 DEBUG --- http response --- 18:14:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:14:59 INFO [http_cmd]: http status code OK 18:14:59 DEBUG --- status code --- 18:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:14:59 DEBUG --- http response --- 18:14:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459299" 18:14:59 INFO [http_cmd]: http status code OK 18:14:59 DEBUG --- status code --- 18:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:14:59 DEBUG --- http response --- 18:14:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459299" 18:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459299" 18:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459299" 18:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459299" 18:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734459299" 18:14:59 INFO [http_cmd]: http status code OK 18:14:59 DEBUG --- status code --- 18:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:14:59 DEBUG --- http response --- 18:14:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734459299" 18:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459299" 18:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459299" 18:14:59 INFO [http_cmd]: http status code OK 18:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459299" 18:14:59 DEBUG --- status code --- 18:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:14:59 DEBUG --- http response --- 18:14:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:14:59 INFO [http_cmd]: http status code OK 18:14:59 DEBUG --- status code --- 18:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:14:59 DEBUG --- http response --- 18:14:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:14:59 INFO [http_cmd]: http status code OK 18:14:59 DEBUG --- status code --- 18:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:14:59 DEBUG --- http response --- 18:14:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:14:59 INFO [http_cmd]: http status code OK 18:14:59 DEBUG --- status code --- 18:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:14:59 DEBUG --- http response --- 18:14:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459299" 18:14:59 INFO [http_cmd]: http status code OK 18:14:59 INFO [http_cmd]: http status code OK 18:14:59 DEBUG --- status code --- 18:14:59 INFO [http_cmd]: http status code OK 18:14:59 DEBUG --- status code --- 18:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:14:59 DEBUG --- status code --- 18:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:14:59 DEBUG --- http response --- 18:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:14:59 DEBUG --- http response --- 18:14:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:14:59 DEBUG --- http response --- 18:14:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:14:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459299" 18:14:59 INFO [http_cmd]: http status code OK 18:14:59 DEBUG --- status code --- 18:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:14:59 DEBUG --- http response --- 18:14:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:14:59 INFO [http_cmd]: http status code OK 18:14:59 DEBUG --- status code --- 18:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:14:59 DEBUG --- http response --- 18:14:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:14:59 INFO [http_cmd]: http status code OK 18:14:59 DEBUG --- status code --- 18:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:14:59 DEBUG --- http response --- 18:14:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734459299" 18:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459299" 18:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459299" 18:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459299" 18:14:59 INFO [http_cmd]: http status code OK 18:14:59 DEBUG --- status code --- 18:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:14:59 ERROR [http_cmd]: ERROR 18:14:59 DEBUG --- http response --- 18:14:59 ERROR --- status code --- 18:14:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:14:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:14:59 ERROR --- http response --- 18:14:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:14:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459299" 18:14:59 INFO [http_cmd]: http status code OK 18:14:59 DEBUG --- status code --- 18:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:14:59 DEBUG --- http response --- 18:14:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459299" 18:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734459299" 18:14:59 INFO [http_cmd]: http status code OK 18:14:59 INFO [http_cmd]: http status code OK 18:14:59 DEBUG --- status code --- 18:14:59 DEBUG --- status code --- 18:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:14:59 DEBUG --- http response --- 18:14:59 DEBUG --- http response --- 18:14:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:14:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:14:59 INFO [http_cmd]: http status code OK 18:14:59 DEBUG --- status code --- 18:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:14:59 DEBUG --- http response --- 18:14:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:14:59 INFO [http_cmd]: http status code OK 18:14:59 DEBUG --- status code --- 18:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:14:59 DEBUG --- http response --- 18:14:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459299" 18:14:59 INFO [http_cmd]: http status code OK 18:14:59 DEBUG --- status code --- 18:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:14:59 DEBUG --- http response --- 18:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:14:59 DEBUG ----- output ----- 18:14:59 DEBUG { 18:14:59 DEBUG "status": "success", 18:14:59 DEBUG "data": { 18:14:59 DEBUG "resultType": "vector", 18:14:59 DEBUG "result": [ 18:14:59 DEBUG { 18:14:59 DEBUG "metric": { 18:14:59 DEBUG "pod": "ds-cts-0" 18:14:59 DEBUG }, 18:14:59 DEBUG "value": [ 18:14:59 DEBUG 18:14:59 DEBUG 18:14:59 DEBUG 18:14:59 DEBUG [...] 18:14:59 DEBUG 18:14:59 DEBUG 18:14:59 DEBUG 18:14:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:14:59 DEBUG }, 18:14:59 DEBUG "value": [ 18:14:59 DEBUG 1734459299, 18:14:59 DEBUG "0" 18:14:59 DEBUG ] 18:14:59 DEBUG } 18:14:59 DEBUG ] 18:14:59 DEBUG } 18:14:59 DEBUG } 18:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459299" 18:14:59 INFO [http_cmd]: http status code OK 18:14:59 DEBUG --- status code --- 18:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:14:59 DEBUG --- http response --- 18:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:14:59 DEBUG ----- output ----- 18:14:59 DEBUG { 18:14:59 DEBUG "status": "success", 18:14:59 DEBUG "data": { 18:14:59 DEBUG "resultType": "vector", 18:14:59 DEBUG "result": [ 18:14:59 DEBUG { 18:14:59 DEBUG "metric": { 18:14:59 DEBUG "pod": "ds-cts-0" 18:14:59 DEBUG }, 18:14:59 DEBUG "value": [ 18:14:59 DEBUG 18:14:59 DEBUG 18:14:59 DEBUG 18:14:59 DEBUG [...] 18:14:59 DEBUG 18:14:59 DEBUG 18:14:59 DEBUG 18:14:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:14:59 DEBUG }, 18:14:59 DEBUG "value": [ 18:14:59 DEBUG 1734459299, 18:14:59 DEBUG "0.005627336012724075" 18:14:59 DEBUG ] 18:14:59 DEBUG } 18:14:59 DEBUG ] 18:14:59 DEBUG } 18:14:59 DEBUG } 18:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459299" 18:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459299" 18:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734459299" 18:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459299" 18:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734459299" 18:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459299" 18:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459299" 18:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459299" 18:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459299" 18:14:59 INFO [http_cmd]: http status code OK 18:14:59 DEBUG --- status code --- 18:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:14:59 INFO [http_cmd]: http status code OK 18:14:59 INFO [http_cmd]: http status code OK 18:14:59 INFO [http_cmd]: http status code OK 18:14:59 DEBUG --- http response --- 18:14:59 DEBUG --- status code --- 18:14:59 DEBUG --- status code --- 18:14:59 DEBUG --- status code --- 18:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:14:59 DEBUG --- http response --- 18:14:59 DEBUG --- http response --- 18:14:59 INFO [http_cmd]: http status code OK 18:14:59 INFO [http_cmd]: http status code OK 18:14:59 INFO [http_cmd]: http status code OK 18:14:59 DEBUG --- http response --- 18:14:59 INFO [http_cmd]: http status code OK 18:14:59 DEBUG ----- output ----- 18:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:14:59 DEBUG --- status code --- 18:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:14:59 DEBUG --- status code --- 18:14:59 DEBUG --- status code --- 18:14:59 DEBUG --- status code --- 18:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:14:59 DEBUG { 18:14:59 DEBUG ----- output ----- 18:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:14:59 DEBUG ----- output ----- 18:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:14:59 DEBUG ----- output ----- 18:14:59 DEBUG "status": "success", 18:14:59 DEBUG { 18:14:59 DEBUG --- http response --- 18:14:59 DEBUG { 18:14:59 DEBUG --- http response --- 18:14:59 DEBUG --- http response --- 18:14:59 DEBUG --- http response --- 18:14:59 DEBUG { 18:14:59 DEBUG "data": { 18:14:59 DEBUG "status": "success", 18:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:14:59 DEBUG "status": "success", 18:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:14:59 INFO [http_cmd]: http status code OK 18:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:14:59 DEBUG "status": "success", 18:14:59 DEBUG "resultType": "vector", 18:14:59 DEBUG "data": { 18:14:59 DEBUG ----- output ----- 18:14:59 DEBUG "data": { 18:14:59 DEBUG ----- output ----- 18:14:59 DEBUG --- status code --- 18:14:59 DEBUG ----- output ----- 18:14:59 DEBUG ----- output ----- 18:14:59 DEBUG "data": { 18:14:59 DEBUG "result": [ 18:14:59 DEBUG "resultType": "vector", 18:14:59 DEBUG { 18:14:59 DEBUG "resultType": "vector", 18:14:59 DEBUG { 18:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:14:59 DEBUG { 18:14:59 DEBUG { 18:14:59 DEBUG "status": "success", 18:14:59 DEBUG "data": { 18:14:59 DEBUG "result": [ 18:14:59 DEBUG "status": "success", 18:14:59 DEBUG "result": [ 18:14:59 DEBUG "status": "success", 18:14:59 DEBUG --- http response --- 18:14:59 DEBUG "status": "success", 18:14:59 DEBUG "resultType": "vector", 18:14:59 DEBUG { 18:14:59 DEBUG "resultType": "vector", 18:14:59 DEBUG { 18:14:59 DEBUG "data": { 18:14:59 DEBUG { 18:14:59 DEBUG "data": { 18:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:14:59 DEBUG "data": { 18:14:59 DEBUG "result": [ 18:14:59 DEBUG "metric": { 18:14:59 DEBUG "result": [ 18:14:59 DEBUG "metric": { 18:14:59 DEBUG "resultType": "vector", 18:14:59 DEBUG "metric": { 18:14:59 DEBUG "resultType": "vector", 18:14:59 DEBUG ----- output ----- 18:14:59 DEBUG "resultType": "vector", 18:14:59 DEBUG { 18:14:59 DEBUG "instance": "10.142.0.11:9100" 18:14:59 DEBUG { 18:14:59 DEBUG "instance": "10.142.0.11:9100" 18:14:59 DEBUG "result": [ 18:14:59 DEBUG "pod": "ds-cts-0" 18:14:59 DEBUG "result": [ 18:14:59 DEBUG { 18:14:59 DEBUG "result": [ 18:14:59 DEBUG "metric": { 18:14:59 DEBUG }, 18:14:59 DEBUG "metric": { 18:14:59 DEBUG }, 18:14:59 DEBUG { 18:14:59 DEBUG }, 18:14:59 DEBUG { 18:14:59 DEBUG "status": "success", 18:14:59 DEBUG { 18:14:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:14:59 DEBUG "value": [ 18:14:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:14:59 DEBUG "value": [ 18:14:59 DEBUG "metric": { 18:14:59 DEBUG "value": [ 18:14:59 DEBUG "metric": { 18:14:59 DEBUG "data": { 18:14:59 DEBUG "metric": { 18:14:59 DEBUG }, 18:14:59 DEBUG 18:14:59 DEBUG }, 18:14:59 DEBUG 18:14:59 DEBUG "instance": "10.142.0.11:9100" 18:14:59 DEBUG 18:14:59 DEBUG "instance": "10.142.0.11:9100" 18:14:59 DEBUG "resultType": "vector", 18:14:59 DEBUG "pod": "ds-cts-0" 18:14:59 DEBUG "value": [ 18:14:59 DEBUG 18:14:59 DEBUG "value": [ 18:14:59 DEBUG 18:14:59 DEBUG }, 18:14:59 DEBUG 18:14:59 DEBUG }, 18:14:59 DEBUG "result": [ 18:14:59 DEBUG }, 18:14:59 DEBUG 18:14:59 DEBUG 18:14:59 DEBUG 18:14:59 DEBUG 18:14:59 DEBUG "value": [ 18:14:59 DEBUG 18:14:59 DEBUG "value": [ 18:14:59 DEBUG { 18:14:59 DEBUG "value": [ 18:14:59 DEBUG 18:14:59 DEBUG [...] 18:14:59 DEBUG 18:14:59 DEBUG [...] 18:14:59 DEBUG 18:14:59 DEBUG [...] 18:14:59 DEBUG 18:14:59 DEBUG "metric": { 18:14:59 DEBUG 18:14:59 DEBUG 18:14:59 DEBUG 18:14:59 DEBUG 18:14:59 DEBUG 18:14:59 DEBUG 18:14:59 DEBUG 18:14:59 DEBUG 18:14:59 DEBUG "instance": "10.142.0.11:9100" 18:14:59 DEBUG 18:14:59 DEBUG [...] 18:14:59 DEBUG 18:14:59 DEBUG [...] 18:14:59 DEBUG 18:14:59 DEBUG 18:14:59 DEBUG 18:14:59 DEBUG 18:14:59 DEBUG }, 18:14:59 DEBUG 18:14:59 DEBUG 18:14:59 DEBUG 18:14:59 DEBUG 18:14:59 DEBUG 18:14:59 DEBUG [...] 18:14:59 DEBUG 18:14:59 DEBUG [...] 18:14:59 DEBUG "value": [ 18:14:59 DEBUG [...] 18:14:59 DEBUG 18:14:59 DEBUG "instance": "10.142.0.51:9100" 18:14:59 DEBUG 18:14:59 DEBUG "instance": "10.142.0.51:9100" 18:14:59 DEBUG }, 18:14:59 DEBUG "value": [ 18:14:59 DEBUG 18:14:59 DEBUG 18:14:59 DEBUG 18:14:59 DEBUG 18:14:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:14:59 DEBUG 18:14:59 DEBUG 18:14:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:14:59 DEBUG 1734459299, 18:14:59 DEBUG 18:14:59 DEBUG 18:14:59 DEBUG 18:14:59 DEBUG }, 18:14:59 DEBUG }, 18:14:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:14:59 DEBUG 18:14:59 DEBUG }, 18:14:59 DEBUG "126606.66666666666" 18:14:59 DEBUG 18:14:59 DEBUG 18:14:59 DEBUG 18:14:59 DEBUG "value": [ 18:14:59 DEBUG "value": [ 18:14:59 DEBUG }, 18:14:59 DEBUG 18:14:59 DEBUG "value": [ 18:14:59 DEBUG ] 18:14:59 DEBUG "instance": "10.142.0.51:9100" 18:14:59 DEBUG [...] 18:14:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:14:59 DEBUG 1734459299, 18:14:59 DEBUG 1734459299, 18:14:59 DEBUG "value": [ 18:14:59 DEBUG "instance": "10.142.0.51:9100" 18:14:59 DEBUG 1734459299, 18:14:59 DEBUG } 18:14:59 DEBUG }, 18:14:59 DEBUG 18:14:59 DEBUG }, 18:14:59 DEBUG "131081.86666666667" 18:14:59 DEBUG "2837532672" 18:14:59 DEBUG 1734459299, 18:14:59 DEBUG }, 18:14:59 DEBUG "708325376" 18:14:59 DEBUG ] 18:14:59 DEBUG "value": [ 18:14:59 DEBUG 18:14:59 DEBUG "value": [ 18:14:59 DEBUG ] 18:14:59 DEBUG ] 18:14:59 DEBUG "1.0558219487205525" 18:14:59 DEBUG "value": [ 18:14:59 DEBUG ] 18:14:59 DEBUG } 18:14:59 DEBUG 1734459299, 18:14:59 DEBUG 18:14:59 DEBUG 1734459299, 18:14:59 DEBUG } 18:14:59 DEBUG } 18:14:59 DEBUG ] 18:14:59 DEBUG } 18:14:59 DEBUG 1734459299, 18:14:59 DEBUG } 18:14:59 DEBUG "0" 18:14:59 DEBUG "instance": "10.142.0.51:9100" 18:14:59 DEBUG "1195.8859338154564" 18:14:59 DEBUG ] 18:14:59 DEBUG ] 18:14:59 DEBUG } 18:14:59 DEBUG ] 18:14:59 DEBUG "0.00043333333333208654" 18:14:59 DEBUG ] 18:14:59 DEBUG }, 18:14:59 DEBUG ] 18:14:59 DEBUG } 18:14:59 DEBUG } 18:14:59 DEBUG ] 18:14:59 DEBUG } 18:14:59 DEBUG ] 18:14:59 DEBUG } 18:14:59 DEBUG "value": [ 18:14:59 DEBUG } 18:14:59 DEBUG } 18:14:59 DEBUG } 18:14:59 DEBUG } 18:14:59 DEBUG } 18:14:59 DEBUG } 18:14:59 DEBUG ] 18:14:59 DEBUG 1734459299, 18:14:59 DEBUG ] 18:14:59 DEBUG } 18:14:59 DEBUG ] 18:14:59 DEBUG } 18:14:59 DEBUG "24849.066666666666" 18:14:59 DEBUG } 18:14:59 DEBUG } 18:14:59 DEBUG } 18:14:59 DEBUG ] 18:14:59 DEBUG } 18:14:59 DEBUG } 18:14:59 DEBUG } 18:14:59 DEBUG ] 18:14:59 DEBUG } 18:14:59 DEBUG } 18:14:59 INFO 18:14:59 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:14:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:14:59 INFO 18:14:59 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:14:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:14:59 INFO [loop_until]: OK (rc = 0) 18:14:59 DEBUG --- stdout --- 18:14:59 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 7m 480Mi ds-idrepo-0 8m 696Mi rcs-5d696ff5b-7ttvr 4m 296Mi rcs-5d696ff5b-wwlbr 3m 337Mi 18:14:59 DEBUG --- stderr --- 18:14:59 DEBUG 18:14:59 INFO [loop_until]: OK (rc = 0) 18:14:59 DEBUG --- stdout --- 18:14:59 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 100m 0% 2265Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1139m 3% 3732Mi 13% 18:14:59 DEBUG --- stderr --- 18:14:59 DEBUG 18:15:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459358" 18:15:58 INFO [http_cmd]: http status code OK 18:15:58 DEBUG --- status code --- 18:15:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:15:58 DEBUG --- http response --- 18:15:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:15:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734459358" 18:15:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459358" 18:15:58 INFO [http_cmd]: http status code OK 18:15:58 INFO [http_cmd]: http status code OK 18:15:58 DEBUG --- status code --- 18:15:58 DEBUG --- status code --- 18:15:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:15:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:15:58 DEBUG --- http response --- 18:15:58 DEBUG --- http response --- 18:15:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:15:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459359" 18:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459359" 18:15:59 INFO [http_cmd]: http status code OK 18:15:59 DEBUG --- status code --- 18:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:15:59 DEBUG --- http response --- 18:15:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:15:59 INFO [http_cmd]: http status code OK 18:15:59 DEBUG --- status code --- 18:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:15:59 DEBUG --- http response --- 18:15:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459359" 18:15:59 INFO [http_cmd]: http status code OK 18:15:59 DEBUG --- status code --- 18:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:15:59 DEBUG --- http response --- 18:15:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459359" 18:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459359" 18:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459359" 18:15:59 INFO [http_cmd]: http status code OK 18:15:59 DEBUG --- status code --- 18:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:15:59 DEBUG --- http response --- 18:15:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:15:59 INFO [http_cmd]: http status code OK 18:15:59 DEBUG --- status code --- 18:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:15:59 DEBUG --- http response --- 18:15:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:15:59 INFO [http_cmd]: http status code OK 18:15:59 DEBUG --- status code --- 18:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:15:59 DEBUG --- http response --- 18:15:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459359" 18:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459359" 18:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459359" 18:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734459359" 18:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459359" 18:15:59 INFO [http_cmd]: http status code OK 18:15:59 DEBUG --- status code --- 18:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:15:59 DEBUG --- http response --- 18:15:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459359" 18:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734459359" 18:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459359" 18:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459359" 18:15:59 INFO [http_cmd]: http status code OK 18:15:59 INFO [http_cmd]: http status code OK 18:15:59 INFO [http_cmd]: http status code OK 18:15:59 INFO [http_cmd]: http status code OK 18:15:59 DEBUG --- status code --- 18:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459359" 18:15:59 DEBUG --- status code --- 18:15:59 DEBUG --- status code --- 18:15:59 DEBUG --- status code --- 18:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:15:59 DEBUG --- http response --- 18:15:59 DEBUG --- http response --- 18:15:59 DEBUG --- http response --- 18:15:59 DEBUG --- http response --- 18:15:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:15:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:15:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:15:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459359" 18:15:59 INFO [http_cmd]: http status code OK 18:15:59 INFO [http_cmd]: http status code OK 18:15:59 INFO [http_cmd]: http status code OK 18:15:59 DEBUG --- status code --- 18:15:59 DEBUG --- status code --- 18:15:59 DEBUG --- status code --- 18:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:15:59 DEBUG --- http response --- 18:15:59 DEBUG --- http response --- 18:15:59 DEBUG --- http response --- 18:15:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:15:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:15:59 INFO [http_cmd]: http status code OK 18:15:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:15:59 DEBUG --- status code --- 18:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:15:59 DEBUG --- http response --- 18:15:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:15:59 INFO [http_cmd]: http status code OK 18:15:59 DEBUG --- status code --- 18:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:15:59 DEBUG --- http response --- 18:15:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:15:59 INFO [http_cmd]: http status code OK 18:15:59 DEBUG --- status code --- 18:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:15:59 DEBUG --- http response --- 18:15:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734459359" 18:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459359" 18:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459359" 18:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459359" 18:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459359" 18:15:59 ERROR [http_cmd]: ERROR 18:15:59 ERROR --- status code --- 18:15:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:15:59 ERROR --- http response --- 18:15:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:15:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:15:59 INFO [http_cmd]: http status code OK 18:15:59 DEBUG --- status code --- 18:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:15:59 DEBUG --- http response --- 18:15:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459359" 18:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734459359" 18:15:59 INFO [http_cmd]: http status code OK 18:15:59 DEBUG --- status code --- 18:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:15:59 DEBUG --- http response --- 18:15:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:15:59 INFO [http_cmd]: http status code OK 18:15:59 DEBUG --- status code --- 18:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:15:59 DEBUG --- http response --- 18:15:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:15:59 INFO [http_cmd]: http status code OK 18:15:59 DEBUG --- status code --- 18:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:15:59 DEBUG --- http response --- 18:15:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:15:59 INFO [http_cmd]: http status code OK 18:15:59 DEBUG --- status code --- 18:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:15:59 DEBUG --- http response --- 18:15:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:15:59 INFO [http_cmd]: http status code OK 18:15:59 DEBUG --- status code --- 18:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:15:59 DEBUG --- http response --- 18:15:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459359" 18:15:59 INFO [http_cmd]: http status code OK 18:15:59 DEBUG --- status code --- 18:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:15:59 DEBUG --- http response --- 18:15:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:15:59 DEBUG ----- output ----- 18:15:59 DEBUG { 18:15:59 DEBUG "status": "success", 18:15:59 DEBUG "data": { 18:15:59 DEBUG "resultType": "vector", 18:15:59 DEBUG "result": [ 18:15:59 DEBUG { 18:15:59 DEBUG "metric": { 18:15:59 DEBUG "pod": "ds-cts-0" 18:15:59 DEBUG }, 18:15:59 DEBUG "value": [ 18:15:59 DEBUG 18:15:59 DEBUG 18:15:59 DEBUG 18:15:59 DEBUG [...] 18:15:59 DEBUG 18:15:59 DEBUG 18:15:59 DEBUG 18:15:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:15:59 DEBUG }, 18:15:59 DEBUG "value": [ 18:15:59 DEBUG 1734459359, 18:15:59 DEBUG "0" 18:15:59 DEBUG ] 18:15:59 DEBUG } 18:15:59 DEBUG ] 18:15:59 DEBUG } 18:15:59 DEBUG } 18:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459359" 18:15:59 INFO [http_cmd]: http status code OK 18:15:59 DEBUG --- status code --- 18:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:15:59 DEBUG --- http response --- 18:15:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:15:59 DEBUG ----- output ----- 18:15:59 DEBUG { 18:15:59 DEBUG "status": "success", 18:15:59 DEBUG "data": { 18:15:59 DEBUG "resultType": "vector", 18:15:59 DEBUG "result": [ 18:15:59 DEBUG { 18:15:59 DEBUG "metric": { 18:15:59 DEBUG "pod": "ds-cts-0" 18:15:59 DEBUG }, 18:15:59 DEBUG "value": [ 18:15:59 DEBUG 18:15:59 DEBUG 18:15:59 DEBUG 18:15:59 DEBUG [...] 18:15:59 DEBUG 18:15:59 DEBUG 18:15:59 DEBUG 18:15:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:15:59 DEBUG }, 18:15:59 DEBUG "value": [ 18:15:59 DEBUG 1734459359, 18:15:59 DEBUG "0.0029725146967503717" 18:15:59 DEBUG ] 18:15:59 DEBUG } 18:15:59 DEBUG ] 18:15:59 DEBUG } 18:15:59 DEBUG } 18:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459359" 18:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459359" 18:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734459359" 18:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459359" 18:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734459359" 18:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459359" 18:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459359" 18:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459359" 18:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459359" 18:15:59 INFO [http_cmd]: http status code OK 18:15:59 DEBUG --- status code --- 18:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:15:59 INFO [http_cmd]: http status code OK 18:15:59 INFO [http_cmd]: http status code OK 18:15:59 INFO [http_cmd]: http status code OK 18:15:59 DEBUG --- http response --- 18:15:59 DEBUG --- status code --- 18:15:59 DEBUG --- status code --- 18:15:59 DEBUG --- status code --- 18:15:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:15:59 INFO [http_cmd]: http status code OK 18:15:59 INFO [http_cmd]: http status code OK 18:15:59 DEBUG ----- output ----- 18:15:59 INFO [http_cmd]: http status code OK 18:15:59 DEBUG --- http response --- 18:15:59 DEBUG --- http response --- 18:15:59 DEBUG --- http response --- 18:15:59 DEBUG --- status code --- 18:15:59 DEBUG --- status code --- 18:15:59 DEBUG { 18:15:59 DEBUG --- status code --- 18:15:59 INFO [http_cmd]: http status code OK 18:15:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:15:59 INFO [http_cmd]: http status code OK 18:15:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:15:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:15:59 DEBUG "status": "success", 18:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:15:59 DEBUG --- status code --- 18:15:59 DEBUG ----- output ----- 18:15:59 [35mDEBUG --- status code --- 18:15:59 DEBUG ----- output ----- 18:15:59 DEBUG ----- output ----- 18:15:59 DEBUG --- http response --- 18:15:59 DEBUG --- http response --- 18:15:59 DEBUG "data": { 18:15:59 DEBUG --- http response --- 18:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:15:59 DEBUG { 18:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:15:59 DEBUG { 18:15:59 DEBUG { 18:15:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:15:59 DEBUG "resultType": "vector", 18:15:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:15:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:15:59 DEBUG --- http response --- 18:15:59 DEBUG "status": "success", 18:15:59 DEBUG --- http response --- 18:15:59 DEBUG "status": "success", 18:15:59 DEBUG "status": "success", 18:15:59 DEBUG ----- output ----- 18:15:59 DEBUG "result": [ 18:15:59 DEBUG ----- output ----- 18:15:59 DEBUG ----- output ----- 18:15:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:15:59 DEBUG "data": { 18:15:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:15:59 DEBUG "data": { 18:15:59 DEBUG "data": { 18:15:59 DEBUG { 18:15:59 DEBUG { 18:15:59 DEBUG { 18:15:59 DEBUG { 18:15:59 DEBUG ----- output ----- 18:15:59 DEBUG "resultType": "vector", 18:15:59 DEBUG ----- output ----- 18:15:59 DEBUG "resultType": "vector", 18:15:59 DEBUG "resultType": "vector", 18:15:59 DEBUG "status": "success", 18:15:59 DEBUG "metric": { 18:15:59 DEBUG "status": "success", 18:15:59 DEBUG "status": "success", 18:15:59 DEBUG { 18:15:59 DEBUG "result": [ 18:15:59 DEBUG { 18:15:59 DEBUG "result": [ 18:15:59 DEBUG "result": [ 18:15:59 DEBUG "data": { 18:15:59 DEBUG "pod": "ds-cts-0" 18:15:59 DEBUG "data": { 18:15:59 DEBUG "data": { 18:15:59 DEBUG "status": "success", 18:15:59 DEBUG { 18:15:59 DEBUG "status": "success", 18:15:59 DEBUG { 18:15:59 DEBUG { 18:15:59 DEBUG "resultType": "vector", 18:15:59 DEBUG }, 18:15:59 DEBUG "resultType": "vector", 18:15:59 DEBUG "resultType": "vector", 18:15:59 DEBUG "data": { 18:15:59 DEBUG "metric": { 18:15:59 DEBUG "data": { 18:15:59 DEBUG "metric": { 18:15:59 DEBUG "metric": { 18:15:59 DEBUG "result": [ 18:15:59 DEBUG "value": [ 18:15:59 DEBUG "result": [ 18:15:59 DEBUG "result": [ 18:15:59 DEBUG "resultType": "vector", 18:15:59 DEBUG "instance": "10.142.0.11:9100" 18:15:59 DEBUG "resultType": "vector", 18:15:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:15:59 DEBUG "instance": "10.142.0.11:9100" 18:15:59 DEBUG { 18:15:59 DEBUG 18:15:59 DEBUG { 18:15:59 DEBUG { 18:15:59 DEBUG "result": [ 18:15:59 DEBUG }, 18:15:59 DEBUG "result": [ 18:15:59 DEBUG }, 18:15:59 DEBUG }, 18:15:59 DEBUG "metric": { 18:15:59 DEBUG 18:15:59 DEBUG "metric": { 18:15:59 DEBUG "metric": { 18:15:59 DEBUG { 18:15:59 DEBUG "value": [ 18:15:59 DEBUG { 18:15:59 DEBUG "value": [ 18:15:59 DEBUG "value": [ 18:15:59 DEBUG "instance": "10.142.0.11:9100" 18:15:59 DEBUG 18:15:59 DEBUG "instance": "10.142.0.11:9100" 18:15:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:15:59 DEBUG "metric": { 18:15:59 DEBUG 18:15:59 DEBUG "metric": { 18:15:59 DEBUG 18:15:59 DEBUG 18:15:59 DEBUG 18:15:59 DEBUG [...] 18:15:59 DEBUG 18:15:59 DEBUG }, 18:15:59 DEBUG "instance": "10.142.0.11:9100" 18:15:59 DEBUG 18:15:59 DEBUG "pod": "ds-cts-0" 18:15:59 DEBUG 18:15:59 DEBUG }, 18:15:59 DEBUG [...] 18:15:59 DEBUG }, 18:15:59 DEBUG 18:15:59 DEBUG "value": [ 18:15:59 DEBUG }, 18:15:59 DEBUG 18:15:59 DEBUG }, 18:15:59 DEBUG 18:15:59 DEBUG "value": [ 18:15:59 DEBUG 18:15:59 DEBUG "value": [ 18:15:59 DEBUG 18:15:59 DEBUG 18:15:59 DEBUG "value": [ 18:15:59 DEBUG [...] 18:15:59 DEBUG "value": [ 18:15:59 DEBUG 18:15:59 DEBUG 18:15:59 DEBUG 18:15:59 DEBUG 18:15:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:15:59 DEBUG 18:15:59 DEBUG 18:15:59 DEBUG 18:15:59 DEBUG 18:15:59 DEBUG [...] 18:15:59 DEBUG 18:15:59 DEBUG 18:15:59 DEBUG 18:15:59 DEBUG }, 18:15:59 DEBUG 18:15:59 DEBUG 18:15:59 DEBUG 18:15:59 DEBUG 18:15:59 DEBUG 18:15:59 DEBUG 18:15:59 DEBUG 18:15:59 DEBUG "instance": "10.142.0.51:9100" 18:15:59 DEBUG }, 18:15:59 DEBUG "value": [ 18:15:59 DEBUG 1734459359, 18:15:59 DEBUG "125550.73333333334" 18:15:59 DEBUG ] 18:15:59 DEBUG } 18:15:59 DEBUG ] 18:15:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:15:59 DEBUG "value": [ 18:15:59 DEBUG [...] 18:15:59 DEBUG 18:15:59 DEBUG 18:15:59 DEBUG 18:15:59 DEBUG 18:15:59 DEBUG } 18:15:59 DEBUG 18:15:59 DEBUG }, 18:15:59 DEBUG 1734459359, 18:15:59 DEBUG 18:15:59 DEBUG [...] 18:15:59 DEBUG "instance": "10.142.0.51:9100" 18:15:59 DEBUG [...] 18:15:59 DEBUG [...] 18:15:59 DEBUG } 18:15:59 DEBUG [...] 18:15:59 DEBUG "value": [ 18:15:59 DEBUG "1.064956680715226" 18:15:59 DEBUG 18:15:59 DEBUG 18:15:59 DEBUG }, 18:15:59 DEBUG 18:15:59 DEBUG 18:15:59 DEBUG 18:15:59 DEBUG 1734459359, 18:15:59 DEBUG ] 18:15:59 DEBUG 18:15:59 DEBUG 18:15:59 DEBUG "value": [ 18:15:59 DEBUG 18:15:59 DEBUG 18:15:59 DEBUG 18:15:59 DEBUG "708677632" 18:15:59 DEBUG } 18:15:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:15:59 DEBUG 18:15:59 DEBUG 1734459359, 18:15:59 DEBUG 18:15:59 DEBUG 18:15:59 DEBUG 18:15:59 DEBUG ] 18:15:59 DEBUG ] 18:15:59 DEBUG }, 18:15:59 DEBUG "instance": "10.142.0.51:9100" 18:15:59 DEBUG "129675.9" 18:15:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:15:59 DEBUG "instance": "10.142.0.51:9100" 18:15:59 DEBUG "instance": "10.142.0.51:9100" 18:15:59 DEBUG } 18:15:59 DEBUG } 18:15:59 DEBUG "value": [ 18:15:59 DEBUG }, 18:15:59 DEBUG ] 18:15:59 DEBUG }, 18:15:59 DEBUG }, 18:15:59 DEBUG }, 18:15:59 DEBUG ] 18:15:59 DEBUG } 18:15:59 DEBUG 1734459359, 18:15:59 DEBUG "value": [ 18:15:59 DEBUG } 18:15:59 DEBUG "value": [ 18:15:59 DEBUG "value": [ 18:15:59 DEBUG "value": [ 18:15:59 DEBUG } 18:15:59 DEBUG "2851270656" 18:15:59 DEBUG 1734459359, 18:15:59 DEBUG ] 18:15:59 DEBUG 1734459359, 18:15:59 DEBUG 1734459359, 18:15:59 DEBUG 1734459359, 18:15:59 DEBUG } 18:15:59 DEBUG ] 18:15:59 DEBUG "0.0014333333333335456" 18:15:59 DEBUG } 18:15:59 DEBUG "1305.6250271821996" 18:15:59 DEBUG "0.0013333333333331865" 18:15:59 DEBUG "32904.53333333333" 18:15:59 DEBUG } 18:15:59 DEBUG ] 18:15:59 DEBUG } 18:15:59 DEBUG ] 18:15:59 DEBUG } 18:15:59 DEBUG ] 18:15:59 DEBUG ] 18:15:59 DEBUG } 18:15:59 DEBUG ] 18:15:59 DEBUG ] 18:15:59 DEBUG } 18:15:59 DEBUG } 18:15:59 DEBUG } 18:15:59 DEBUG } 18:15:59 DEBUG } 18:15:59 DEBUG ] 18:15:59 DEBUG ] 18:15:59 DEBUG ] 18:15:59 DEBUG } 18:15:59 DEBUG } 18:15:59 DEBUG } 18:15:59 DEBUG } 18:15:59 DEBUG } 18:15:59 DEBUG } 18:15:59 DEBUG } 18:15:59 INFO 18:15:59 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:15:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:15:59 INFO 18:15:59 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:15:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:15:59 INFO [loop_until]: OK (rc = 0) 18:15:59 DEBUG --- stdout --- 18:15:59 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 8m 483Mi ds-idrepo-0 7m 701Mi rcs-5d696ff5b-7ttvr 3m 296Mi rcs-5d696ff5b-wwlbr 2m 337Mi 18:15:59 DEBUG --- stderr --- 18:15:59 DEBUG 18:15:59 INFO [loop_until]: OK (rc = 0) 18:15:59 DEBUG --- stdout --- 18:15:59 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 100m 0% 2266Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1131m 3% 3739Mi 13% 18:15:59 DEBUG --- stderr --- 18:15:59 DEBUG 18:16:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459418" 18:16:58 INFO [http_cmd]: http status code OK 18:16:58 DEBUG --- status code --- 18:16:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:16:58 DEBUG --- http response --- 18:16:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:16:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459418" 18:16:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734459418" 18:16:58 INFO [http_cmd]: http status code OK 18:16:58 INFO [http_cmd]: http status code OK 18:16:58 DEBUG --- status code --- 18:16:58 DEBUG --- status code --- 18:16:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:16:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:16:58 DEBUG --- http response --- 18:16:58 DEBUG --- http response --- 18:16:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:16:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459419" 18:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459419" 18:16:59 INFO [http_cmd]: http status code OK 18:16:59 DEBUG --- status code --- 18:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:16:59 DEBUG --- http response --- 18:16:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:16:59 INFO [http_cmd]: http status code OK 18:16:59 DEBUG --- status code --- 18:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:16:59 DEBUG --- http response --- 18:16:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459419" 18:16:59 INFO [http_cmd]: http status code OK 18:16:59 DEBUG --- status code --- 18:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:16:59 DEBUG --- http response --- 18:16:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459419" 18:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459419" 18:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459419" 18:16:59 INFO [http_cmd]: http status code OK 18:16:59 DEBUG --- status code --- 18:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:16:59 DEBUG --- http response --- 18:16:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:16:59 INFO [http_cmd]: http status code OK 18:16:59 DEBUG --- status code --- 18:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:16:59 DEBUG --- http response --- 18:16:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:16:59 INFO [http_cmd]: http status code OK 18:16:59 DEBUG --- status code --- 18:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:16:59 DEBUG --- http response --- 18:16:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459419" 18:16:59 INFO [http_cmd]: http status code OK 18:16:59 DEBUG --- status code --- 18:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:16:59 DEBUG --- http response --- 18:16:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459419" 18:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459419" 18:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734459419" 18:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459419" 18:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734459419" 18:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459419" 18:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459419" 18:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459419" 18:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459419" 18:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459419" 18:16:59 INFO [http_cmd]: http status code OK 18:16:59 DEBUG --- status code --- 18:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:16:59 DEBUG --- http response --- 18:16:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:16:59 INFO [http_cmd]: http status code OK 18:16:59 DEBUG --- status code --- 18:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:16:59 INFO [http_cmd]: http status code OK 18:16:59 DEBUG --- http response --- 18:16:59 DEBUG --- status code --- 18:16:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:16:59 DEBUG --- http response --- 18:16:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:16:59 INFO [http_cmd]: http status code OK 18:16:59 DEBUG --- status code --- 18:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:16:59 DEBUG --- http response --- 18:16:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:16:59 INFO [http_cmd]: http status code OK 18:16:59 DEBUG --- status code --- 18:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:16:59 DEBUG --- http response --- 18:16:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:16:59 INFO [http_cmd]: http status code OK 18:16:59 DEBUG --- status code --- 18:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:16:59 DEBUG --- http response --- 18:16:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:16:59 INFO [http_cmd]: http status code OK 18:16:59 INFO [http_cmd]: http status code OK 18:16:59 DEBUG --- status code --- 18:16:59 DEBUG --- status code --- 18:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:16:59 DEBUG --- http response --- 18:16:59 DEBUG --- http response --- 18:16:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:16:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:16:59 INFO [http_cmd]: http status code OK 18:16:59 INFO [http_cmd]: http status code OK 18:16:59 DEBUG --- status code --- 18:16:59 DEBUG --- status code --- 18:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:16:59 DEBUG --- http response --- 18:16:59 DEBUG --- http response --- 18:16:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:16:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734459419" 18:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459419" 18:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459419" 18:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459419" 18:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734459419" 18:16:59 ERROR [http_cmd]: ERROR 18:16:59 ERROR --- status code --- 18:16:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:16:59 ERROR --- http response --- 18:16:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:16:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459419" 18:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459419" 18:16:59 INFO [http_cmd]: http status code OK 18:16:59 DEBUG --- status code --- 18:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:16:59 DEBUG --- http response --- 18:16:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:16:59 INFO [http_cmd]: http status code OK 18:16:59 INFO [http_cmd]: http status code OK 18:16:59 DEBUG --- status code --- 18:16:59 INFO [http_cmd]: http status code OK 18:16:59 INFO [http_cmd]: http status code OK 18:16:59 INFO [http_cmd]: http status code OK 18:16:59 DEBUG --- status code --- 18:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:16:59 DEBUG --- http response --- 18:16:59 DEBUG --- status code --- 18:16:59 DEBUG --- status code --- 18:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:16:59 DEBUG --- status code --- 18:16:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:16:59 DEBUG --- http response --- 18:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:16:59 DEBUG --- http response --- 18:16:59 DEBUG --- http response --- 18:16:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:16:59 DEBUG --- http response --- 18:16:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:16:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:16:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459419" 18:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459419" 18:16:59 INFO [http_cmd]: http status code OK 18:16:59 DEBUG --- status code --- 18:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:16:59 DEBUG --- http response --- 18:16:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:16:59 DEBUG ----- output ----- 18:16:59 DEBUG { 18:16:59 DEBUG "status": "success", 18:16:59 DEBUG "data": { 18:16:59 DEBUG "resultType": "vector", 18:16:59 DEBUG "result": [ 18:16:59 DEBUG { 18:16:59 DEBUG "metric": { 18:16:59 DEBUG "pod": "ds-cts-0" 18:16:59 DEBUG }, 18:16:59 DEBUG "value": [ 18:16:59 DEBUG 18:16:59 DEBUG 18:16:59 DEBUG 18:16:59 DEBUG [...] 18:16:59 DEBUG 18:16:59 DEBUG 18:16:59 DEBUG 18:16:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:16:59 DEBUG }, 18:16:59 DEBUG "value": [ 18:16:59 DEBUG 1734459419, 18:16:59 DEBUG "0" 18:16:59 DEBUG ] 18:16:59 DEBUG } 18:16:59 DEBUG ] 18:16:59 DEBUG } 18:16:59 DEBUG } 18:16:59 INFO [http_cmd]: http status code OK 18:16:59 DEBUG --- status code --- 18:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:16:59 DEBUG --- http response --- 18:16:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:16:59 DEBUG ----- output ----- 18:16:59 DEBUG { 18:16:59 DEBUG "status": "success", 18:16:59 DEBUG "data": { 18:16:59 DEBUG "resultType": "vector", 18:16:59 DEBUG "result": [ 18:16:59 DEBUG { 18:16:59 DEBUG "metric": { 18:16:59 DEBUG "pod": "ds-cts-0" 18:16:59 DEBUG }, 18:16:59 DEBUG "value": [ 18:16:59 DEBUG 18:16:59 DEBUG 18:16:59 DEBUG 18:16:59 DEBUG [...] 18:16:59 DEBUG 18:16:59 DEBUG 18:16:59 DEBUG 18:16:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:16:59 DEBUG }, 18:16:59 DEBUG "value": [ 18:16:59 DEBUG 1734459419, 18:16:59 DEBUG "0.00321756923460951" 18:16:59 DEBUG ] 18:16:59 DEBUG } 18:16:59 DEBUG ] 18:16:59 DEBUG } 18:16:59 DEBUG } 18:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459419" 18:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734459419" 18:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459419" 18:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459419" 18:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734459419" 18:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459419" 18:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459419" 18:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459419" 18:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459419" 18:16:59 INFO [http_cmd]: http status code OK 18:16:59 DEBUG --- status code --- 18:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:16:59 DEBUG --- http response --- 18:16:59 INFO [http_cmd]: http status code OK 18:16:59 DEBUG --- status code --- 18:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:16:59 DEBUG --- http response --- 18:16:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:16:59 DEBUG ----- output ----- 18:16:59 DEBUG { 18:16:59 DEBUG "status": "success", 18:16:59 DEBUG "data": { 18:16:59 DEBUG "resultType": "vector", 18:16:59 DEBUG "result": [ 18:16:59 DEBUG { 18:16:59 DEBUG "metric": { 18:16:59 DEBUG "instance": "10.142.0.11:9100" 18:16:59 DEBUG }, 18:16:59 DEBUG "value": [ 18:16:59 DEBUG 18:16:59 DEBUG 18:16:59 DEBUG 18:16:59 DEBUG [...] 18:16:59 DEBUG 18:16:59 DEBUG 18:16:59 DEBUG 18:16:59 DEBUG "instance": "10.142.0.51:9100" 18:16:59 DEBUG }, 18:16:59 DEBUG "value": [ 18:16:59 DEBUG 1734459419, 18:16:59 DEBUG "124432.76666666668" 18:16:59 DEBUG ] 18:16:59 DEBUG } 18:16:59 DEBUG ] 18:16:59 INFO [http_cmd]: http status code OK 18:16:59 INFO [http_cmd]: http status code OK 18:16:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:16:59 DEBUG } 18:16:59 DEBUG --- status code --- 18:16:59 DEBUG --- status code --- 18:16:59 INFO [http_cmd]: http status code OK 18:16:59 INFO [http_cmd]: http status code OK 18:16:59 INFO [http_cmd]: http status code OK 18:16:59 INFO [http_cmd]: http status code OK 18:16:59 DEBUG ----- output ----- 18:16:59 DEBUG } 18:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:16:59 DEBUG --- status code --- 18:16:59 DEBUG --- status code --- 18:16:59 DEBUG --- status code --- 18:16:59 DEBUG --- status code --- 18:16:59 DEBUG { 18:16:59 DEBUG --- http response --- 18:16:59 INFO [http_cmd]: http status code OK 18:16:59 DEBUG --- http response --- 18:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:16:59 DEBUG "status": "success", 18:16:59 DEBUG --- status code --- 18:16:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:16:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:16:59 DEBUG ----- output ----- 18:16:59 DEBUG --- http response --- 18:16:59 DEBUG --- http response --- 18:16:59 DEBUG --- http response --- 18:16:59 DEBUG "data": { 18:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:16:59 DEBUG ----- output ----- 18:16:59 DEBUG --- http response --- 18:16:59 DEBUG { 18:16:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:16:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:16:59 DEBUG "resultType": "vector", 18:16:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:16:59 DEBUG --- http response --- 18:16:59 DEBUG { 18:16:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:16:59 DEBUG "status": "success", 18:16:59 DEBUG ----- output ----- 18:16:59 DEBUG ----- output ----- 18:16:59 DEBUG "result": [ 18:16:59 DEBUG ----- output ----- 18:16:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:16:59 DEBUG "status": "success", 18:16:59 DEBUG ----- output ----- 18:16:59 DEBUG "data": { 18:16:59 DEBUG { 18:16:59 DEBUG { 18:16:59 DEBUG { 18:16:59 DEBUG { 18:16:59 DEBUG ----- output ----- 18:16:59 DEBUG "data": { 18:16:59 DEBUG { 18:16:59 DEBUG "resultType": "vector", 18:16:59 DEBUG "status": "success", 18:16:59 DEBUG "status": "success", 18:16:59 DEBUG "metric": { 18:16:59 DEBUG "status": "success", 18:16:59 DEBUG { 18:16:59 DEBUG "resultType": "vector", 18:16:59 DEBUG "status": "success", 18:16:59 DEBUG "result": [ 18:16:59 DEBUG "data": { 18:16:59 DEBUG "data": { 18:16:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:16:59 DEBUG "data": { 18:16:59 DEBUG "status": "success", 18:16:59 DEBUG "result": [ 18:16:59 DEBUG "data": { 18:16:59 DEBUG { 18:16:59 DEBUG "resultType": "vector", 18:16:59 DEBUG "resultType": "vector", 18:16:59 DEBUG }, 18:16:59 DEBUG "resultType": "vector", 18:16:59 DEBUG "data": { 18:16:59 DEBUG { 18:16:59 DEBUG "resultType": "vector", 18:16:59 DEBUG "metric": { 18:16:59 DEBUG "result": [ 18:16:59 DEBUG "result": [ 18:16:59 DEBUG "value": [ 18:16:59 DEBUG "result": [ 18:16:59 DEBUG "resultType": "vector", 18:16:59 DEBUG "metric": { 18:16:59 DEBUG "result": [ 18:16:59 DEBUG "pod": "ds-cts-0" 18:16:59 DEBUG { 18:16:59 DEBUG { 18:16:59 DEBUG 18:16:59 DEBUG { 18:16:59 DEBUG "result": [ 18:16:59 DEBUG "instance": "10.142.0.11:9100" 18:16:59 DEBUG { 18:16:59 DEBUG }, 18:16:59 DEBUG "metric": { 18:16:59 DEBUG "metric": { 18:16:59 DEBUG 18:16:59 DEBUG 18:16:59 DEBUG [...] 18:16:59 DEBUG 18:16:59 DEBUG 18:16:59 DEBUG "value": [ 18:16:59 DEBUG "instance": "10.142.0.11:9100" 18:16:59 DEBUG }, 18:16:59 DEBUG "value": [ 18:16:59 DEBUG 18:16:59 DEBUG 18:16:59 DEBUG 18:16:59 DEBUG [...] 18:16:59 DEBUG 18:16:59 DEBUG 18:16:59 DEBUG 18:16:59 DEBUG "metric": { 18:16:59 DEBUG }, 18:16:59 DEBUG "metric": { 18:16:59 DEBUG 18:16:59 DEBUG 18:16:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:16:59 DEBUG { 18:16:59 DEBUG "instance": "10.142.0.51:9100" 18:16:59 DEBUG "instance": "10.142.0.11:9100" 18:16:59 DEBUG "value": [ 18:16:59 DEBUG "pod": "ds-cts-0" 18:16:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:16:59 DEBUG 18:16:59 DEBUG }, 18:16:59 DEBUG "metric": { 18:16:59 DEBUG }, 18:16:59 DEBUG }, 18:16:59 DEBUG 18:16:59 DEBUG }, 18:16:59 DEBUG }, 18:16:59 DEBUG 18:16:59 DEBUG "value": [ 18:16:59 DEBUG "instance": "10.142.0.11:9100" 18:16:59 DEBUG "value": [ 18:16:59 DEBUG "value": [ 18:16:59 DEBUG 18:16:59 DEBUG "value": [ 18:16:59 DEBUG "value": [ 18:16:59 DEBUG [...] 18:16:59 DEBUG 18:16:59 DEBUG }, 18:16:59 DEBUG 1734459419, 18:16:59 DEBUG 18:16:59 DEBUG 18:16:59 DEBUG 18:16:59 DEBUG 1734459419, 18:16:59 DEBUG 18:16:59 DEBUG 18:16:59 DEBUG "value": [ 18:16:59 DEBUG "29354.666666666668" 18:16:59 DEBUG 18:16:59 DEBUG [...] 18:16:59 DEBUG 18:16:59 DEBUG "1.061372808586071" 18:16:59 DEBUG 18:16:59 DEBUG 18:16:59 DEBUG 18:16:59 DEBUG ] 18:16:59 DEBUG 18:16:59 DEBUG 18:16:59 DEBUG 18:16:59 DEBUG ] 18:16:59 DEBUG 18:16:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:16:59 DEBUG 18:16:59 DEBUG } 18:16:59 DEBUG [...] 18:16:59 DEBUG 18:16:59 DEBUG [...] 18:16:59 DEBUG } 18:16:59 DEBUG [...] 18:16:59 DEBUG }, 18:16:59 DEBUG 18:16:59 DEBUG ] 18:16:59 DEBUG 18:16:59 DEBUG 18:16:59 DEBUG 18:16:59 DEBUG ] 18:16:59 DEBUG 18:16:59 DEBUG "value": [ 18:16:59 DEBUG [...] 18:16:59 DEBUG } 18:16:59 DEBUG 18:16:59 DEBUG "instance": "10.142.0.51:9100" 18:16:59 DEBUG 18:16:59 DEBUG } 18:16:59 DEBUG } 18:16:59 DEBUG 1734459419, 18:16:59 DEBUG 18:16:59 DEBUG } 18:16:59 DEBUG 18:16:59 DEBUG }, 18:16:59 DEBUG 18:16:59 DEBUG 18:16:59 DEBUG "708784128" 18:16:59 DEBUG 18:16:59 DEBUG "instance": "10.142.0.51:9100" 18:16:59 DEBUG "value": [ 18:16:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:16:59 DEBUG 18:16:59 DEBUG ] 18:16:59 DEBUG 18:16:59 DEBUG }, 18:16:59 DEBUG 1734459419, 18:16:59 DEBUG }, 18:16:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:16:59 DEBUG } 18:16:59 DEBUG "instance": "10.142.0.51:9100" 18:16:59 DEBUG "value": [ 18:16:59 DEBUG "120711.33333333333" 18:16:59 DEBUG "value": [ 18:16:59 DEBUG }, 18:16:59 DEBUG ] 18:16:59 DEBUG }, 18:16:59 DEBUG 1734459419, 18:16:59 DEBUG ] 18:16:59 DEBUG 1734459419, 18:16:59 DEBUG "value": [ 18:16:59 DEBUG } 18:16:59 DEBUG "value": [ 18:16:59 DEBUG 1734459419, 18:16:59 DEBUG } 18:16:59 DEBUG ] 18:16:59 DEBUG } 18:16:59 DEBUG 1734459419, 18:16:59 DEBUG "0.0006666666666667708" 18:16:59 DEBUG "0.0006333333333320942" 18:16:59 DEBUG "1202.556987677785" 18:16:59 DEBUG } 18:16:59 DEBUG } 18:16:59 DEBUG "2864848896" 18:16:59 DEBUG ] 18:16:59 DEBUG ] 18:16:59 DEBUG ] 18:16:59 DEBUG ] 18:16:59 DEBUG } 18:16:59 DEBUG } 18:16:59 DEBUG } 18:16:59 DEBUG } 18:16:59 DEBUG ] 18:16:59 DEBUG } 18:16:59 DEBUG } 18:16:59 DEBUG ] 18:16:59 DEBUG ] 18:16:59 DEBUG ] 18:16:59 DEBUG } 18:16:59 DEBUG } 18:16:59 DEBUG } 18:16:59 DEBUG } 18:16:59 DEBUG } 18:16:59 DEBUG } 18:16:59 INFO 18:16:59 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:16:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:16:59 INFO 18:16:59 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:16:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:16:59 INFO [loop_until]: OK (rc = 0) 18:16:59 DEBUG --- stdout --- 18:16:59 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 9m 487Mi ds-idrepo-0 7m 703Mi rcs-5d696ff5b-7ttvr 3m 296Mi rcs-5d696ff5b-wwlbr 1m 337Mi 18:16:59 DEBUG --- stderr --- 18:16:59 DEBUG 18:16:59 INFO [loop_until]: OK (rc = 0) 18:16:59 DEBUG --- stdout --- 18:16:59 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 114m 0% 2258Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1142m 3% 3746Mi 13% 18:16:59 DEBUG --- stderr --- 18:16:59 DEBUG 18:17:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459478" 18:17:58 INFO [http_cmd]: http status code OK 18:17:58 DEBUG --- status code --- 18:17:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:17:58 DEBUG --- http response --- 18:17:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:17:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459478" 18:17:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734459478" 18:17:58 INFO [http_cmd]: http status code OK 18:17:58 DEBUG --- status code --- 18:17:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:17:58 DEBUG --- http response --- 18:17:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:17:58 INFO [http_cmd]: http status code OK 18:17:58 DEBUG --- status code --- 18:17:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:17:58 DEBUG --- http response --- 18:17:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459479" 18:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459479" 18:17:59 INFO [http_cmd]: http status code OK 18:17:59 DEBUG --- status code --- 18:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:17:59 DEBUG --- http response --- 18:17:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:17:59 INFO [http_cmd]: http status code OK 18:17:59 DEBUG --- status code --- 18:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:17:59 DEBUG --- http response --- 18:17:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459479" 18:17:59 INFO [http_cmd]: http status code OK 18:17:59 DEBUG --- status code --- 18:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:17:59 DEBUG --- http response --- 18:17:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459479" 18:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459479" 18:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459479" 18:17:59 INFO [http_cmd]: http status code OK 18:17:59 DEBUG --- status code --- 18:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:17:59 DEBUG --- http response --- 18:17:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:17:59 INFO [http_cmd]: http status code OK 18:17:59 DEBUG --- status code --- 18:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:17:59 INFO [http_cmd]: http status code OK 18:17:59 DEBUG --- http response --- 18:17:59 DEBUG --- status code --- 18:17:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:17:59 DEBUG --- http response --- 18:17:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459479" 18:17:59 INFO [http_cmd]: http status code OK 18:17:59 DEBUG --- status code --- 18:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:17:59 DEBUG --- http response --- 18:17:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459479" 18:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734459479" 18:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459479" 18:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459479" 18:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459479" 18:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459479" 18:17:59 INFO [http_cmd]: http status code OK 18:17:59 DEBUG --- status code --- 18:17:59 INFO [http_cmd]: http status code OK 18:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:17:59 DEBUG --- status code --- 18:17:59 DEBUG --- http response --- 18:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459479" 18:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459479" 18:17:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:17:59 DEBUG --- http response --- 18:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734459479" 18:17:59 INFO [http_cmd]: http status code OK 18:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459479" 18:17:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:17:59 DEBUG --- status code --- 18:17:59 INFO [http_cmd]: http status code OK 18:17:59 INFO [http_cmd]: http status code OK 18:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:17:59 DEBUG --- status code --- 18:17:59 INFO [http_cmd]: http status code OK 18:17:59 DEBUG --- status code --- 18:17:59 DEBUG --- http response --- 18:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:17:59 DEBUG --- status code --- 18:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:17:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:17:59 DEBUG --- http response --- 18:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:17:59 DEBUG --- http response --- 18:17:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:17:59 DEBUG --- http response --- 18:17:59 INFO [http_cmd]: http status code OK 18:17:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:17:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:17:59 INFO [http_cmd]: http status code OK 18:17:59 INFO [http_cmd]: http status code OK 18:17:59 DEBUG --- status code --- 18:17:59 DEBUG --- status code --- 18:17:59 DEBUG --- status code --- 18:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:17:59 DEBUG --- http response --- 18:17:59 DEBUG --- http response --- 18:17:59 DEBUG --- http response --- 18:17:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:17:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:17:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734459479" 18:17:59 INFO [http_cmd]: http status code OK 18:17:59 DEBUG --- status code --- 18:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:17:59 DEBUG --- http response --- 18:17:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:17:59 ERROR [http_cmd]: ERROR 18:17:59 ERROR --- status code --- 18:17:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:17:59 ERROR --- http response --- 18:17:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:17:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459479" 18:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459479" 18:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734459479" 18:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459479" 18:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459479" 18:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459479" 18:17:59 INFO [http_cmd]: http status code OK 18:17:59 DEBUG --- status code --- 18:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:17:59 DEBUG --- http response --- 18:17:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:17:59 INFO [http_cmd]: http status code OK 18:17:59 DEBUG --- status code --- 18:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:17:59 INFO [http_cmd]: http status code OK 18:17:59 DEBUG --- http response --- 18:17:59 DEBUG --- status code --- 18:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:17:59 DEBUG --- http response --- 18:17:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:17:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:17:59 INFO [http_cmd]: http status code OK 18:17:59 DEBUG --- status code --- 18:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:17:59 DEBUG --- http response --- 18:17:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:17:59 INFO [http_cmd]: http status code OK 18:17:59 INFO [http_cmd]: http status code OK 18:17:59 DEBUG --- status code --- 18:17:59 DEBUG --- status code --- 18:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:17:59 DEBUG --- http response --- 18:17:59 DEBUG --- http response --- 18:17:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:17:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459479" 18:17:59 INFO [http_cmd]: http status code OK 18:17:59 DEBUG --- status code --- 18:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:17:59 DEBUG --- http response --- 18:17:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:17:59 DEBUG ----- output ----- 18:17:59 DEBUG { 18:17:59 DEBUG "status": "success", 18:17:59 DEBUG "data": { 18:17:59 DEBUG "resultType": "vector", 18:17:59 DEBUG "result": [ 18:17:59 DEBUG { 18:17:59 DEBUG "metric": { 18:17:59 DEBUG "pod": "ds-cts-0" 18:17:59 DEBUG }, 18:17:59 DEBUG "value": [ 18:17:59 DEBUG 18:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459479" 18:17:59 DEBUG 18:17:59 DEBUG 18:17:59 DEBUG [...] 18:17:59 DEBUG 18:17:59 DEBUG 18:17:59 DEBUG 18:17:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:17:59 DEBUG }, 18:17:59 DEBUG "value": [ 18:17:59 DEBUG 1734459479, 18:17:59 DEBUG "0" 18:17:59 DEBUG ] 18:17:59 DEBUG } 18:17:59 DEBUG ] 18:17:59 DEBUG } 18:17:59 DEBUG } 18:17:59 INFO [http_cmd]: http status code OK 18:17:59 DEBUG --- status code --- 18:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:17:59 DEBUG --- http response --- 18:17:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:17:59 DEBUG ----- output ----- 18:17:59 DEBUG { 18:17:59 DEBUG "status": "success", 18:17:59 DEBUG "data": { 18:17:59 DEBUG "resultType": "vector", 18:17:59 DEBUG "result": [ 18:17:59 DEBUG { 18:17:59 DEBUG "metric": { 18:17:59 DEBUG "pod": "ds-cts-0" 18:17:59 DEBUG }, 18:17:59 DEBUG "value": [ 18:17:59 DEBUG 18:17:59 DEBUG 18:17:59 DEBUG 18:17:59 DEBUG [...] 18:17:59 DEBUG 18:17:59 DEBUG 18:17:59 DEBUG 18:17:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:17:59 DEBUG }, 18:17:59 DEBUG "value": [ 18:17:59 DEBUG 1734459479, 18:17:59 DEBUG "0.002791436068707239" 18:17:59 DEBUG ] 18:17:59 DEBUG } 18:17:59 DEBUG ] 18:17:59 DEBUG } 18:17:59 DEBUG } 18:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459479" 18:17:59 INFO [http_cmd]: http status code OK 18:17:59 DEBUG --- status code --- 18:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:17:59 DEBUG --- http response --- 18:17:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:17:59 DEBUG ----- output ----- 18:17:59 DEBUG { 18:17:59 DEBUG "status": "success", 18:17:59 DEBUG "data": { 18:17:59 DEBUG "resultType": "vector", 18:17:59 DEBUG "result": [ 18:17:59 DEBUG { 18:17:59 DEBUG "metric": { 18:17:59 DEBUG "instance": "10.142.0.11:9100" 18:17:59 DEBUG }, 18:17:59 DEBUG "value": [ 18:17:59 DEBUG 18:17:59 DEBUG 18:17:59 DEBUG 18:17:59 DEBUG [...] 18:17:59 DEBUG 18:17:59 DEBUG 18:17:59 DEBUG 18:17:59 DEBUG "instance": "10.142.0.51:9100" 18:17:59 DEBUG }, 18:17:59 DEBUG "value": [ 18:17:59 DEBUG 1734459479, 18:17:59 DEBUG "131326.0333333333" 18:17:59 DEBUG ] 18:17:59 DEBUG } 18:17:59 DEBUG ] 18:17:59 DEBUG } 18:17:59 DEBUG } 18:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734459479" 18:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459479" 18:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459479" 18:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459479" 18:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734459479" 18:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459479" 18:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459479" 18:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459479" 18:17:59 INFO [http_cmd]: http status code OK 18:17:59 INFO [http_cmd]: http status code OK 18:17:59 INFO [http_cmd]: http status code OK 18:17:59 DEBUG --- status code --- 18:17:59 DEBUG --- status code --- 18:17:59 DEBUG --- status code --- 18:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:17:59 DEBUG --- http response --- 18:17:59 DEBUG --- http response --- 18:17:59 DEBUG --- http response --- 18:17:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:17:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:17:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:17:59 DEBUG ----- output ----- 18:17:59 DEBUG ----- output ----- 18:17:59 DEBUG ----- output ----- 18:17:59 DEBUG { 18:17:59 DEBUG { 18:17:59 DEBUG { 18:17:59 DEBUG "status": "success", 18:17:59 DEBUG "status": "success", 18:17:59 DEBUG "status": "success", 18:17:59 DEBUG "data": { 18:17:59 DEBUG "data": { 18:17:59 DEBUG "data": { 18:17:59 DEBUG "resultType": "vector", 18:17:59 DEBUG "resultType": "vector", 18:17:59 DEBUG "resultType": "vector", 18:17:59 DEBUG "result": [ 18:17:59 DEBUG "result": [ 18:17:59 DEBUG "result": [ 18:17:59 DEBUG { 18:17:59 DEBUG { 18:17:59 DEBUG { 18:17:59 DEBUG "metric": { 18:17:59 DEBUG "metric": { 18:17:59 DEBUG "metric": { 18:17:59 DEBUG "instance": "10.142.0.11:9100" 18:17:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:17:59 DEBUG "pod": "ds-cts-0" 18:17:59 DEBUG }, 18:17:59 DEBUG }, 18:17:59 DEBUG }, 18:17:59 DEBUG "value": [ 18:17:59 DEBUG "value": [ 18:17:59 DEBUG "value": [ 18:17:59 DEBUG 18:17:59 DEBUG 18:17:59 DEBUG 18:17:59 INFO [http_cmd]: http status code OK 18:17:59 INFO [http_cmd]: http status code OK 18:17:59 DEBUG 18:17:59 DEBUG 18:17:59 DEBUG 18:17:59 DEBUG --- status code --- 18:17:59 INFO [http_cmd]: http status code OK 18:17:59 INFO [http_cmd]: http status code OK 18:17:59 DEBUG --- status code --- 18:17:59 DEBUG 18:17:59 DEBUG 18:17:59 DEBUG 18:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:17:59 DEBUG --- status code --- 18:17:59 DEBUG --- status code --- 18:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:17:59 DEBUG [...] 18:17:59 DEBUG [...] 18:17:59 DEBUG [...] 18:17:59 DEBUG --- http response --- 18:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:17:59 DEBUG --- http response --- 18:17:59 DEBUG 18:17:59 DEBUG 18:17:59 DEBUG 18:17:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:17:59 DEBUG --- http response --- 18:17:59 DEBUG --- http response --- 18:17:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:17:59 DEBUG 18:17:59 DEBUG 18:17:59 DEBUG 18:17:59 DEBUG ----- output ----- 18:17:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:17:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:17:59 DEBUG ----- output ----- 18:17:59 DEBUG 18:17:59 DEBUG 18:17:59 DEBUG 18:17:59 DEBUG { 18:17:59 DEBUG ----- output ----- 18:17:59 DEBUG ----- output ----- 18:17:59 DEBUG { 18:17:59 DEBUG "instance": "10.142.0.51:9100" 18:17:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:17:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:17:59 DEBUG "status": "success", 18:17:59 DEBUG { 18:17:59 DEBUG { 18:17:59 DEBUG "status": "success", 18:17:59 DEBUG }, 18:17:59 DEBUG }, 18:17:59 DEBUG }, 18:17:59 DEBUG "data": { 18:17:59 DEBUG "status": "success", 18:17:59 DEBUG "data": { 18:17:59 DEBUG "resultType": "vector", 18:17:59 DEBUG "value": [ 18:17:59 DEBUG "value": [ 18:17:59 DEBUG "value": [ 18:17:59 DEBUG "resultType": "vector", 18:17:59 DEBUG "status": "success", 18:17:59 DEBUG "data": { 18:17:59 DEBUG "result": [ 18:17:59 DEBUG 1734459479, 18:17:59 DEBUG 1734459479, 18:17:59 DEBUG 1734459479, 18:17:59 DEBUG "result": [ 18:17:59 DEBUG "data": { 18:17:59 DEBUG "resultType": "vector", 18:17:59 DEBUG { 18:17:59 DEBUG "31129.6" 18:17:59 DEBUG "1.0672896965782783" 18:17:59 DEBUG "708866048" 18:17:59 DEBUG { 18:17:59 DEBUG "resultType": "vector", 18:17:59 DEBUG "result": [ 18:17:59 DEBUG "metric": { 18:17:59 DEBUG ] 18:17:59 DEBUG ] 18:17:59 DEBUG ] 18:17:59 DEBUG "metric": { 18:17:59 DEBUG "result": [ 18:17:59 DEBUG { 18:17:59 DEBUG "instance": "10.142.0.11:9100" 18:17:59 DEBUG } 18:17:59 DEBUG } 18:17:59 DEBUG } 18:17:59 DEBUG "instance": "10.142.0.11:9100" 18:17:59 DEBUG { 18:17:59 DEBUG "metric": { 18:17:59 DEBUG }, 18:17:59 DEBUG ] 18:17:59 DEBUG ] 18:17:59 DEBUG ] 18:17:59 DEBUG }, 18:17:59 DEBUG "metric": { 18:17:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:17:59 DEBUG "value": [ 18:17:59 DEBUG } 18:17:59 DEBUG } 18:17:59 DEBUG } 18:17:59 DEBUG "value": [ 18:17:59 DEBUG "instance": "10.142.0.11:9100" 18:17:59 DEBUG }, 18:17:59 DEBUG 18:17:59 DEBUG } 18:17:59 DEBUG } 18:17:59 DEBUG } 18:17:59 DEBUG 18:17:59 DEBUG }, 18:17:59 DEBUG "value": [ 18:17:59 DEBUG 18:17:59 DEBUG 18:17:59 INFO [http_cmd]: http status code OK 18:17:59 DEBUG "value": [ 18:17:59 DEBUG 18:17:59 DEBUG 18:17:59 DEBUG 18:17:59 DEBUG --- status code --- 18:17:59 DEBUG 18:17:59 DEBUG 18:17:59 DEBUG [...] 18:17:59 DEBUG [...] 18:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:17:59 DEBUG 18:17:59 DEBUG 18:17:59 DEBUG 18:17:59 DEBUG 18:17:59 DEBUG --- http response --- 18:17:59 DEBUG 18:17:59 DEBUG [...] 18:17:59 DEBUG 18:17:59 DEBUG 18:17:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:17:59 DEBUG [...] 18:17:59 DEBUG 18:17:59 DEBUG 18:17:59 DEBUG 18:17:59 DEBUG ----- output ----- 18:17:59 DEBUG 18:17:59 DEBUG 18:17:59 DEBUG "instance": "10.142.0.51:9100" 18:17:59 DEBUG "instance": "10.142.0.51:9100" 18:17:59 DEBUG { 18:17:59 DEBUG 18:17:59 DEBUG 18:17:59 DEBUG }, 18:17:59 DEBUG }, 18:17:59 DEBUG "status": "success", 18:17:59 DEBUG 18:17:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:17:59 DEBUG "value": [ 18:17:59 DEBUG "value": [ 18:17:59 DEBUG "data": { 18:17:59 DEBUG "instance": "10.142.0.51:9100" 18:17:59 DEBUG }, 18:17:59 DEBUG 1734459479, 18:17:59 DEBUG 1734459479, 18:17:59 DEBUG "resultType": "vector", 18:17:59 DEBUG }, 18:17:59 DEBUG "value": [ 18:17:59 DEBUG "0.0003333333333333854" 18:17:59 DEBUG "0.0005333333333344588" 18:17:59 DEBUG "result": [ 18:17:59 DEBUG "value": [ 18:17:59 DEBUG 1734459479, 18:17:59 DEBUG ] 18:17:59 DEBUG ] 18:17:59 DEBUG { 18:17:59 DEBUG 1734459479, 18:17:59 DEBUG "2873303040" 18:17:59 DEBUG } 18:17:59 DEBUG } 18:17:59 DEBUG "metric": { 18:17:59 DEBUG "127780.00000000001" 18:17:59 DEBUG ] 18:17:59 DEBUG ] 18:17:59 DEBUG ] 18:17:59 DEBUG "pod": "ds-cts-0" 18:17:59 DEBUG ] 18:17:59 DEBUG } 18:17:59 DEBUG } 18:17:59 DEBUG } 18:17:59 DEBUG }, 18:17:59 DEBUG } 18:17:59 DEBUG ] 18:17:59 DEBUG } 18:17:59 DEBUG } 18:17:59 DEBUG "value": [ 18:17:59 DEBUG ] 18:17:59 DEBUG } 18:17:59 DEBUG 18:17:59 DEBUG } 18:17:59 DEBUG } 18:17:59 DEBUG 18:17:59 DEBUG 18:17:59 DEBUG } 18:17:59 DEBUG [...] 18:17:59 DEBUG 18:17:59 DEBUG 18:17:59 DEBUG 18:17:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:17:59 DEBUG }, 18:17:59 DEBUG "value": [ 18:17:59 DEBUG 1734459479, 18:17:59 DEBUG "1291.345959622845" 18:17:59 DEBUG ] 18:17:59 DEBUG } 18:17:59 DEBUG ] 18:17:59 DEBUG } 18:17:59 DEBUG } 18:17:59 INFO 18:17:59 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:17:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:17:59 INFO [loop_until]: OK (rc = 0) 18:17:59 DEBUG --- stdout --- 18:17:59 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 11m 491Mi ds-idrepo-0 7m 704Mi rcs-5d696ff5b-7ttvr 3m 296Mi rcs-5d696ff5b-wwlbr 2m 337Mi 18:17:59 DEBUG --- stderr --- 18:17:59 DEBUG 18:17:59 INFO 18:17:59 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:17:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:17:59 INFO [loop_until]: OK (rc = 0) 18:17:59 DEBUG --- stdout --- 18:17:59 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 102m 0% 2266Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1136m 3% 3752Mi 13% 18:17:59 DEBUG --- stderr --- 18:17:59 DEBUG 18:18:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459538" 18:18:58 INFO [http_cmd]: http status code OK 18:18:58 DEBUG --- status code --- 18:18:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:18:58 DEBUG --- http response --- 18:18:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:18:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734459538" 18:18:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459538" 18:18:58 INFO [http_cmd]: http status code OK 18:18:58 DEBUG --- status code --- 18:18:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:18:58 DEBUG --- http response --- 18:18:58 INFO [http_cmd]: http status code OK 18:18:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:18:58 DEBUG --- status code --- 18:18:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:18:58 DEBUG --- http response --- 18:18:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459539" 18:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459539" 18:18:59 INFO [http_cmd]: http status code OK 18:18:59 DEBUG --- status code --- 18:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:18:59 DEBUG --- http response --- 18:18:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:18:59 INFO [http_cmd]: http status code OK 18:18:59 DEBUG --- status code --- 18:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:18:59 DEBUG --- http response --- 18:18:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459539" 18:18:59 INFO [http_cmd]: http status code OK 18:18:59 DEBUG --- status code --- 18:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:18:59 DEBUG --- http response --- 18:18:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459539" 18:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459539" 18:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459539" 18:18:59 INFO [http_cmd]: http status code OK 18:18:59 DEBUG --- status code --- 18:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:18:59 DEBUG --- http response --- 18:18:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:18:59 INFO [http_cmd]: http status code OK 18:18:59 INFO [http_cmd]: http status code OK 18:18:59 DEBUG --- status code --- 18:18:59 DEBUG --- status code --- 18:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:18:59 DEBUG --- http response --- 18:18:59 DEBUG --- http response --- 18:18:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:18:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459539" 18:18:59 INFO [http_cmd]: http status code OK 18:18:59 DEBUG --- status code --- 18:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:18:59 DEBUG --- http response --- 18:18:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459539" 18:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459539" 18:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734459539" 18:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459539" 18:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459539" 18:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459539" 18:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459539" 18:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734459539" 18:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459539" 18:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459539" 18:18:59 INFO [http_cmd]: http status code OK 18:18:59 DEBUG --- status code --- 18:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:18:59 DEBUG --- http response --- 18:18:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:18:59 INFO [http_cmd]: http status code OK 18:18:59 DEBUG --- status code --- 18:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:18:59 DEBUG --- http response --- 18:18:59 INFO [http_cmd]: http status code OK 18:18:59 DEBUG --- status code --- 18:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:18:59 DEBUG --- http response --- 18:18:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:18:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734459539" 18:18:59 INFO [http_cmd]: http status code OK 18:18:59 DEBUG --- status code --- 18:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:18:59 DEBUG --- http response --- 18:18:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:18:59 INFO [http_cmd]: http status code OK 18:18:59 DEBUG --- status code --- 18:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:18:59 DEBUG --- http response --- 18:18:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:18:59 INFO [http_cmd]: http status code OK 18:18:59 DEBUG --- status code --- 18:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:18:59 DEBUG --- http response --- 18:18:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:18:59 INFO [http_cmd]: http status code OK 18:18:59 DEBUG --- status code --- 18:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:18:59 DEBUG --- http response --- 18:18:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:18:59 INFO [http_cmd]: http status code OK 18:18:59 DEBUG --- status code --- 18:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:18:59 DEBUG --- http response --- 18:18:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:18:59 INFO [http_cmd]: http status code OK 18:18:59 DEBUG --- status code --- 18:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:18:59 DEBUG --- http response --- 18:18:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:18:59 INFO [http_cmd]: http status code OK 18:18:59 DEBUG --- status code --- 18:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:18:59 DEBUG --- http response --- 18:18:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:18:59 ERROR [http_cmd]: ERROR 18:18:59 ERROR --- status code --- 18:18:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:18:59 ERROR --- http response --- 18:18:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:18:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459539" 18:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734459539" 18:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459539" 18:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459539" 18:18:59 INFO [http_cmd]: http status code OK 18:18:59 DEBUG --- status code --- 18:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:18:59 DEBUG --- http response --- 18:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459539" 18:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459539" 18:18:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:18:59 INFO [http_cmd]: http status code OK 18:18:59 INFO [http_cmd]: http status code OK 18:18:59 DEBUG --- status code --- 18:18:59 DEBUG --- status code --- 18:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:18:59 DEBUG --- http response --- 18:18:59 DEBUG --- http response --- 18:18:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:18:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:18:59 INFO [http_cmd]: http status code OK 18:18:59 DEBUG --- status code --- 18:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:18:59 DEBUG --- http response --- 18:18:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:18:59 INFO [http_cmd]: http status code OK 18:18:59 DEBUG --- status code --- 18:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:18:59 DEBUG --- http response --- 18:18:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:18:59 INFO [http_cmd]: http status code OK 18:18:59 DEBUG --- status code --- 18:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:18:59 DEBUG --- http response --- 18:18:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459539" 18:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459539" 18:18:59 INFO [http_cmd]: http status code OK 18:18:59 DEBUG --- status code --- 18:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:18:59 DEBUG --- http response --- 18:18:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:18:59 DEBUG ----- output ----- 18:18:59 DEBUG { 18:18:59 DEBUG "status": "success", 18:18:59 DEBUG "data": { 18:18:59 DEBUG "resultType": "vector", 18:18:59 DEBUG "result": [ 18:18:59 DEBUG { 18:18:59 DEBUG "metric": { 18:18:59 DEBUG "pod": "ds-cts-0" 18:18:59 DEBUG }, 18:18:59 DEBUG "value": [ 18:18:59 DEBUG 18:18:59 DEBUG 18:18:59 DEBUG 18:18:59 DEBUG [...] 18:18:59 DEBUG 18:18:59 DEBUG 18:18:59 DEBUG 18:18:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:18:59 DEBUG }, 18:18:59 DEBUG "value": [ 18:18:59 DEBUG 1734459539, 18:18:59 DEBUG "0" 18:18:59 DEBUG ] 18:18:59 DEBUG } 18:18:59 DEBUG ] 18:18:59 DEBUG } 18:18:59 DEBUG } 18:18:59 INFO [http_cmd]: http status code OK 18:18:59 DEBUG --- status code --- 18:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:18:59 DEBUG --- http response --- 18:18:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:18:59 DEBUG ----- output ----- 18:18:59 DEBUG { 18:18:59 DEBUG "status": "success", 18:18:59 DEBUG "data": { 18:18:59 DEBUG "resultType": "vector", 18:18:59 DEBUG "result": [ 18:18:59 DEBUG { 18:18:59 DEBUG "metric": { 18:18:59 DEBUG "pod": "ds-cts-0" 18:18:59 DEBUG }, 18:18:59 DEBUG "value": [ 18:18:59 DEBUG 18:18:59 DEBUG 18:18:59 DEBUG 18:18:59 DEBUG [...] 18:18:59 DEBUG 18:18:59 DEBUG 18:18:59 DEBUG 18:18:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:18:59 DEBUG }, 18:18:59 DEBUG "value": [ 18:18:59 DEBUG 1734459539, 18:18:59 DEBUG "0.0037803363067347804" 18:18:59 DEBUG ] 18:18:59 DEBUG } 18:18:59 DEBUG ] 18:18:59 DEBUG } 18:18:59 DEBUG } 18:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459539" 18:18:59 INFO [http_cmd]: http status code OK 18:18:59 DEBUG --- status code --- 18:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:18:59 DEBUG --- http response --- 18:18:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:18:59 DEBUG ----- output ----- 18:18:59 DEBUG { 18:18:59 DEBUG "status": "success", 18:18:59 DEBUG "data": { 18:18:59 DEBUG "resultType": "vector", 18:18:59 DEBUG "result": [ 18:18:59 DEBUG { 18:18:59 DEBUG "metric": { 18:18:59 DEBUG "instance": "10.142.0.11:9100" 18:18:59 DEBUG }, 18:18:59 DEBUG "value": [ 18:18:59 DEBUG 18:18:59 DEBUG 18:18:59 DEBUG 18:18:59 DEBUG [...] 18:18:59 DEBUG 18:18:59 DEBUG 18:18:59 DEBUG 18:18:59 DEBUG "instance": "10.142.0.51:9100" 18:18:59 DEBUG }, 18:18:59 DEBUG "value": [ 18:18:59 DEBUG 1734459539, 18:18:59 DEBUG "124820.96666666666" 18:18:59 DEBUG ] 18:18:59 DEBUG } 18:18:59 DEBUG ] 18:18:59 DEBUG } 18:18:59 DEBUG } 18:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734459539" 18:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459539" 18:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459539" 18:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459539" 18:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459539" 18:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734459539" 18:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459539" 18:18:59 INFO [http_cmd]: http status code OK 18:18:59 DEBUG --- status code --- 18:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:18:59 DEBUG --- http response --- 18:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459539" 18:18:59 INFO [http_cmd]: http status code OK 18:18:59 DEBUG --- status code --- 18:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:18:59 DEBUG --- http response --- 18:18:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:18:59 DEBUG ----- output ----- 18:18:59 DEBUG { 18:18:59 DEBUG "status": "success", 18:18:59 DEBUG "data": { 18:18:59 DEBUG "resultType": "vector", 18:18:59 DEBUG "result": [ 18:18:59 DEBUG { 18:18:59 DEBUG "metric": { 18:18:59 DEBUG "pod": "ds-cts-0" 18:18:59 DEBUG }, 18:18:59 DEBUG "value": [ 18:18:59 DEBUG 18:18:59 DEBUG 18:18:59 DEBUG 18:18:59 DEBUG [...] 18:18:59 DEBUG 18:18:59 DEBUG 18:18:59 DEBUG 18:18:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:18:59 DEBUG }, 18:18:59 DEBUG "value": [ 18:18:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:18:59 DEBUG 1734459539, 18:18:59 DEBUG ----- output ----- 18:18:59 DEBUG "709099520" 18:18:59 DEBUG { 18:18:59 DEBUG ] 18:18:59 DEBUG "status": "success", 18:18:59 DEBUG } 18:18:59 DEBUG "data": { 18:18:59 DEBUG ] 18:18:59 DEBUG "resultType": "vector", 18:18:59 DEBUG } 18:18:59 INFO [http_cmd]: http status code OK 18:18:59 DEBUG "result": [ 18:18:59 INFO [http_cmd]: http status code OK 18:18:59 DEBUG } 18:18:59 DEBUG --- status code --- 18:18:59 DEBUG { 18:18:59 DEBUG --- status code --- 18:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:18:59 DEBUG "metric": { 18:18:59 INFO [http_cmd]: http status code OK 18:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:18:59 INFO [http_cmd]: http status code OK 18:18:59 DEBUG --- http response --- 18:18:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:18:59 DEBUG --- status code --- 18:18:59 DEBUG --- http response --- 18:18:59 DEBUG --- status code --- 18:18:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:18:59 DEBUG }, 18:18:59 DEBUG "value": [ 18:18:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:18:59 INFO [http_cmd]: http status code OK 18:18:59 DEBUG ----- output ----- 18:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:18:59 DEBUG 18:18:59 DEBUG ----- output ----- 18:18:59 DEBUG --- http response --- 18:18:59 DEBUG --- status code --- 18:18:59 DEBUG { 18:18:59 DEBUG --- http response --- 18:18:59 DEBUG 18:18:59 DEBUG { 18:18:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:18:59 DEBUG "status": "success", 18:18:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:18:59 DEBUG 18:18:59 DEBUG "status": "success", 18:18:59 INFO [http_cmd]: http status code OK 18:18:59 DEBUG ----- output ----- 18:18:59 DEBUG --- http response --- 18:18:59 DEBUG "data": { 18:18:59 DEBUG ----- output ----- 18:18:59 DEBUG [...] 18:18:59 DEBUG "data": { 18:18:59 DEBUG --- status code --- 18:18:59 DEBUG { 18:18:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:18:59 DEBUG "resultType": "vector", 18:18:59 DEBUG { 18:18:59 DEBUG 18:18:59 DEBUG 18:18:59 DEBUG 18:18:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:18:59 DEBUG ----- output ----- 18:18:59 DEBUG "result": [ 18:18:59 DEBUG "status": "success", 18:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:18:59 DEBUG "resultType": "vector", 18:18:59 DEBUG "status": "success", 18:18:59 DEBUG }, 18:18:59 DEBUG { 18:18:59 DEBUG { 18:18:59 DEBUG "data": { 18:18:59 DEBUG --- http response --- 18:18:59 DEBUG "result": [ 18:18:59 DEBUG "data": { 18:18:59 DEBUG "value": [ 18:18:59 DEBUG "status": "success", 18:18:59 DEBUG "metric": { 18:18:59 DEBUG "resultType": "vector", 18:18:59 DEBUG { 18:18:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:18:59 DEBUG "resultType": "vector", 18:18:59 DEBUG 1734459539, 18:18:59 DEBUG "data": { 18:18:59 DEBUG "instance": "10.142.0.11:9100" 18:18:59 DEBUG "result": [ 18:18:59 DEBUG "metric": { 18:18:59 DEBUG ----- output ----- 18:18:59 DEBUG "result": [ 18:18:59 DEBUG "1.0581054519975677" 18:18:59 DEBUG "resultType": "vector", 18:18:59 DEBUG }, 18:18:59 DEBUG { 18:18:59 DEBUG "instance": "10.142.0.11:9100" 18:18:59 DEBUG { 18:18:59 DEBUG { 18:18:59 DEBUG ] 18:18:59 DEBUG "result": [ 18:18:59 DEBUG "value": [ 18:18:59 DEBUG 18:18:59 DEBUG 18:18:59 DEBUG 18:18:59 DEBUG "metric": { 18:18:59 DEBUG } 18:18:59 DEBUG { 18:18:59 DEBUG "metric": { 18:18:59 DEBUG }, 18:18:59 DEBUG "status": "success", 18:18:59 DEBUG [...] 18:18:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:18:59 DEBUG ] 18:18:59 DEBUG "metric": { 18:18:59 DEBUG "instance": "10.142.0.11:9100" 18:18:59 DEBUG "value": [ 18:18:59 DEBUG "data": { 18:18:59 DEBUG 18:18:59 DEBUG }, 18:18:59 DEBUG } 18:18:59 DEBUG "instance": "10.142.0.11:9100" 18:18:59 DEBUG }, 18:18:59 DEBUG 18:18:59 DEBUG "resultType": "vector", 18:18:59 DEBUG 18:18:59 DEBUG "value": [ 18:18:59 DEBUG } 18:18:59 DEBUG }, 18:18:59 DEBUG "value": [ 18:18:59 DEBUG 18:18:59 DEBUG "result": [ 18:18:59 DEBUG 18:18:59 DEBUG 18:18:59 DEBUG "value": [ 18:18:59 DEBUG 18:18:59 DEBUG 18:18:59 DEBUG { 18:18:59 DEBUG "instance": "10.142.0.51:9100" 18:18:59 DEBUG 18:18:59 DEBUG 18:18:59 DEBUG 18:18:59 DEBUG [...] 18:18:59 DEBUG "metric": { 18:18:59 DEBUG }, 18:18:59 DEBUG 18:18:59 DEBUG 18:18:59 DEBUG 18:18:59 DEBUG 18:18:59 DEBUG "pod": "ds-cts-0" 18:18:59 DEBUG "value": [ 18:18:59 DEBUG [...] 18:18:59 DEBUG 18:18:59 DEBUG [...] 18:18:59 DEBUG 18:18:59 DEBUG }, 18:18:59 DEBUG 1734459539, 18:18:59 DEBUG 18:18:59 DEBUG [...] 18:18:59 DEBUG 18:18:59 DEBUG 18:18:59 DEBUG "value": [ 18:18:59 DEBUG "45465.6" 18:18:59 DEBUG 18:18:59 DEBUG 18:18:59 DEBUG 18:18:59 DEBUG "instance": "10.142.0.51:9100" 18:18:59 DEBUG 18:18:59 DEBUG ] 18:18:59 DEBUG 18:18:59 DEBUG 18:18:59 DEBUG 18:18:59 DEBUG }, 18:18:59 DEBUG 18:18:59 DEBUG } 18:18:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:18:59 DEBUG 18:18:59 DEBUG "instance": "10.142.0.51:9100" 18:18:59 DEBUG "value": [ 18:18:59 DEBUG 18:18:59 DEBUG ] 18:18:59 DEBUG }, 18:18:59 DEBUG "instance": "10.142.0.51:9100" 18:18:59 DEBUG }, 18:18:59 DEBUG 1734459539, 18:18:59 DEBUG [...] 18:18:59 DEBUG } 18:18:59 DEBUG "value": [ 18:18:59 DEBUG }, 18:18:59 DEBUG "value": [ 18:18:59 DEBUG "0.0006999999999995529" 18:18:59 DEBUG 18:18:59 DEBUG } 18:18:59 DEBUG 1734459539, 18:18:59 DEBUG "value": [ 18:18:59 DEBUG 1734459539, 18:18:59 DEBUG "0.0003333333333333854" 18:18:59 DEBUG 18:18:59 DEBUG "2883448832" 18:18:59 DEBUG 1734459539, 18:18:59 DEBUG ] 18:18:59 DEBUG ] 18:18:59 DEBUG 18:18:59 DEBUG ] 18:18:59 DEBUG "120524.06666666667" 18:18:59 DEBUG } 18:18:59 DEBUG } 18:18:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:18:59 DEBUG } 18:18:59 DEBUG ] 18:18:59 DEBUG ] 18:18:59 DEBUG ] 18:18:59 DEBUG }, 18:18:59 DEBUG ] 18:18:59 DEBUG } 18:18:59 DEBUG ] 18:18:59 DEBUG } 18:18:59 DEBUG } 18:18:59 DEBUG } 18:18:59 DEBUG } 18:18:59 DEBUG } 18:18:59 DEBUG "value": [ 18:18:59 DEBUG } 18:18:59 DEBUG } 18:18:59 DEBUG } 18:18:59 DEBUG 1734459539, 18:18:59 DEBUG "1082.1575839518732" 18:18:59 DEBUG ] 18:18:59 DEBUG } 18:18:59 DEBUG ] 18:18:59 DEBUG } 18:18:59 DEBUG } 18:18:59 INFO 18:18:59 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:18:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:18:59 INFO [loop_until]: OK (rc = 0) 18:18:59 DEBUG --- stdout --- 18:18:59 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 7m 494Mi ds-idrepo-0 7m 705Mi rcs-5d696ff5b-7ttvr 3m 297Mi rcs-5d696ff5b-wwlbr 2m 337Mi 18:18:59 DEBUG --- stderr --- 18:18:59 DEBUG 18:18:59 INFO 18:18:59 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:18:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:19:00 INFO [loop_until]: OK (rc = 0) 18:19:00 DEBUG --- stdout --- 18:19:00 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 98m 0% 2275Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1125m 3% 3754Mi 13% 18:19:00 DEBUG --- stderr --- 18:19:00 DEBUG 18:19:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459598" 18:19:58 INFO [http_cmd]: http status code OK 18:19:58 DEBUG --- status code --- 18:19:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:19:58 DEBUG --- http response --- 18:19:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:19:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459598" 18:19:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734459598" 18:19:58 INFO [http_cmd]: http status code OK 18:19:58 DEBUG --- status code --- 18:19:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:19:58 DEBUG --- http response --- 18:19:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:19:58 INFO [http_cmd]: http status code OK 18:19:58 DEBUG --- status code --- 18:19:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:19:58 DEBUG --- http response --- 18:19:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459599" 18:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459599" 18:19:59 INFO [http_cmd]: http status code OK 18:19:59 INFO [http_cmd]: http status code OK 18:19:59 DEBUG --- status code --- 18:19:59 DEBUG --- status code --- 18:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:19:59 DEBUG --- http response --- 18:19:59 DEBUG --- http response --- 18:19:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:19:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459599" 18:19:59 INFO [http_cmd]: http status code OK 18:19:59 DEBUG --- status code --- 18:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:19:59 DEBUG --- http response --- 18:19:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459599" 18:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459599" 18:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459599" 18:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459599" 18:19:59 INFO [http_cmd]: http status code OK 18:19:59 DEBUG --- status code --- 18:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:19:59 DEBUG --- http response --- 18:19:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:19:59 INFO [http_cmd]: http status code OK 18:19:59 DEBUG --- status code --- 18:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:19:59 DEBUG --- http response --- 18:19:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:19:59 INFO [http_cmd]: http status code OK 18:19:59 DEBUG --- status code --- 18:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:19:59 DEBUG --- http response --- 18:19:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:19:59 INFO [http_cmd]: http status code OK 18:19:59 DEBUG --- status code --- 18:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:19:59 DEBUG --- http response --- 18:19:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459599" 18:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734459599" 18:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459599" 18:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459599" 18:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459599" 18:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459599" 18:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459599" 18:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734459599" 18:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459599" 18:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734459599" 18:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459599" 18:19:59 INFO [http_cmd]: http status code OK 18:19:59 INFO [http_cmd]: http status code OK 18:19:59 DEBUG --- status code --- 18:19:59 INFO [http_cmd]: http status code OK 18:19:59 INFO [http_cmd]: http status code OK 18:19:59 INFO [http_cmd]: http status code OK 18:19:59 DEBUG --- status code --- 18:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:19:59 DEBUG --- status code --- 18:19:59 DEBUG --- status code --- 18:19:59 DEBUG --- status code --- 18:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:19:59 DEBUG --- http response --- 18:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:19:59 INFO [http_cmd]: http status code OK 18:19:59 INFO [http_cmd]: http status code OK 18:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:19:59 DEBUG --- http response --- 18:19:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:19:59 DEBUG --- http response --- 18:19:59 INFO [http_cmd]: http status code OK 18:19:59 DEBUG --- status code --- 18:19:59 ERROR [http_cmd]: ERROR 18:19:59 DEBUG --- status code --- 18:19:59 DEBUG --- http response --- 18:19:59 DEBUG --- http response --- 18:19:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:19:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:19:59 INFO [http_cmd]: http status code OK 18:19:59 DEBUG --- status code --- 18:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:19:59 ERROR --- status code --- 18:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:19:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:19:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:19:59 DEBUG --- status code --- 18:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:19:59 DEBUG --- http response --- 18:19:59 INFO [http_cmd]: http status code OK 18:19:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:19:59 DEBUG --- http response --- 18:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459599" 18:19:59 DEBUG --- http response --- 18:19:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:19:59 DEBUG --- status code --- 18:19:59 ERROR --- http response --- 18:19:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:19:59 DEBUG --- http response --- 18:19:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:19:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:19:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:19:59 DEBUG --- http response --- 18:19:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459599" 18:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734459599" 18:19:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459599" 18:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459599" 18:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459599" 18:19:59 INFO [http_cmd]: http status code OK 18:19:59 DEBUG --- status code --- 18:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:19:59 DEBUG --- http response --- 18:19:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:19:59 INFO [http_cmd]: http status code OK 18:19:59 DEBUG --- status code --- 18:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:19:59 DEBUG --- http response --- 18:19:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:19:59 INFO [http_cmd]: http status code OK 18:19:59 DEBUG --- status code --- 18:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:19:59 INFO [http_cmd]: http status code OK 18:19:59 INFO [http_cmd]: http status code OK 18:19:59 DEBUG --- http response --- 18:19:59 INFO [http_cmd]: http status code OK 18:19:59 DEBUG --- status code --- 18:19:59 DEBUG --- status code --- 18:19:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:19:59 DEBUG --- status code --- 18:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:19:59 DEBUG --- http response --- 18:19:59 DEBUG --- http response --- 18:19:59 DEBUG --- http response --- 18:19:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:19:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:19:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459599" 18:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459599" 18:19:59 INFO [http_cmd]: http status code OK 18:19:59 DEBUG --- status code --- 18:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:19:59 DEBUG --- http response --- 18:19:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:19:59 DEBUG ----- output ----- 18:19:59 DEBUG { 18:19:59 DEBUG "status": "success", 18:19:59 DEBUG "data": { 18:19:59 DEBUG "resultType": "vector", 18:19:59 DEBUG "result": [ 18:19:59 DEBUG { 18:19:59 DEBUG "metric": { 18:19:59 DEBUG "pod": "ds-cts-0" 18:19:59 DEBUG }, 18:19:59 DEBUG "value": [ 18:19:59 DEBUG 18:19:59 DEBUG 18:19:59 DEBUG 18:19:59 DEBUG [...] 18:19:59 DEBUG 18:19:59 DEBUG 18:19:59 DEBUG 18:19:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:19:59 DEBUG }, 18:19:59 DEBUG "value": [ 18:19:59 DEBUG 1734459599, 18:19:59 DEBUG "0" 18:19:59 DEBUG ] 18:19:59 DEBUG } 18:19:59 DEBUG ] 18:19:59 DEBUG } 18:19:59 DEBUG } 18:19:59 INFO [http_cmd]: http status code OK 18:19:59 DEBUG --- status code --- 18:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:19:59 DEBUG --- http response --- 18:19:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:19:59 DEBUG ----- output ----- 18:19:59 DEBUG { 18:19:59 DEBUG "status": "success", 18:19:59 DEBUG "data": { 18:19:59 DEBUG "resultType": "vector", 18:19:59 DEBUG "result": [ 18:19:59 DEBUG { 18:19:59 DEBUG "metric": { 18:19:59 DEBUG "pod": "ds-cts-0" 18:19:59 DEBUG }, 18:19:59 DEBUG "value": [ 18:19:59 DEBUG 18:19:59 DEBUG 18:19:59 DEBUG 18:19:59 DEBUG [...] 18:19:59 DEBUG 18:19:59 DEBUG 18:19:59 DEBUG 18:19:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:19:59 DEBUG }, 18:19:59 DEBUG "value": [ 18:19:59 DEBUG 1734459599, 18:19:59 DEBUG "0.007637364908038858" 18:19:59 DEBUG ] 18:19:59 DEBUG } 18:19:59 DEBUG ] 18:19:59 DEBUG } 18:19:59 DEBUG } 18:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459599" 18:19:59 INFO [http_cmd]: http status code OK 18:19:59 DEBUG --- status code --- 18:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:19:59 DEBUG --- http response --- 18:19:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:19:59 DEBUG ----- output ----- 18:19:59 DEBUG { 18:19:59 DEBUG "status": "success", 18:19:59 DEBUG "data": { 18:19:59 DEBUG "resultType": "vector", 18:19:59 DEBUG "result": [ 18:19:59 DEBUG { 18:19:59 DEBUG "metric": { 18:19:59 DEBUG "instance": "10.142.0.11:9100" 18:19:59 DEBUG }, 18:19:59 DEBUG "value": [ 18:19:59 DEBUG 18:19:59 DEBUG 18:19:59 DEBUG 18:19:59 DEBUG [...] 18:19:59 DEBUG 18:19:59 DEBUG 18:19:59 DEBUG 18:19:59 DEBUG "instance": "10.142.0.51:9100" 18:19:59 DEBUG }, 18:19:59 DEBUG "value": [ 18:19:59 DEBUG 1734459599, 18:19:59 DEBUG "128082.96666666667" 18:19:59 DEBUG ] 18:19:59 DEBUG } 18:19:59 DEBUG ] 18:19:59 DEBUG } 18:19:59 DEBUG } 18:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459599" 18:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734459599" 18:19:59 INFO [http_cmd]: http status code OK 18:19:59 DEBUG --- status code --- 18:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:19:59 DEBUG --- http response --- 18:19:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:19:59 DEBUG ----- output ----- 18:19:59 DEBUG { 18:19:59 DEBUG "status": "success", 18:19:59 DEBUG "data": { 18:19:59 DEBUG "resultType": "vector", 18:19:59 DEBUG "result": [ 18:19:59 DEBUG { 18:19:59 DEBUG "metric": { 18:19:59 DEBUG "pod": "ds-cts-0" 18:19:59 DEBUG }, 18:19:59 DEBUG "value": [ 18:19:59 DEBUG 18:19:59 DEBUG 18:19:59 DEBUG 18:19:59 DEBUG [...] 18:19:59 DEBUG 18:19:59 DEBUG 18:19:59 DEBUG 18:19:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:19:59 DEBUG }, 18:19:59 DEBUG "value": [ 18:19:59 DEBUG 1734459599, 18:19:59 DEBUG "709242880" 18:19:59 DEBUG ] 18:19:59 DEBUG } 18:19:59 DEBUG ] 18:19:59 DEBUG } 18:19:59 DEBUG } 18:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459599" 18:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459599" 18:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734459599" 18:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459599" 18:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459599" 18:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459599" 18:19:59 INFO [http_cmd]: http status code OK 18:19:59 DEBUG --- status code --- 18:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:19:59 DEBUG --- http response --- 18:19:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:19:59 DEBUG ----- output ----- 18:19:59 DEBUG { 18:19:59 DEBUG "status": "success", 18:19:59 DEBUG "data": { 18:19:59 DEBUG "resultType": "vector", 18:19:59 DEBUG "result": [ 18:19:59 DEBUG { 18:19:59 DEBUG "metric": { 18:19:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:19:59 DEBUG }, 18:19:59 DEBUG "value": [ 18:19:59 DEBUG 18:19:59 DEBUG 18:19:59 DEBUG 18:19:59 DEBUG [...] 18:19:59 DEBUG 18:19:59 DEBUG 18:19:59 DEBUG 18:19:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:19:59 DEBUG }, 18:19:59 DEBUG "value": [ 18:19:59 DEBUG 1734459599, 18:19:59 DEBUG "1.0554743312314123" 18:19:59 DEBUG ] 18:19:59 DEBUG } 18:19:59 DEBUG ] 18:19:59 DEBUG } 18:19:59 DEBUG } 18:19:59 INFO [http_cmd]: http status code OK 18:19:59 DEBUG --- status code --- 18:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:19:59 DEBUG --- http response --- 18:19:59 INFO [http_cmd]: http status code OK 18:19:59 DEBUG --- status code --- 18:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:19:59 DEBUG --- http response --- 18:19:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:19:59 DEBUG ----- output ----- 18:19:59 DEBUG { 18:19:59 DEBUG "status": "success", 18:19:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:19:59 INFO [http_cmd]: http status code OK 18:19:59 DEBUG "data": { 18:19:59 DEBUG ----- output ----- 18:19:59 DEBUG --- status code --- 18:19:59 DEBUG "resultType": "vector", 18:19:59 DEBUG { 18:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:19:59 DEBUG "result": [ 18:19:59 DEBUG "status": "success", 18:19:59 DEBUG --- http response --- 18:19:59 DEBUG { 18:19:59 DEBUG "data": { 18:19:59 INFO [http_cmd]: http status code OK 18:19:59 DEBUG "metric": { 18:19:59 INFO [http_cmd]: http status code OK 18:19:59 INFO [http_cmd]: http status code OK 18:19:59 DEBUG "resultType": "vector", 18:19:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:19:59 DEBUG --- status code --- 18:19:59 DEBUG "instance": "10.142.0.11:9100" 18:19:59 DEBUG --- status code --- 18:19:59 DEBUG --- status code --- 18:19:59 DEBUG "result": [ 18:19:59 DEBUG ----- output ----- 18:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:19:59 DEBUG }, 18:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:19:59 DEBUG --- http response --- 18:19:59 DEBUG { 18:19:59 DEBUG { 18:19:59 DEBUG --- http response --- 18:19:59 DEBUG "value": [ 18:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:19:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:19:59 DEBUG "metric": { 18:19:59 DEBUG "status": "success", 18:19:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:19:59 DEBUG 18:19:59 DEBUG --- http response --- 18:19:59 DEBUG ----- output ----- 18:19:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:19:59 DEBUG "data": { 18:19:59 DEBUG ----- output ----- 18:19:59 DEBUG 18:19:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:19:59 DEBUG { 18:19:59 DEBUG }, 18:19:59 DEBUG "resultType": "vector", 18:19:59 DEBUG { 18:19:59 DEBUG 18:19:59 DEBUG ----- output ----- 18:19:59 DEBUG "status": "success", 18:19:59 DEBUG "value": [ 18:19:59 DEBUG "result": [ 18:19:59 DEBUG "status": "success", 18:19:59 DEBUG [...] 18:19:59 DEBUG { 18:19:59 DEBUG "data": { 18:19:59 DEBUG 18:19:59 DEBUG { 18:19:59 DEBUG "data": { 18:19:59 DEBUG 18:19:59 DEBUG "status": "success", 18:19:59 DEBUG "resultType": "vector", 18:19:59 DEBUG 18:19:59 DEBUG "metric": { 18:19:59 DEBUG "resultType": "vector", 18:19:59 DEBUG 18:19:59 DEBUG "data": { 18:19:59 DEBUG "result": [ 18:19:59 DEBUG 18:19:59 DEBUG "instance": "10.142.0.11:9100" 18:19:59 DEBUG "result": [ 18:19:59 DEBUG 18:19:59 DEBUG "resultType": "vector", 18:19:59 DEBUG { 18:19:59 DEBUG [...] 18:19:59 DEBUG }, 18:19:59 DEBUG { 18:19:59 DEBUG "instance": "10.142.0.51:9100" 18:19:59 DEBUG "result": [ 18:19:59 DEBUG "metric": { 18:19:59 DEBUG 18:19:59 DEBUG "value": [ 18:19:59 DEBUG "metric": { 18:19:59 DEBUG }, 18:19:59 DEBUG { 18:19:59 DEBUG "instance": "10.142.0.11:9100" 18:19:59 DEBUG 18:19:59 DEBUG 18:19:59 DEBUG "instance": "10.142.0.11:9100" 18:19:59 DEBUG "value": [ 18:19:59 DEBUG 1734459599, 18:19:59 DEBUG }, 18:19:59 DEBUG 18:19:59 DEBUG 18:19:59 DEBUG }, 18:19:59 DEBUG "metric": { 18:19:59 DEBUG "123205.93333333333" 18:19:59 DEBUG "value": [ 18:19:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:19:59 DEBUG 18:19:59 DEBUG "value": [ 18:19:59 DEBUG "pod": "ds-cts-0" 18:19:59 DEBUG ] 18:19:59 DEBUG 18:19:59 DEBUG }, 18:19:59 DEBUG [...] 18:19:59 DEBUG 18:19:59 DEBUG }, 18:19:59 DEBUG } 18:19:59 DEBUG 18:19:59 DEBUG "value": [ 18:19:59 DEBUG 18:19:59 DEBUG 18:19:59 DEBUG "value": [ 18:19:59 DEBUG ] 18:19:59 DEBUG 18:19:59 DEBUG 1734459599, 18:19:59 DEBUG 18:19:59 DEBUG 18:19:59 DEBUG [...] 18:19:59 DEBUG 18:19:59 DEBUG [...] 18:19:59 DEBUG "2892505088" 18:19:59 DEBUG 18:19:59 DEBUG 18:19:59 DEBUG } 18:19:59 DEBUG 18:19:59 DEBUG 18:19:59 DEBUG ] 18:19:59 DEBUG "instance": "10.142.0.51:9100" 18:19:59 DEBUG 18:19:59 DEBUG } 18:19:59 DEBUG 18:19:59 DEBUG 18:19:59 DEBUG } 18:19:59 DEBUG }, 18:19:59 DEBUG 18:19:59 DEBUG [...] 18:19:59 DEBUG 18:19:59 DEBUG ] 18:19:59 DEBUG "value": [ 18:19:59 DEBUG "instance": "10.142.0.51:9100" 18:19:59 DEBUG 18:19:59 DEBUG "instance": "10.142.0.51:9100" 18:19:59 DEBUG } 18:19:59 DEBUG 1734459599, 18:19:59 DEBUG }, 18:19:59 DEBUG 18:19:59 DEBUG }, 18:19:59 DEBUG } 18:19:59 DEBUG "27306.666666666664" 18:19:59 DEBUG "value": [ 18:19:59 DEBUG 18:19:59 DEBUG "value": [ 18:19:59 DEBUG 1734459599, 18:19:59 DEBUG 1734459599, 18:19:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:19:59 DEBUG ] 18:19:59 DEBUG "0.0006666666666666524" 18:19:59 DEBUG "0.00046666666666463167" 18:19:59 DEBUG }, 18:19:59 DEBUG } 18:19:59 DEBUG ] 18:19:59 DEBUG ] 18:19:59 DEBUG "value": [ 18:19:59 DEBUG ] 18:19:59 DEBUG } 18:19:59 DEBUG } 18:19:59 DEBUG 1734459599, 18:19:59 DEBUG } 18:19:59 DEBUG ] 18:19:59 DEBUG ] 18:19:59 DEBUG "1617.0633210551346" 18:19:59 DEBUG } 18:19:59 DEBUG } 18:19:59 DEBUG } 18:19:59 DEBUG ] 18:19:59 DEBUG } 18:19:59 DEBUG } 18:19:59 DEBUG } 18:19:59 DEBUG ] 18:19:59 DEBUG } 18:19:59 DEBUG } 18:19:59 INFO 18:19:59 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:19:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:20:00 INFO [loop_until]: OK (rc = 0) 18:20:00 DEBUG --- stdout --- 18:20:00 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 499Mi ds-idrepo-0 8m 707Mi rcs-5d696ff5b-7ttvr 9m 296Mi rcs-5d696ff5b-wwlbr 8m 337Mi 18:20:00 DEBUG --- stderr --- 18:20:00 DEBUG 18:20:00 INFO 18:20:00 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:20:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:20:00 INFO [loop_until]: OK (rc = 0) 18:20:00 DEBUG --- stdout --- 18:20:00 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 106m 0% 2274Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1135m 3% 3756Mi 13% 18:20:00 DEBUG --- stderr --- 18:20:00 DEBUG 18:20:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459658" 18:20:58 INFO [http_cmd]: http status code OK 18:20:58 DEBUG --- status code --- 18:20:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:20:58 DEBUG --- http response --- 18:20:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:20:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734459658" 18:20:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459658" 18:20:58 INFO [http_cmd]: http status code OK 18:20:58 DEBUG --- status code --- 18:20:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:20:58 INFO [http_cmd]: http status code OK 18:20:58 DEBUG --- http response --- 18:20:58 DEBUG --- status code --- 18:20:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:20:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:20:58 DEBUG --- http response --- 18:20:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459659" 18:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459659" 18:20:59 INFO [http_cmd]: http status code OK 18:20:59 DEBUG --- status code --- 18:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:20:59 INFO [http_cmd]: http status code OK 18:20:59 DEBUG --- http response --- 18:20:59 DEBUG --- status code --- 18:20:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:20:59 DEBUG --- http response --- 18:20:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459659" 18:20:59 INFO [http_cmd]: http status code OK 18:20:59 DEBUG --- status code --- 18:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:20:59 DEBUG --- http response --- 18:20:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459659" 18:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459659" 18:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459659" 18:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459659" 18:20:59 INFO [http_cmd]: http status code OK 18:20:59 DEBUG --- status code --- 18:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:20:59 INFO [http_cmd]: http status code OK 18:20:59 DEBUG --- http response --- 18:20:59 DEBUG --- status code --- 18:20:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:20:59 DEBUG --- http response --- 18:20:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:20:59 INFO [http_cmd]: http status code OK 18:20:59 DEBUG --- status code --- 18:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:20:59 DEBUG --- http response --- 18:20:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:20:59 INFO [http_cmd]: http status code OK 18:20:59 DEBUG --- status code --- 18:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:20:59 DEBUG --- http response --- 18:20:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734459659" 18:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459659" 18:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459659" 18:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459659" 18:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459659" 18:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459659" 18:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734459659" 18:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459659" 18:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734459659" 18:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459659" 18:20:59 INFO [http_cmd]: http status code OK 18:20:59 INFO [http_cmd]: http status code OK 18:20:59 DEBUG --- status code --- 18:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459659" 18:20:59 DEBUG --- status code --- 18:20:59 INFO [http_cmd]: http status code OK 18:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:20:59 INFO [http_cmd]: http status code OK 18:20:59 INFO [http_cmd]: http status code OK 18:20:59 INFO [http_cmd]: http status code OK 18:20:59 INFO [http_cmd]: http status code OK 18:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459659" 18:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:20:59 DEBUG --- status code --- 18:20:59 DEBUG --- http response --- 18:20:59 DEBUG --- status code --- 18:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734459659" 18:20:59 DEBUG --- status code --- 18:20:59 INFO [http_cmd]: http status code OK 18:20:59 DEBUG --- status code --- 18:20:59 ERROR [http_cmd]: ERROR 18:20:59 DEBUG --- http response --- 18:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:20:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:20:59 DEBUG --- status code --- 18:20:59 DEBUG --- http response --- 18:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:20:59 INFO [http_cmd]: http status code OK 18:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459659" 18:20:59 DEBUG --- status code --- 18:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:20:59 ERROR --- status code --- 18:20:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:20:59 DEBUG --- http response --- 18:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:20:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:20:59 DEBUG --- http response --- 18:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459659" 18:20:59 DEBUG --- status code --- 18:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459659" 18:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459659" 18:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:20:59 DEBUG --- http response --- 18:20:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:20:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:20:59 DEBUG --- http response --- 18:20:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:20:59 DEBUG --- http response --- 18:20:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:20:59 INFO [http_cmd]: http status code OK 18:20:59 ERROR --- http response --- 18:20:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:20:59 INFO [http_cmd]: http status code OK 18:20:59 INFO [http_cmd]: http status code OK 18:20:59 DEBUG --- http response --- 18:20:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:20:59 INFO [http_cmd]: http status code OK 18:20:59 INFO [http_cmd]: http status code OK 18:20:59 DEBUG --- status code --- 18:20:59 INFO [http_cmd]: http status code OK 18:20:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:20:59 DEBUG --- status code --- 18:20:59 DEBUG --- status code --- 18:20:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:20:59 DEBUG --- status code --- 18:20:59 DEBUG --- status code --- 18:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:20:59 DEBUG --- status code --- 18:20:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:20:59 DEBUG --- http response --- 18:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:20:59 INFO [http_cmd]: http status code OK 18:20:59 DEBUG --- http response --- 18:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:20:59 DEBUG --- http response --- 18:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:20:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:20:59 DEBUG --- http response --- 18:20:59 DEBUG --- status code --- 18:20:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:20:59 DEBUG --- http response --- 18:20:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:20:59 DEBUG --- http response --- 18:20:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:20:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:20:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:20:59 DEBUG --- http response --- 18:20:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459659" 18:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459659" 18:20:59 INFO [http_cmd]: http status code OK 18:20:59 DEBUG --- status code --- 18:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:20:59 DEBUG --- http response --- 18:20:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:20:59 DEBUG ----- output ----- 18:20:59 DEBUG { 18:20:59 DEBUG "status": "success", 18:20:59 DEBUG "data": { 18:20:59 DEBUG "resultType": "vector", 18:20:59 DEBUG "result": [ 18:20:59 DEBUG { 18:20:59 DEBUG "metric": { 18:20:59 DEBUG "pod": "ds-cts-0" 18:20:59 DEBUG }, 18:20:59 DEBUG "value": [ 18:20:59 DEBUG 18:20:59 DEBUG 18:20:59 DEBUG 18:20:59 DEBUG [...] 18:20:59 DEBUG 18:20:59 DEBUG 18:20:59 DEBUG 18:20:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:20:59 DEBUG }, 18:20:59 DEBUG "value": [ 18:20:59 DEBUG 1734459659, 18:20:59 DEBUG "0" 18:20:59 DEBUG ] 18:20:59 DEBUG } 18:20:59 DEBUG ] 18:20:59 DEBUG } 18:20:59 DEBUG } 18:20:59 INFO [http_cmd]: http status code OK 18:20:59 DEBUG --- status code --- 18:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:20:59 DEBUG --- http response --- 18:20:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:20:59 DEBUG ----- output ----- 18:20:59 DEBUG { 18:20:59 DEBUG "status": "success", 18:20:59 DEBUG "data": { 18:20:59 DEBUG "resultType": "vector", 18:20:59 DEBUG "result": [ 18:20:59 DEBUG { 18:20:59 DEBUG "metric": { 18:20:59 DEBUG "pod": "ds-cts-0" 18:20:59 DEBUG }, 18:20:59 DEBUG "value": [ 18:20:59 DEBUG 18:20:59 DEBUG 18:20:59 DEBUG 18:20:59 DEBUG [...] 18:20:59 DEBUG 18:20:59 DEBUG 18:20:59 DEBUG 18:20:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:20:59 DEBUG }, 18:20:59 DEBUG "value": [ 18:20:59 DEBUG 1734459659, 18:20:59 DEBUG "0.004048166938749783" 18:20:59 DEBUG ] 18:20:59 DEBUG } 18:20:59 DEBUG ] 18:20:59 DEBUG } 18:20:59 DEBUG } 18:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459659" 18:20:59 INFO [http_cmd]: http status code OK 18:20:59 DEBUG --- status code --- 18:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:20:59 DEBUG --- http response --- 18:20:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:20:59 DEBUG ----- output ----- 18:20:59 DEBUG { 18:20:59 DEBUG "status": "success", 18:20:59 DEBUG "data": { 18:20:59 DEBUG "resultType": "vector", 18:20:59 DEBUG "result": [ 18:20:59 DEBUG { 18:20:59 DEBUG "metric": { 18:20:59 DEBUG "instance": "10.142.0.11:9100" 18:20:59 DEBUG }, 18:20:59 DEBUG "value": [ 18:20:59 DEBUG 18:20:59 DEBUG 18:20:59 DEBUG 18:20:59 DEBUG [...] 18:20:59 DEBUG 18:20:59 DEBUG 18:20:59 DEBUG 18:20:59 DEBUG "instance": "10.142.0.51:9100" 18:20:59 DEBUG }, 18:20:59 DEBUG "value": [ 18:20:59 DEBUG 1734459659, 18:20:59 DEBUG "125633.23333333332" 18:20:59 DEBUG ] 18:20:59 DEBUG } 18:20:59 DEBUG ] 18:20:59 DEBUG } 18:20:59 DEBUG } 18:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459659" 18:20:59 INFO [http_cmd]: http status code OK 18:20:59 DEBUG --- status code --- 18:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:20:59 DEBUG --- http response --- 18:20:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:20:59 DEBUG ----- output ----- 18:20:59 DEBUG { 18:20:59 DEBUG "status": "success", 18:20:59 DEBUG "data": { 18:20:59 DEBUG "resultType": "vector", 18:20:59 DEBUG "result": [ 18:20:59 DEBUG { 18:20:59 DEBUG "metric": { 18:20:59 DEBUG "pod": "ds-cts-0" 18:20:59 DEBUG }, 18:20:59 DEBUG "value": [ 18:20:59 DEBUG 18:20:59 DEBUG 18:20:59 DEBUG 18:20:59 DEBUG [...] 18:20:59 DEBUG 18:20:59 DEBUG 18:20:59 DEBUG 18:20:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:20:59 DEBUG }, 18:20:59 DEBUG "value": [ 18:20:59 DEBUG 1734459659, 18:20:59 DEBUG "709443584" 18:20:59 DEBUG ] 18:20:59 DEBUG } 18:20:59 DEBUG ] 18:20:59 DEBUG } 18:20:59 DEBUG } 18:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734459659" 18:20:59 INFO [http_cmd]: http status code OK 18:20:59 DEBUG --- status code --- 18:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:20:59 DEBUG --- http response --- 18:20:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:20:59 DEBUG ----- output ----- 18:20:59 DEBUG { 18:20:59 DEBUG "status": "success", 18:20:59 DEBUG "data": { 18:20:59 DEBUG "resultType": "vector", 18:20:59 DEBUG "result": [ 18:20:59 DEBUG { 18:20:59 DEBUG "metric": { 18:20:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:20:59 DEBUG }, 18:20:59 DEBUG "value": [ 18:20:59 DEBUG 18:20:59 DEBUG 18:20:59 DEBUG 18:20:59 DEBUG [...] 18:20:59 DEBUG 18:20:59 DEBUG 18:20:59 DEBUG 18:20:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:20:59 DEBUG }, 18:20:59 DEBUG "value": [ 18:20:59 DEBUG 1734459659, 18:20:59 DEBUG "1.0519708908294678" 18:20:59 DEBUG ] 18:20:59 DEBUG } 18:20:59 DEBUG ] 18:20:59 DEBUG } 18:20:59 DEBUG } 18:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459659" 18:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734459659" 18:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459659" 18:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459659" 18:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459659" 18:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459659" 18:20:59 INFO [http_cmd]: http status code OK 18:20:59 DEBUG --- status code --- 18:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:20:59 DEBUG --- http response --- 18:20:59 INFO [http_cmd]: http status code OK 18:20:59 DEBUG --- status code --- 18:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:20:59 DEBUG --- http response --- 18:20:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:20:59 DEBUG ----- output ----- 18:20:59 DEBUG { 18:20:59 DEBUG "status": "success", 18:20:59 DEBUG "data": { 18:20:59 DEBUG "resultType": "vector", 18:20:59 DEBUG "result": [ 18:20:59 DEBUG { 18:20:59 DEBUG "metric": { 18:20:59 DEBUG "instance": "10.142.0.11:9100" 18:20:59 DEBUG }, 18:20:59 DEBUG "value": [ 18:20:59 DEBUG 18:20:59 DEBUG 18:20:59 DEBUG 18:20:59 DEBUG [...] 18:20:59 DEBUG 18:20:59 DEBUG 18:20:59 DEBUG 18:20:59 DEBUG "instance": "10.142.0.51:9100" 18:20:59 DEBUG }, 18:20:59 DEBUG "value": [ 18:20:59 DEBUG 1734459659, 18:20:59 DEBUG "121785.26666666668" 18:20:59 DEBUG ] 18:20:59 DEBUG } 18:20:59 DEBUG ] 18:20:59 DEBUG } 18:20:59 DEBUG } 18:20:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:20:59 DEBUG ----- output ----- 18:20:59 DEBUG { 18:20:59 DEBUG "status": "success", 18:20:59 INFO [http_cmd]: http status code OK 18:20:59 DEBUG "data": { 18:20:59 DEBUG --- status code --- 18:20:59 DEBUG "resultType": "vector", 18:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:20:59 DEBUG "result": [ 18:20:59 DEBUG --- http response --- 18:20:59 DEBUG { 18:20:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:20:59 DEBUG "metric": { 18:20:59 DEBUG ----- output ----- 18:20:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:20:59 DEBUG { 18:20:59 DEBUG }, 18:20:59 DEBUG "status": "success", 18:20:59 DEBUG "value": [ 18:20:59 DEBUG "data": { 18:20:59 DEBUG 18:20:59 DEBUG "resultType": "vector", 18:20:59 DEBUG 18:20:59 DEBUG "result": [ 18:20:59 DEBUG 18:20:59 DEBUG { 18:20:59 DEBUG [...] 18:20:59 DEBUG "metric": { 18:20:59 DEBUG 18:20:59 DEBUG "instance": "10.142.0.11:9100" 18:20:59 DEBUG 18:20:59 DEBUG }, 18:20:59 DEBUG 18:20:59 DEBUG "value": [ 18:20:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:20:59 DEBUG 18:20:59 DEBUG }, 18:20:59 DEBUG 18:20:59 DEBUG "value": [ 18:20:59 INFO [http_cmd]: http status code OK 18:20:59 DEBUG 18:20:59 DEBUG 1734459659, 18:20:59 DEBUG --- status code --- 18:20:59 DEBUG [...] 18:20:59 DEBUG "2906341376" 18:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:20:59 DEBUG 18:20:59 DEBUG ] 18:20:59 DEBUG --- http response --- 18:20:59 DEBUG 18:20:59 DEBUG } 18:20:59 DEBUG 18:20:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:20:59 DEBUG ] 18:20:59 DEBUG "instance": "10.142.0.51:9100" 18:20:59 DEBUG ----- output ----- 18:20:59 DEBUG } 18:20:59 DEBUG }, 18:20:59 DEBUG { 18:20:59 DEBUG } 18:20:59 DEBUG "value": [ 18:20:59 DEBUG "status": "success", 18:20:59 DEBUG 1734459659, 18:20:59 DEBUG "data": { 18:20:59 DEBUG "25941.333333333336" 18:20:59 DEBUG "resultType": "vector", 18:20:59 DEBUG ] 18:20:59 DEBUG "result": [ 18:20:59 DEBUG } 18:20:59 DEBUG { 18:20:59 DEBUG ] 18:20:59 DEBUG "metric": { 18:20:59 DEBUG } 18:20:59 INFO [http_cmd]: http status code OK 18:20:59 DEBUG "pod": "ds-cts-0" 18:20:59 DEBUG } 18:20:59 DEBUG --- status code --- 18:20:59 DEBUG }, 18:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:20:59 DEBUG "value": [ 18:20:59 INFO [http_cmd]: http status code OK 18:20:59 DEBUG --- http response --- 18:20:59 DEBUG 18:20:59 DEBUG --- status code --- 18:20:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:20:59 DEBUG 18:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:20:59 DEBUG ----- output ----- 18:20:59 DEBUG 18:20:59 DEBUG --- http response --- 18:20:59 DEBUG { 18:20:59 DEBUG [...] 18:20:59 DEBUG "status": "success", 18:20:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:20:59 DEBUG 18:20:59 DEBUG "data": { 18:20:59 DEBUG ----- output ----- 18:20:59 DEBUG 18:20:59 DEBUG "resultType": "vector", 18:20:59 DEBUG { 18:20:59 DEBUG 18:20:59 DEBUG "result": [ 18:20:59 DEBUG "status": "success", 18:20:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:20:59 DEBUG { 18:20:59 DEBUG "data": { 18:20:59 DEBUG }, 18:20:59 DEBUG "metric": { 18:20:59 DEBUG "resultType": "vector", 18:20:59 DEBUG "value": [ 18:20:59 DEBUG "instance": "10.142.0.11:9100" 18:20:59 DEBUG "result": [ 18:20:59 DEBUG 1734459659, 18:20:59 DEBUG }, 18:20:59 DEBUG { 18:20:59 DEBUG "1298.9074079783234" 18:20:59 DEBUG "value": [ 18:20:59 DEBUG "metric": { 18:20:59 DEBUG ] 18:20:59 DEBUG 18:20:59 DEBUG "instance": "10.142.0.11:9100" 18:20:59 DEBUG } 18:20:59 DEBUG 18:20:59 DEBUG }, 18:20:59 DEBUG ] 18:20:59 DEBUG 18:20:59 DEBUG "value": [ 18:20:59 DEBUG } 18:20:59 DEBUG [...] 18:20:59 DEBUG 18:20:59 DEBUG } 18:20:59 DEBUG 18:20:59 DEBUG 18:20:59 DEBUG 18:20:59 DEBUG 18:20:59 DEBUG 18:20:59 DEBUG [...] 18:20:59 DEBUG "instance": "10.142.0.51:9100" 18:20:59 DEBUG 18:20:59 DEBUG }, 18:20:59 DEBUG 18:20:59 DEBUG "value": [ 18:20:59 DEBUG 18:20:59 DEBUG 1734459659, 18:20:59 DEBUG "instance": "10.142.0.51:9100" 18:20:59 DEBUG "0.0005333333333335115" 18:20:59 DEBUG }, 18:20:59 DEBUG ] 18:20:59 DEBUG "value": [ 18:20:59 DEBUG } 18:20:59 DEBUG 1734459659, 18:20:59 DEBUG ] 18:20:59 DEBUG "0.000333333333333267" 18:20:59 DEBUG } 18:20:59 DEBUG ] 18:20:59 DEBUG } 18:20:59 DEBUG } 18:20:59 DEBUG ] 18:20:59 DEBUG } 18:20:59 DEBUG } 18:21:00 INFO 18:21:00 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:21:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:21:00 INFO [loop_until]: OK (rc = 0) 18:21:00 DEBUG --- stdout --- 18:21:00 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 501Mi ds-idrepo-0 7m 709Mi rcs-5d696ff5b-7ttvr 3m 296Mi rcs-5d696ff5b-wwlbr 2m 338Mi 18:21:00 DEBUG --- stderr --- 18:21:00 DEBUG 18:21:00 INFO 18:21:00 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:21:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:21:00 INFO [loop_until]: OK (rc = 0) 18:21:00 DEBUG --- stdout --- 18:21:00 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 100m 0% 2268Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1135m 3% 3763Mi 13% 18:21:00 DEBUG --- stderr --- 18:21:00 DEBUG 18:21:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459718" 18:21:58 INFO [http_cmd]: http status code OK 18:21:58 DEBUG --- status code --- 18:21:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:21:58 DEBUG --- http response --- 18:21:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:21:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459718" 18:21:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734459718" 18:21:58 INFO [http_cmd]: http status code OK 18:21:58 DEBUG --- status code --- 18:21:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:21:58 DEBUG --- http response --- 18:21:59 INFO [http_cmd]: http status code OK 18:21:59 DEBUG --- status code --- 18:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:21:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:21:59 DEBUG --- http response --- 18:21:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459719" 18:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459719" 18:21:59 INFO [http_cmd]: http status code OK 18:21:59 DEBUG --- status code --- 18:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:21:59 DEBUG --- http response --- 18:21:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:21:59 INFO [http_cmd]: http status code OK 18:21:59 DEBUG --- status code --- 18:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:21:59 DEBUG --- http response --- 18:21:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459719" 18:21:59 INFO [http_cmd]: http status code OK 18:21:59 DEBUG --- status code --- 18:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:21:59 DEBUG --- http response --- 18:21:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459719" 18:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459719" 18:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459719" 18:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459719" 18:21:59 INFO [http_cmd]: http status code OK 18:21:59 DEBUG --- status code --- 18:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:21:59 DEBUG --- http response --- 18:21:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:21:59 INFO [http_cmd]: http status code OK 18:21:59 INFO [http_cmd]: http status code OK 18:21:59 DEBUG --- status code --- 18:21:59 DEBUG --- status code --- 18:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:21:59 DEBUG --- http response --- 18:21:59 DEBUG --- http response --- 18:21:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:21:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:21:59 INFO [http_cmd]: http status code OK 18:21:59 DEBUG --- status code --- 18:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:21:59 DEBUG --- http response --- 18:21:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459719" 18:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734459719" 18:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459719" 18:21:59 INFO [http_cmd]: http status code OK 18:21:59 DEBUG --- status code --- 18:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:21:59 DEBUG --- http response --- 18:21:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459719" 18:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459719" 18:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459719" 18:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734459719" 18:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459719" 18:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459719" 18:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734459719" 18:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459719" 18:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459719" 18:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459719" 18:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459719" 18:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459719" 18:21:59 INFO [http_cmd]: http status code OK 18:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459719" 18:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734459719" 18:21:59 INFO [http_cmd]: http status code OK 18:21:59 INFO [http_cmd]: http status code OK 18:21:59 INFO [http_cmd]: http status code OK 18:21:59 DEBUG --- status code --- 18:21:59 INFO [http_cmd]: http status code OK 18:21:59 DEBUG --- status code --- 18:21:59 INFO [http_cmd]: http status code OK 18:21:59 INFO [http_cmd]: http status code OK 18:21:59 DEBUG --- status code --- 18:21:59 INFO [http_cmd]: http status code OK 18:21:59 DEBUG --- status code --- 18:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:21:59 DEBUG --- status code --- 18:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:21:59 DEBUG --- status code --- 18:21:59 DEBUG --- status code --- 18:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:21:59 DEBUG --- status code --- 18:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:21:59 DEBUG --- http response --- 18:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:21:59 DEBUG --- http response --- 18:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:21:59 DEBUG --- http response --- 18:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:21:59 DEBUG --- http response --- 18:21:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:21:59 DEBUG --- http response --- 18:21:59 ERROR [http_cmd]: ERROR 18:21:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:21:59 DEBUG --- http response --- 18:21:59 INFO [http_cmd]: http status code OK 18:21:59 DEBUG --- http response --- 18:21:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:21:59 DEBUG --- http response --- 18:21:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:21:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:21:59 ERROR --- status code --- 18:21:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:21:59 DEBUG --- status code --- 18:21:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:21:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:21:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:21:59 ERROR --- http response --- 18:21:59 DEBUG --- http response --- 18:21:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:21:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:21:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:21:59 INFO [http_cmd]: http status code OK 18:21:59 DEBUG --- status code --- 18:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:21:59 DEBUG --- http response --- 18:21:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:21:59 INFO [http_cmd]: http status code OK 18:21:59 DEBUG --- status code --- 18:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:21:59 DEBUG --- http response --- 18:21:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:21:59 INFO [http_cmd]: http status code OK 18:21:59 DEBUG --- status code --- 18:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:21:59 DEBUG --- http response --- 18:21:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:21:59 INFO [http_cmd]: http status code OK 18:21:59 DEBUG --- status code --- 18:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:21:59 DEBUG --- http response --- 18:21:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:21:59 INFO [http_cmd]: http status code OK 18:21:59 DEBUG --- status code --- 18:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:21:59 DEBUG --- http response --- 18:21:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:21:59 INFO [http_cmd]: http status code OK 18:21:59 DEBUG --- status code --- 18:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:21:59 DEBUG --- http response --- 18:21:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459719" 18:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459719" 18:21:59 INFO [http_cmd]: http status code OK 18:21:59 DEBUG --- status code --- 18:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:21:59 DEBUG --- http response --- 18:21:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:21:59 DEBUG ----- output ----- 18:21:59 DEBUG { 18:21:59 DEBUG "status": "success", 18:21:59 DEBUG "data": { 18:21:59 DEBUG "resultType": "vector", 18:21:59 DEBUG "result": [ 18:21:59 DEBUG { 18:21:59 DEBUG "metric": { 18:21:59 DEBUG "pod": "ds-cts-0" 18:21:59 DEBUG }, 18:21:59 DEBUG "value": [ 18:21:59 DEBUG 18:21:59 DEBUG 18:21:59 DEBUG 18:21:59 DEBUG [...] 18:21:59 DEBUG 18:21:59 DEBUG 18:21:59 DEBUG 18:21:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:21:59 DEBUG }, 18:21:59 DEBUG "value": [ 18:21:59 DEBUG 1734459719, 18:21:59 DEBUG "0" 18:21:59 DEBUG ] 18:21:59 DEBUG } 18:21:59 DEBUG ] 18:21:59 DEBUG } 18:21:59 DEBUG } 18:21:59 INFO [http_cmd]: http status code OK 18:21:59 DEBUG --- status code --- 18:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:21:59 DEBUG --- http response --- 18:21:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:21:59 DEBUG ----- output ----- 18:21:59 DEBUG { 18:21:59 DEBUG "status": "success", 18:21:59 DEBUG "data": { 18:21:59 DEBUG "resultType": "vector", 18:21:59 DEBUG "result": [ 18:21:59 DEBUG { 18:21:59 DEBUG "metric": { 18:21:59 DEBUG "pod": "ds-cts-0" 18:21:59 DEBUG }, 18:21:59 DEBUG "value": [ 18:21:59 DEBUG 18:21:59 DEBUG 18:21:59 DEBUG 18:21:59 DEBUG [...] 18:21:59 DEBUG 18:21:59 DEBUG 18:21:59 DEBUG 18:21:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:21:59 DEBUG }, 18:21:59 DEBUG "value": [ 18:21:59 DEBUG 1734459719, 18:21:59 DEBUG "0.003937160633241531" 18:21:59 DEBUG ] 18:21:59 DEBUG } 18:21:59 DEBUG ] 18:21:59 DEBUG } 18:21:59 DEBUG } 18:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459719" 18:21:59 INFO [http_cmd]: http status code OK 18:21:59 DEBUG --- status code --- 18:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:21:59 DEBUG --- http response --- 18:21:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:21:59 DEBUG ----- output ----- 18:21:59 DEBUG { 18:21:59 DEBUG "status": "success", 18:21:59 DEBUG "data": { 18:21:59 DEBUG "resultType": "vector", 18:21:59 DEBUG "result": [ 18:21:59 DEBUG { 18:21:59 DEBUG "metric": { 18:21:59 DEBUG "instance": "10.142.0.11:9100" 18:21:59 DEBUG }, 18:21:59 DEBUG "value": [ 18:21:59 DEBUG 18:21:59 DEBUG 18:21:59 DEBUG 18:21:59 DEBUG [...] 18:21:59 DEBUG 18:21:59 DEBUG 18:21:59 DEBUG 18:21:59 DEBUG "instance": "10.142.0.51:9100" 18:21:59 DEBUG }, 18:21:59 DEBUG "value": [ 18:21:59 DEBUG 1734459719, 18:21:59 DEBUG "133483.33333333334" 18:21:59 DEBUG ] 18:21:59 DEBUG } 18:21:59 DEBUG ] 18:21:59 DEBUG } 18:21:59 DEBUG } 18:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459719" 18:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734459719" 18:21:59 INFO [http_cmd]: http status code OK 18:21:59 DEBUG --- status code --- 18:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:21:59 DEBUG --- http response --- 18:21:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:21:59 DEBUG ----- output ----- 18:21:59 DEBUG { 18:21:59 DEBUG "status": "success", 18:21:59 DEBUG "data": { 18:21:59 DEBUG "resultType": "vector", 18:21:59 DEBUG "result": [ 18:21:59 DEBUG { 18:21:59 DEBUG "metric": { 18:21:59 DEBUG "pod": "ds-cts-0" 18:21:59 DEBUG }, 18:21:59 DEBUG "value": [ 18:21:59 DEBUG 18:21:59 DEBUG 18:21:59 DEBUG 18:21:59 DEBUG [...] 18:21:59 DEBUG 18:21:59 DEBUG 18:21:59 DEBUG 18:21:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:21:59 DEBUG }, 18:21:59 DEBUG "value": [ 18:21:59 DEBUG 1734459719, 18:21:59 DEBUG "709505024" 18:21:59 DEBUG ] 18:21:59 DEBUG } 18:21:59 DEBUG ] 18:21:59 DEBUG } 18:21:59 DEBUG } 18:21:59 INFO [http_cmd]: http status code OK 18:21:59 DEBUG --- status code --- 18:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:21:59 DEBUG --- http response --- 18:21:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:21:59 DEBUG ----- output ----- 18:21:59 DEBUG { 18:21:59 DEBUG "status": "success", 18:21:59 DEBUG "data": { 18:21:59 DEBUG "resultType": "vector", 18:21:59 DEBUG "result": [ 18:21:59 DEBUG { 18:21:59 DEBUG "metric": { 18:21:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:21:59 DEBUG }, 18:21:59 DEBUG "value": [ 18:21:59 DEBUG 18:21:59 DEBUG 18:21:59 DEBUG 18:21:59 DEBUG [...] 18:21:59 DEBUG 18:21:59 DEBUG 18:21:59 DEBUG 18:21:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:21:59 DEBUG }, 18:21:59 DEBUG "value": [ 18:21:59 DEBUG 1734459719, 18:21:59 DEBUG "1.0617163553784355" 18:21:59 DEBUG ] 18:21:59 DEBUG } 18:21:59 DEBUG ] 18:21:59 DEBUG } 18:21:59 DEBUG } 18:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459719" 18:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734459719" 18:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459719" 18:21:59 INFO [http_cmd]: http status code OK 18:21:59 DEBUG --- status code --- 18:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:21:59 DEBUG --- http response --- 18:21:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:21:59 DEBUG ----- output ----- 18:21:59 DEBUG { 18:21:59 DEBUG "status": "success", 18:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459719" 18:21:59 DEBUG "data": { 18:21:59 DEBUG "resultType": "vector", 18:21:59 DEBUG "result": [ 18:21:59 DEBUG { 18:21:59 DEBUG "metric": { 18:21:59 DEBUG "instance": "10.142.0.11:9100" 18:21:59 DEBUG }, 18:21:59 DEBUG "value": [ 18:21:59 DEBUG 18:21:59 DEBUG 18:21:59 DEBUG 18:21:59 DEBUG [...] 18:21:59 DEBUG 18:21:59 DEBUG 18:21:59 DEBUG 18:21:59 DEBUG "instance": "10.142.0.51:9100" 18:21:59 DEBUG }, 18:21:59 DEBUG "value": [ 18:21:59 DEBUG 1734459719, 18:21:59 DEBUG "129526.50000000001" 18:21:59 DEBUG ] 18:21:59 DEBUG } 18:21:59 DEBUG ] 18:21:59 DEBUG } 18:21:59 DEBUG } 18:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459719" 18:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459719" 18:21:59 INFO [http_cmd]: http status code OK 18:21:59 INFO [http_cmd]: http status code OK 18:21:59 DEBUG --- status code --- 18:21:59 DEBUG --- status code --- 18:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:21:59 DEBUG --- http response --- 18:21:59 DEBUG --- http response --- 18:21:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:21:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:21:59 DEBUG ----- output ----- 18:21:59 DEBUG ----- output ----- 18:21:59 DEBUG { 18:21:59 DEBUG { 18:21:59 DEBUG "status": "success", 18:21:59 DEBUG "status": "success", 18:21:59 DEBUG "data": { 18:21:59 DEBUG "data": { 18:21:59 DEBUG "resultType": "vector", 18:21:59 DEBUG "resultType": "vector", 18:21:59 DEBUG "result": [ 18:21:59 DEBUG { 18:21:59 DEBUG "result": [ 18:21:59 DEBUG "metric": { 18:21:59 DEBUG { 18:21:59 DEBUG "instance": "10.142.0.11:9100" 18:21:59 DEBUG "metric": { 18:21:59 DEBUG }, 18:21:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:21:59 DEBUG "value": [ 18:21:59 DEBUG }, 18:21:59 DEBUG 18:21:59 DEBUG "value": [ 18:21:59 DEBUG 18:21:59 DEBUG 18:21:59 DEBUG 18:21:59 DEBUG 18:21:59 DEBUG [...] 18:21:59 DEBUG 18:21:59 DEBUG 18:21:59 DEBUG [...] 18:21:59 DEBUG 18:21:59 DEBUG 18:21:59 DEBUG 18:21:59 DEBUG 18:21:59 DEBUG "instance": "10.142.0.51:9100" 18:21:59 DEBUG 18:21:59 DEBUG }, 18:21:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:21:59 DEBUG "value": [ 18:21:59 INFO [http_cmd]: http status code OK 18:21:59 DEBUG }, 18:21:59 DEBUG 1734459719, 18:21:59 DEBUG --- status code --- 18:21:59 DEBUG "value": [ 18:21:59 DEBUG "27716.266666666666" 18:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:21:59 DEBUG 1734459719, 18:21:59 DEBUG ] 18:21:59 DEBUG --- http response --- 18:21:59 DEBUG "2914635776" 18:21:59 DEBUG } 18:21:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:21:59 DEBUG ] 18:21:59 DEBUG ] 18:21:59 DEBUG ----- output ----- 18:21:59 DEBUG } 18:21:59 DEBUG } 18:21:59 DEBUG { 18:21:59 DEBUG ] 18:21:59 DEBUG } 18:21:59 DEBUG "status": "success", 18:21:59 DEBUG } 18:21:59 DEBUG "data": { 18:21:59 INFO [http_cmd]: http status code OK 18:21:59 DEBUG } 18:21:59 DEBUG "resultType": "vector", 18:21:59 DEBUG --- status code --- 18:21:59 DEBUG "result": [ 18:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:21:59 DEBUG { 18:21:59 DEBUG --- http response --- 18:21:59 DEBUG "metric": { 18:21:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:21:59 DEBUG "pod": "ds-cts-0" 18:21:59 DEBUG ----- output ----- 18:21:59 DEBUG }, 18:21:59 DEBUG { 18:21:59 DEBUG "value": [ 18:21:59 DEBUG "status": "success", 18:21:59 DEBUG 18:21:59 DEBUG "data": { 18:21:59 DEBUG 18:21:59 DEBUG "resultType": "vector", 18:21:59 DEBUG 18:21:59 DEBUG "result": [ 18:21:59 DEBUG [...] 18:21:59 DEBUG { 18:21:59 DEBUG 18:21:59 DEBUG "metric": { 18:21:59 DEBUG 18:21:59 DEBUG "instance": "10.142.0.11:9100" 18:21:59 DEBUG 18:21:59 DEBUG }, 18:21:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:21:59 DEBUG "value": [ 18:21:59 DEBUG }, 18:21:59 DEBUG 18:21:59 DEBUG "value": [ 18:21:59 DEBUG 18:21:59 DEBUG 1734459719, 18:21:59 DEBUG 18:21:59 DEBUG "1950.307421243976" 18:21:59 DEBUG [...] 18:21:59 INFO [http_cmd]: http status code OK 18:21:59 DEBUG ] 18:21:59 DEBUG 18:21:59 DEBUG --- status code --- 18:21:59 DEBUG } 18:21:59 DEBUG 18:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:21:59 DEBUG ] 18:21:59 DEBUG 18:21:59 DEBUG --- http response --- 18:21:59 DEBUG } 18:21:59 DEBUG "instance": "10.142.0.51:9100" 18:21:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:21:59 DEBUG } 18:21:59 DEBUG }, 18:21:59 DEBUG ----- output ----- 18:21:59 DEBUG "value": [ 18:21:59 DEBUG { 18:21:59 DEBUG 1734459719, 18:21:59 DEBUG "status": "success", 18:21:59 DEBUG "0.00043333333333208654" 18:21:59 DEBUG "data": { 18:21:59 DEBUG ] 18:21:59 DEBUG "resultType": "vector", 18:21:59 DEBUG } 18:21:59 DEBUG "result": [ 18:21:59 DEBUG ] 18:21:59 DEBUG { 18:21:59 DEBUG } 18:21:59 DEBUG "metric": { 18:21:59 DEBUG } 18:21:59 DEBUG "instance": "10.142.0.11:9100" 18:21:59 DEBUG }, 18:21:59 DEBUG "value": [ 18:21:59 DEBUG 18:21:59 DEBUG 18:21:59 DEBUG 18:21:59 DEBUG [...] 18:21:59 DEBUG 18:21:59 DEBUG 18:21:59 DEBUG 18:21:59 DEBUG "instance": "10.142.0.51:9100" 18:21:59 DEBUG }, 18:21:59 DEBUG "value": [ 18:21:59 DEBUG 1734459719, 18:21:59 DEBUG "0" 18:21:59 DEBUG ] 18:21:59 DEBUG } 18:21:59 DEBUG ] 18:21:59 DEBUG } 18:21:59 DEBUG } 18:22:00 INFO 18:22:00 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:22:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:22:00 INFO [loop_until]: OK (rc = 0) 18:22:00 DEBUG --- stdout --- 18:22:00 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 504Mi ds-idrepo-0 7m 710Mi rcs-5d696ff5b-7ttvr 5m 297Mi rcs-5d696ff5b-wwlbr 3m 338Mi 18:22:00 DEBUG --- stderr --- 18:22:00 DEBUG 18:22:00 INFO 18:22:00 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:22:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:22:00 INFO [loop_until]: OK (rc = 0) 18:22:00 DEBUG --- stdout --- 18:22:00 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 98m 0% 2253Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1138m 3% 3762Mi 13% 18:22:00 DEBUG --- stderr --- 18:22:00 DEBUG 18:22:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459778" 18:22:58 INFO [http_cmd]: http status code OK 18:22:58 DEBUG --- status code --- 18:22:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:22:58 DEBUG --- http response --- 18:22:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:22:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734459779" 18:22:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459779" 18:22:59 INFO [http_cmd]: http status code OK 18:22:59 DEBUG --- status code --- 18:22:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:22:59 DEBUG --- http response --- 18:22:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:22:59 INFO [http_cmd]: http status code OK 18:22:59 DEBUG --- status code --- 18:22:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:22:59 DEBUG --- http response --- 18:22:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:22:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459779" 18:22:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459779" 18:22:59 INFO [http_cmd]: http status code OK 18:22:59 DEBUG --- status code --- 18:22:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:22:59 DEBUG --- http response --- 18:22:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:22:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459779" 18:22:59 INFO [http_cmd]: http status code OK 18:22:59 DEBUG --- status code --- 18:22:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:22:59 DEBUG --- http response --- 18:22:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:22:59 INFO [http_cmd]: http status code OK 18:22:59 DEBUG --- status code --- 18:22:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:22:59 DEBUG --- http response --- 18:22:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:22:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459779" 18:22:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459779" 18:22:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459779" 18:22:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459779" 18:22:59 INFO [http_cmd]: http status code OK 18:22:59 DEBUG --- status code --- 18:22:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:22:59 DEBUG --- http response --- 18:22:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:22:59 INFO [http_cmd]: http status code OK 18:22:59 DEBUG --- status code --- 18:22:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:22:59 DEBUG --- http response --- 18:22:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:22:59 INFO [http_cmd]: http status code OK 18:22:59 DEBUG --- status code --- 18:22:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:22:59 INFO [http_cmd]: http status code OK 18:22:59 DEBUG --- http response --- 18:22:59 DEBUG --- status code --- 18:22:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:22:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:22:59 DEBUG --- http response --- 18:22:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:22:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459779" 18:22:59 INFO [http_cmd]: http status code OK 18:22:59 DEBUG --- status code --- 18:22:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:22:59 DEBUG --- http response --- 18:22:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:22:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459779" 18:22:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459779" 18:22:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734459779" 18:22:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459779" 18:22:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459779" 18:22:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459779" 18:22:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459779" 18:22:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734459779" 18:22:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734459779" 18:22:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459779" 18:22:59 INFO [http_cmd]: http status code OK 18:22:59 INFO [http_cmd]: http status code OK 18:22:59 INFO [http_cmd]: http status code OK 18:22:59 INFO [http_cmd]: http status code OK 18:22:59 DEBUG --- status code --- 18:22:59 INFO [http_cmd]: http status code OK 18:22:59 DEBUG --- status code --- 18:22:59 INFO [http_cmd]: http status code OK 18:22:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459779" 18:22:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459779" 18:22:59 DEBUG --- status code --- 18:22:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459779" 18:22:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734459779" 18:22:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459779" 18:22:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459779" 18:22:59 DEBUG --- status code --- 18:22:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:22:59 DEBUG --- status code --- 18:22:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:22:59 DEBUG --- status code --- 18:22:59 ERROR [http_cmd]: ERROR 18:22:59 INFO [http_cmd]: http status code OK 18:22:59 INFO [http_cmd]: http status code OK 18:22:59 DEBUG --- status code --- 18:22:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:22:59 DEBUG --- http response --- 18:22:59 DEBUG --- http response --- 18:22:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:22:59 DEBUG --- http response --- 18:22:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:22:59 ERROR --- status code --- 18:22:59 INFO [http_cmd]: http status code OK 18:22:59 INFO [http_cmd]: http status code OK 18:22:59 DEBUG --- status code --- 18:22:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:22:59 INFO [http_cmd]: http status code OK 18:22:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:22:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:22:59 DEBUG --- http response --- 18:22:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:22:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:22:59 DEBUG --- http response --- 18:22:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:22:59 DEBUG --- status code --- 18:22:59 DEBUG --- status code --- 18:22:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:22:59 DEBUG --- http response --- 18:22:59 DEBUG --- status code --- 18:22:59 DEBUG --- http response --- 18:22:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:22:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:22:59 ERROR --- http response --- 18:22:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:22:59 DEBUG --- http response --- 18:22:59 DEBUG --- http response --- 18:22:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:22:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:22:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:22:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:22:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:22:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:22:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:22:59 DEBUG --- http response --- 18:22:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:22:59 DEBUG --- http response --- 18:22:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:22:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:22:59 INFO [http_cmd]: http status code OK 18:22:59 DEBUG --- status code --- 18:22:59 INFO [http_cmd]: http status code OK 18:22:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:22:59 INFO [http_cmd]: http status code OK 18:22:59 INFO [http_cmd]: http status code OK 18:22:59 DEBUG --- status code --- 18:22:59 DEBUG --- http response --- 18:22:59 DEBUG --- status code --- 18:22:59 DEBUG --- status code --- 18:22:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:22:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:22:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:22:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:22:59 DEBUG --- http response --- 18:22:59 DEBUG --- http response --- 18:22:59 DEBUG --- http response --- 18:22:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:22:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:22:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:22:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459779" 18:22:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459779" 18:22:59 INFO [http_cmd]: http status code OK 18:22:59 DEBUG --- status code --- 18:22:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:22:59 DEBUG --- http response --- 18:22:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:22:59 DEBUG ----- output ----- 18:22:59 DEBUG { 18:22:59 DEBUG "status": "success", 18:22:59 DEBUG "data": { 18:22:59 DEBUG "resultType": "vector", 18:22:59 DEBUG "result": [ 18:22:59 DEBUG { 18:22:59 DEBUG "metric": { 18:22:59 DEBUG "pod": "ds-cts-0" 18:22:59 DEBUG }, 18:22:59 DEBUG "value": [ 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG [...] 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:22:59 DEBUG }, 18:22:59 DEBUG "value": [ 18:22:59 DEBUG 1734459779, 18:22:59 DEBUG "0" 18:22:59 DEBUG ] 18:22:59 DEBUG } 18:22:59 DEBUG ] 18:22:59 DEBUG } 18:22:59 DEBUG } 18:22:59 INFO [http_cmd]: http status code OK 18:22:59 DEBUG --- status code --- 18:22:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:22:59 DEBUG --- http response --- 18:22:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:22:59 DEBUG ----- output ----- 18:22:59 DEBUG { 18:22:59 DEBUG "status": "success", 18:22:59 DEBUG "data": { 18:22:59 DEBUG "resultType": "vector", 18:22:59 DEBUG "result": [ 18:22:59 DEBUG { 18:22:59 DEBUG "metric": { 18:22:59 DEBUG "pod": "ds-cts-0" 18:22:59 DEBUG }, 18:22:59 DEBUG "value": [ 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG [...] 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:22:59 DEBUG }, 18:22:59 DEBUG "value": [ 18:22:59 DEBUG 1734459779, 18:22:59 DEBUG "0.0165242586078307" 18:22:59 DEBUG ] 18:22:59 DEBUG } 18:22:59 DEBUG ] 18:22:59 DEBUG } 18:22:59 DEBUG } 18:22:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459779" 18:22:59 INFO [http_cmd]: http status code OK 18:22:59 DEBUG --- status code --- 18:22:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:22:59 DEBUG --- http response --- 18:22:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:22:59 DEBUG ----- output ----- 18:22:59 DEBUG { 18:22:59 DEBUG "status": "success", 18:22:59 DEBUG "data": { 18:22:59 DEBUG "resultType": "vector", 18:22:59 DEBUG "result": [ 18:22:59 DEBUG { 18:22:59 DEBUG "metric": { 18:22:59 DEBUG "instance": "10.142.0.11:9100" 18:22:59 DEBUG }, 18:22:59 DEBUG "value": [ 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG [...] 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG "instance": "10.142.0.51:9100" 18:22:59 DEBUG }, 18:22:59 DEBUG "value": [ 18:22:59 DEBUG 1734459779, 18:22:59 DEBUG "124862.56666666667" 18:22:59 DEBUG ] 18:22:59 DEBUG } 18:22:59 DEBUG ] 18:22:59 DEBUG } 18:22:59 DEBUG } 18:22:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459779" 18:22:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734459779" 18:22:59 INFO [http_cmd]: http status code OK 18:22:59 DEBUG --- status code --- 18:22:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:22:59 DEBUG --- http response --- 18:22:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:22:59 DEBUG ----- output ----- 18:22:59 DEBUG { 18:22:59 DEBUG "status": "success", 18:22:59 DEBUG "data": { 18:22:59 DEBUG "resultType": "vector", 18:22:59 DEBUG "result": [ 18:22:59 DEBUG { 18:22:59 DEBUG "metric": { 18:22:59 DEBUG "pod": "ds-cts-0" 18:22:59 DEBUG }, 18:22:59 DEBUG "value": [ 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG [...] 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:22:59 DEBUG }, 18:22:59 DEBUG "value": [ 18:22:59 DEBUG 1734459779, 18:22:59 DEBUG "710103040" 18:22:59 DEBUG ] 18:22:59 DEBUG } 18:22:59 DEBUG ] 18:22:59 DEBUG } 18:22:59 DEBUG } 18:22:59 INFO [http_cmd]: http status code OK 18:22:59 DEBUG --- status code --- 18:22:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:22:59 DEBUG --- http response --- 18:22:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:22:59 DEBUG ----- output ----- 18:22:59 DEBUG { 18:22:59 DEBUG "status": "success", 18:22:59 DEBUG "data": { 18:22:59 DEBUG "resultType": "vector", 18:22:59 DEBUG "result": [ 18:22:59 DEBUG { 18:22:59 DEBUG "metric": { 18:22:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:22:59 DEBUG }, 18:22:59 DEBUG "value": [ 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG [...] 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:22:59 DEBUG }, 18:22:59 DEBUG "value": [ 18:22:59 DEBUG 1734459779, 18:22:59 DEBUG "1.052737061631541" 18:22:59 DEBUG ] 18:22:59 DEBUG } 18:22:59 DEBUG ] 18:22:59 DEBUG } 18:22:59 DEBUG } 18:22:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459779" 18:22:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459779" 18:22:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734459779" 18:22:59 INFO [http_cmd]: http status code OK 18:22:59 DEBUG --- status code --- 18:22:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:22:59 DEBUG --- http response --- 18:22:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:22:59 DEBUG ----- output ----- 18:22:59 DEBUG { 18:22:59 DEBUG "status": "success", 18:22:59 DEBUG "data": { 18:22:59 DEBUG "resultType": "vector", 18:22:59 DEBUG "result": [ 18:22:59 DEBUG { 18:22:59 DEBUG "metric": { 18:22:59 DEBUG "instance": "10.142.0.11:9100" 18:22:59 DEBUG }, 18:22:59 DEBUG "value": [ 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG [...] 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG "instance": "10.142.0.51:9100" 18:22:59 DEBUG }, 18:22:59 DEBUG "value": [ 18:22:59 DEBUG 1734459779, 18:22:59 DEBUG "124632.13333333332" 18:22:59 DEBUG ] 18:22:59 DEBUG } 18:22:59 DEBUG ] 18:22:59 DEBUG } 18:22:59 DEBUG } 18:22:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459779" 18:22:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459779" 18:22:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459779" 18:22:59 INFO [http_cmd]: http status code OK 18:22:59 INFO [http_cmd]: http status code OK 18:22:59 DEBUG --- status code --- 18:22:59 DEBUG --- status code --- 18:22:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:22:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:22:59 DEBUG --- http response --- 18:22:59 DEBUG --- http response --- 18:22:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:22:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:22:59 DEBUG ----- output ----- 18:22:59 DEBUG ----- output ----- 18:22:59 DEBUG { 18:22:59 DEBUG { 18:22:59 DEBUG "status": "success", 18:22:59 DEBUG "status": "success", 18:22:59 DEBUG "data": { 18:22:59 DEBUG "data": { 18:22:59 DEBUG "resultType": "vector", 18:22:59 DEBUG "resultType": "vector", 18:22:59 DEBUG "result": [ 18:22:59 DEBUG "result": [ 18:22:59 DEBUG { 18:22:59 DEBUG { 18:22:59 DEBUG "metric": { 18:22:59 DEBUG "metric": { 18:22:59 DEBUG "instance": "10.142.0.11:9100" 18:22:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:22:59 DEBUG }, 18:22:59 DEBUG }, 18:22:59 DEBUG "value": [ 18:22:59 DEBUG "value": [ 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG [...] 18:22:59 DEBUG [...] 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG "instance": "10.142.0.51:9100" 18:22:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:22:59 DEBUG }, 18:22:59 DEBUG }, 18:22:59 DEBUG "value": [ 18:22:59 DEBUG "value": [ 18:22:59 DEBUG 1734459779, 18:22:59 DEBUG 1734459779, 18:22:59 DEBUG "25395.199999999997" 18:22:59 DEBUG "2923929600" 18:22:59 DEBUG ] 18:22:59 DEBUG ] 18:22:59 DEBUG } 18:22:59 DEBUG } 18:22:59 DEBUG ] 18:22:59 INFO [http_cmd]: http status code OK 18:22:59 DEBUG ] 18:22:59 DEBUG } 18:22:59 DEBUG --- status code --- 18:22:59 DEBUG } 18:22:59 DEBUG } 18:22:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:22:59 DEBUG } 18:22:59 DEBUG --- http response --- 18:22:59 INFO [http_cmd]: http status code OK 18:22:59 INFO [http_cmd]: http status code OK 18:22:59 DEBUG --- status code --- 18:22:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:22:59 DEBUG --- status code --- 18:22:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:22:59 DEBUG --- http response --- 18:22:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:22:59 DEBUG ----- output ----- 18:22:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:22:59 DEBUG --- http response --- 18:22:59 DEBUG { 18:22:59 DEBUG ----- output ----- 18:22:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:22:59 DEBUG "status": "success", 18:22:59 DEBUG { 18:22:59 DEBUG "status": "success", 18:22:59 DEBUG "data": { 18:22:59 DEBUG ----- output ----- 18:22:59 DEBUG "data": { 18:22:59 DEBUG "resultType": "vector", 18:22:59 DEBUG { 18:22:59 DEBUG "resultType": "vector", 18:22:59 DEBUG "result": [ 18:22:59 DEBUG "status": "success", 18:22:59 DEBUG "result": [ 18:22:59 DEBUG { 18:22:59 DEBUG "data": { 18:22:59 DEBUG { 18:22:59 DEBUG "metric": { 18:22:59 DEBUG "resultType": "vector", 18:22:59 DEBUG "metric": { 18:22:59 DEBUG "pod": "ds-cts-0" 18:22:59 DEBUG "result": [ 18:22:59 DEBUG "instance": "10.142.0.11:9100" 18:22:59 DEBUG }, 18:22:59 DEBUG { 18:22:59 DEBUG }, 18:22:59 DEBUG "value": [ 18:22:59 DEBUG "metric": { 18:22:59 DEBUG "value": [ 18:22:59 DEBUG 18:22:59 DEBUG "instance": "10.142.0.11:9100" 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG }, 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG "value": [ 18:22:59 DEBUG 18:22:59 DEBUG [...] 18:22:59 DEBUG 18:22:59 DEBUG [...] 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG 18:22:59 DEBUG [...] 18:22:59 DEBUG 18:22:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:22:59 DEBUG 18:22:59 DEBUG "instance": "10.142.0.51:9100" 18:22:59 DEBUG }, 18:22:59 DEBUG 18:22:59 DEBUG }, 18:22:59 DEBUG "value": [ 18:22:59 DEBUG 18:22:59 DEBUG "value": [ 18:22:59 DEBUG 1734459779, 18:22:59 DEBUG "instance": "10.142.0.51:9100" 18:22:59 DEBUG 1734459779, 18:22:59 DEBUG "1062.1265017416213" 18:22:59 DEBUG }, 18:22:59 DEBUG "0.00039999999999954146" 18:22:59 DEBUG ] 18:22:59 DEBUG "value": [ 18:22:59 DEBUG ] 18:22:59 DEBUG } 18:22:59 DEBUG 1734459779, 18:22:59 DEBUG } 18:22:59 DEBUG ] 18:22:59 DEBUG "0.000333333333333267" 18:22:59 DEBUG ] 18:22:59 DEBUG } 18:22:59 DEBUG ] 18:22:59 DEBUG } 18:22:59 DEBUG } 18:22:59 DEBUG } 18:22:59 DEBUG } 18:22:59 DEBUG ] 18:22:59 DEBUG } 18:22:59 DEBUG } 18:23:00 INFO 18:23:00 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:23:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:23:00 INFO [loop_until]: OK (rc = 0) 18:23:00 DEBUG --- stdout --- 18:23:00 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 508Mi ds-idrepo-0 7m 711Mi rcs-5d696ff5b-7ttvr 2m 297Mi rcs-5d696ff5b-wwlbr 7m 338Mi 18:23:00 DEBUG --- stderr --- 18:23:00 DEBUG 18:23:00 INFO 18:23:00 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:23:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:23:00 INFO [loop_until]: OK (rc = 0) 18:23:00 DEBUG --- stdout --- 18:23:00 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 111m 0% 2268Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1128m 3% 3765Mi 13% 18:23:00 DEBUG --- stderr --- 18:23:00 DEBUG 18:23:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459838" 18:23:58 INFO [http_cmd]: http status code OK 18:23:58 DEBUG --- status code --- 18:23:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:23:58 DEBUG --- http response --- 18:23:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:23:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734459839" 18:23:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459839" 18:23:59 INFO [http_cmd]: http status code OK 18:23:59 INFO [http_cmd]: http status code OK 18:23:59 DEBUG --- status code --- 18:23:59 DEBUG --- status code --- 18:23:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:23:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:23:59 DEBUG --- http response --- 18:23:59 DEBUG --- http response --- 18:23:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:23:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:23:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459839" 18:23:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459839" 18:23:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459839" 18:23:59 INFO [http_cmd]: http status code OK 18:23:59 DEBUG --- status code --- 18:23:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:23:59 DEBUG --- http response --- 18:23:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:23:59 INFO [http_cmd]: http status code OK 18:23:59 INFO [http_cmd]: http status code OK 18:23:59 DEBUG --- status code --- 18:23:59 DEBUG --- status code --- 18:23:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:23:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:23:59 DEBUG --- http response --- 18:23:59 DEBUG --- http response --- 18:23:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:23:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:23:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459839" 18:23:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459839" 18:23:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459839" 18:23:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459839" 18:23:59 INFO [http_cmd]: http status code OK 18:23:59 DEBUG --- status code --- 18:23:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:23:59 DEBUG --- http response --- 18:23:59 INFO [http_cmd]: http status code OK 18:23:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:23:59 INFO [http_cmd]: http status code OK 18:23:59 DEBUG --- status code --- 18:23:59 DEBUG --- status code --- 18:23:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:23:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:23:59 DEBUG --- http response --- 18:23:59 DEBUG --- http response --- 18:23:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:23:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:23:59 INFO [http_cmd]: http status code OK 18:23:59 DEBUG --- status code --- 18:23:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:23:59 DEBUG --- http response --- 18:23:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:23:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459839" 18:23:59 INFO [http_cmd]: http status code OK 18:23:59 DEBUG --- status code --- 18:23:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:23:59 DEBUG --- http response --- 18:23:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:23:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734459839" 18:23:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459839" 18:23:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734459839" 18:23:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459839" 18:23:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459839" 18:23:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734459839" 18:23:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459839" 18:23:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459839" 18:23:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459839" 18:23:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459839" 18:23:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459839" 18:23:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459839" 18:23:59 INFO [http_cmd]: http status code OK 18:23:59 DEBUG --- status code --- 18:23:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:23:59 DEBUG --- http response --- 18:23:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:23:59 INFO [http_cmd]: http status code OK 18:23:59 DEBUG --- status code --- 18:23:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:23:59 DEBUG --- http response --- 18:23:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:23:59 INFO [http_cmd]: http status code OK 18:23:59 DEBUG --- status code --- 18:23:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:23:59 DEBUG --- http response --- 18:23:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:23:59 ERROR [http_cmd]: ERROR 18:23:59 ERROR --- status code --- 18:23:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:23:59 ERROR --- http response --- 18:23:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:23:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:23:59 INFO [http_cmd]: http status code OK 18:23:59 INFO [http_cmd]: http status code OK 18:23:59 DEBUG --- status code --- 18:23:59 INFO [http_cmd]: http status code OK 18:23:59 INFO [http_cmd]: http status code OK 18:23:59 INFO [http_cmd]: http status code OK 18:23:59 INFO [http_cmd]: http status code OK 18:23:59 DEBUG --- status code --- 18:23:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:23:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459839" 18:23:59 DEBUG --- status code --- 18:23:59 DEBUG --- status code --- 18:23:59 INFO [http_cmd]: http status code OK 18:23:59 DEBUG --- status code --- 18:23:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459839" 18:23:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459839" 18:23:59 INFO [http_cmd]: http status code OK 18:23:59 DEBUG --- status code --- 18:23:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734459839" 18:23:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:23:59 DEBUG --- http response --- 18:23:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:23:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:23:59 DEBUG --- status code --- 18:23:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:23:59 DEBUG --- status code --- 18:23:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:23:59 DEBUG --- http response --- 18:23:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:23:59 DEBUG --- http response --- 18:23:59 DEBUG --- http response --- 18:23:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:23:59 DEBUG --- http response --- 18:23:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:23:59 DEBUG --- http response --- 18:23:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:23:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:23:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:23:59 DEBUG --- http response --- 18:23:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:23:59 DEBUG --- http response --- 18:23:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:23:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:23:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:23:59 INFO [http_cmd]: http status code OK 18:23:59 DEBUG --- status code --- 18:23:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:23:59 DEBUG --- http response --- 18:23:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:23:59 INFO [http_cmd]: http status code OK 18:23:59 INFO [http_cmd]: http status code OK 18:23:59 DEBUG --- status code --- 18:23:59 INFO [http_cmd]: http status code OK 18:23:59 DEBUG --- status code --- 18:23:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:23:59 DEBUG --- status code --- 18:23:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:23:59 DEBUG --- http response --- 18:23:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:23:59 DEBUG --- http response --- 18:23:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:23:59 DEBUG --- http response --- 18:23:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:23:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:23:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459839" 18:23:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459839" 18:23:59 INFO [http_cmd]: http status code OK 18:23:59 DEBUG --- status code --- 18:23:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:23:59 DEBUG --- http response --- 18:23:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:23:59 DEBUG ----- output ----- 18:23:59 DEBUG { 18:23:59 DEBUG "status": "success", 18:23:59 DEBUG "data": { 18:23:59 DEBUG "resultType": "vector", 18:23:59 DEBUG "result": [ 18:23:59 DEBUG { 18:23:59 DEBUG "metric": { 18:23:59 DEBUG "pod": "ds-cts-0" 18:23:59 DEBUG }, 18:23:59 DEBUG "value": [ 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG [...] 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:23:59 DEBUG }, 18:23:59 DEBUG "value": [ 18:23:59 DEBUG 1734459839, 18:23:59 DEBUG "0" 18:23:59 DEBUG ] 18:23:59 DEBUG } 18:23:59 DEBUG ] 18:23:59 DEBUG } 18:23:59 DEBUG } 18:23:59 INFO [http_cmd]: http status code OK 18:23:59 DEBUG --- status code --- 18:23:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:23:59 DEBUG --- http response --- 18:23:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:23:59 DEBUG ----- output ----- 18:23:59 DEBUG { 18:23:59 DEBUG "status": "success", 18:23:59 DEBUG "data": { 18:23:59 DEBUG "resultType": "vector", 18:23:59 DEBUG "result": [ 18:23:59 DEBUG { 18:23:59 DEBUG "metric": { 18:23:59 DEBUG "pod": "ds-cts-0" 18:23:59 DEBUG }, 18:23:59 DEBUG "value": [ 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG [...] 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:23:59 DEBUG }, 18:23:59 DEBUG "value": [ 18:23:59 DEBUG 1734459839, 18:23:59 DEBUG "0.0027781487609794264" 18:23:59 DEBUG ] 18:23:59 DEBUG } 18:23:59 DEBUG ] 18:23:59 DEBUG } 18:23:59 DEBUG } 18:23:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459839" 18:23:59 INFO [http_cmd]: http status code OK 18:23:59 DEBUG --- status code --- 18:23:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:23:59 DEBUG --- http response --- 18:23:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:23:59 DEBUG ----- output ----- 18:23:59 DEBUG { 18:23:59 DEBUG "status": "success", 18:23:59 DEBUG "data": { 18:23:59 DEBUG "resultType": "vector", 18:23:59 DEBUG "result": [ 18:23:59 DEBUG { 18:23:59 DEBUG "metric": { 18:23:59 DEBUG "instance": "10.142.0.11:9100" 18:23:59 DEBUG }, 18:23:59 DEBUG "value": [ 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG [...] 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG "instance": "10.142.0.51:9100" 18:23:59 DEBUG }, 18:23:59 DEBUG "value": [ 18:23:59 DEBUG 1734459839, 18:23:59 DEBUG "128185.29999999999" 18:23:59 DEBUG ] 18:23:59 DEBUG } 18:23:59 DEBUG ] 18:23:59 DEBUG } 18:23:59 DEBUG } 18:23:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459839" 18:23:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734459839" 18:23:59 INFO [http_cmd]: http status code OK 18:23:59 DEBUG --- status code --- 18:23:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:23:59 DEBUG --- http response --- 18:23:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:23:59 DEBUG ----- output ----- 18:23:59 DEBUG { 18:23:59 DEBUG "status": "success", 18:23:59 DEBUG "data": { 18:23:59 DEBUG "resultType": "vector", 18:23:59 DEBUG "result": [ 18:23:59 DEBUG { 18:23:59 DEBUG "metric": { 18:23:59 DEBUG "pod": "ds-cts-0" 18:23:59 DEBUG }, 18:23:59 DEBUG "value": [ 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG [...] 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:23:59 DEBUG }, 18:23:59 DEBUG "value": [ 18:23:59 DEBUG 1734459839, 18:23:59 DEBUG "710852608" 18:23:59 DEBUG ] 18:23:59 DEBUG } 18:23:59 DEBUG ] 18:23:59 DEBUG } 18:23:59 DEBUG } 18:23:59 INFO [http_cmd]: http status code OK 18:23:59 DEBUG --- status code --- 18:23:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:23:59 DEBUG --- http response --- 18:23:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:23:59 DEBUG ----- output ----- 18:23:59 DEBUG { 18:23:59 DEBUG "status": "success", 18:23:59 DEBUG "data": { 18:23:59 DEBUG "resultType": "vector", 18:23:59 DEBUG "result": [ 18:23:59 DEBUG { 18:23:59 DEBUG "metric": { 18:23:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:23:59 DEBUG }, 18:23:59 DEBUG "value": [ 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG [...] 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:23:59 DEBUG }, 18:23:59 DEBUG "value": [ 18:23:59 DEBUG 1734459839, 18:23:59 DEBUG "1.0558741363721609" 18:23:59 DEBUG ] 18:23:59 DEBUG } 18:23:59 DEBUG ] 18:23:59 DEBUG } 18:23:59 DEBUG } 18:23:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459839" 18:23:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459839" 18:23:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734459839" 18:23:59 INFO [http_cmd]: http status code OK 18:23:59 DEBUG --- status code --- 18:23:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:23:59 DEBUG --- http response --- 18:23:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:23:59 DEBUG ----- output ----- 18:23:59 DEBUG { 18:23:59 DEBUG "status": "success", 18:23:59 DEBUG "data": { 18:23:59 DEBUG "resultType": "vector", 18:23:59 DEBUG "result": [ 18:23:59 DEBUG { 18:23:59 DEBUG "metric": { 18:23:59 DEBUG "instance": "10.142.0.11:9100" 18:23:59 DEBUG }, 18:23:59 DEBUG "value": [ 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG [...] 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG "instance": "10.142.0.51:9100" 18:23:59 DEBUG }, 18:23:59 DEBUG "value": [ 18:23:59 DEBUG 1734459839, 18:23:59 DEBUG "126504.29999999999" 18:23:59 DEBUG ] 18:23:59 DEBUG } 18:23:59 DEBUG ] 18:23:59 DEBUG } 18:23:59 DEBUG } 18:23:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459839" 18:23:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459839" 18:23:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459839" 18:23:59 INFO [http_cmd]: http status code OK 18:23:59 DEBUG --- status code --- 18:23:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:23:59 DEBUG --- http response --- 18:23:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:23:59 INFO [http_cmd]: http status code OK 18:23:59 DEBUG ----- output ----- 18:23:59 DEBUG --- status code --- 18:23:59 DEBUG { 18:23:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:23:59 DEBUG "status": "success", 18:23:59 DEBUG --- http response --- 18:23:59 DEBUG "data": { 18:23:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:23:59 DEBUG "resultType": "vector", 18:23:59 DEBUG ----- output ----- 18:23:59 DEBUG "result": [ 18:23:59 DEBUG { 18:23:59 DEBUG { 18:23:59 DEBUG "status": "success", 18:23:59 DEBUG "metric": { 18:23:59 DEBUG "data": { 18:23:59 DEBUG "instance": "10.142.0.11:9100" 18:23:59 DEBUG "resultType": "vector", 18:23:59 DEBUG }, 18:23:59 DEBUG "result": [ 18:23:59 DEBUG "value": [ 18:23:59 DEBUG { 18:23:59 DEBUG 18:23:59 DEBUG "metric": { 18:23:59 DEBUG 18:23:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:23:59 DEBUG 18:23:59 DEBUG }, 18:23:59 DEBUG [...] 18:23:59 DEBUG "value": [ 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG "instance": "10.142.0.51:9100" 18:23:59 DEBUG [...] 18:23:59 DEBUG }, 18:23:59 DEBUG 18:23:59 DEBUG "value": [ 18:23:59 INFO [http_cmd]: http status code OK 18:23:59 DEBUG 18:23:59 INFO [http_cmd]: http status code OK 18:23:59 INFO [http_cmd]: http status code OK 18:23:59 DEBUG 1734459839, 18:23:59 DEBUG --- status code --- 18:23:59 DEBUG 18:23:59 DEBUG --- status code --- 18:23:59 DEBUG --- status code --- 18:23:59 DEBUG "26760.533333333333" 18:23:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:23:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:23:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:23:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:23:59 DEBUG ] 18:23:59 DEBUG --- http response --- 18:23:59 DEBUG }, 18:23:59 DEBUG --- http response --- 18:23:59 DEBUG --- http response --- 18:23:59 DEBUG } 18:23:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:23:59 DEBUG "value": [ 18:23:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:23:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:23:59 DEBUG ] 18:23:59 DEBUG ----- output ----- 18:23:59 DEBUG 1734459839, 18:23:59 DEBUG ----- output ----- 18:23:59 DEBUG ----- output ----- 18:23:59 DEBUG } 18:23:59 DEBUG { 18:23:59 DEBUG "2929852416" 18:23:59 DEBUG { 18:23:59 DEBUG { 18:23:59 DEBUG } 18:23:59 DEBUG "status": "success", 18:23:59 DEBUG ] 18:23:59 DEBUG "status": "success", 18:23:59 DEBUG "status": "success", 18:23:59 DEBUG "data": { 18:23:59 DEBUG } 18:23:59 DEBUG "data": { 18:23:59 DEBUG "data": { 18:23:59 DEBUG "resultType": "vector", 18:23:59 DEBUG ] 18:23:59 DEBUG "resultType": "vector", 18:23:59 DEBUG "resultType": "vector", 18:23:59 DEBUG "result": [ 18:23:59 DEBUG } 18:23:59 DEBUG "result": [ 18:23:59 DEBUG "result": [ 18:23:59 DEBUG { 18:23:59 DEBUG } 18:23:59 DEBUG { 18:23:59 DEBUG { 18:23:59 DEBUG "metric": { 18:23:59 DEBUG "metric": { 18:23:59 DEBUG "metric": { 18:23:59 DEBUG "instance": "10.142.0.11:9100" 18:23:59 DEBUG "pod": "ds-cts-0" 18:23:59 DEBUG "instance": "10.142.0.11:9100" 18:23:59 DEBUG }, 18:23:59 DEBUG }, 18:23:59 DEBUG }, 18:23:59 DEBUG "value": [ 18:23:59 DEBUG "value": [ 18:23:59 DEBUG "value": [ 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG [...] 18:23:59 DEBUG [...] 18:23:59 DEBUG [...] 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG 18:23:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:23:59 DEBUG "instance": "10.142.0.51:9100" 18:23:59 DEBUG 18:23:59 DEBUG }, 18:23:59 DEBUG }, 18:23:59 DEBUG "instance": "10.142.0.51:9100" 18:23:59 DEBUG "value": [ 18:23:59 DEBUG "value": [ 18:23:59 DEBUG }, 18:23:59 DEBUG 1734459839, 18:23:59 DEBUG 1734459839, 18:23:59 DEBUG "value": [ 18:23:59 DEBUG "1422.1736493449139" 18:23:59 DEBUG "0.0004666666666688949" 18:23:59 DEBUG 1734459839, 18:23:59 DEBUG ] 18:23:59 DEBUG ] 18:23:59 DEBUG "0.0003333333333333854" 18:23:59 DEBUG } 18:23:59 DEBUG } 18:23:59 DEBUG ] 18:23:59 DEBUG ] 18:23:59 DEBUG } 18:23:59 DEBUG } 18:23:59 DEBUG ] 18:23:59 DEBUG } 18:23:59 DEBUG ] 18:23:59 DEBUG } 18:23:59 DEBUG } 18:23:59 DEBUG } 18:23:59 DEBUG } 18:24:00 INFO 18:24:00 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:24:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:24:00 INFO [loop_until]: OK (rc = 0) 18:24:00 DEBUG --- stdout --- 18:24:00 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 8m 512Mi ds-idrepo-0 7m 711Mi rcs-5d696ff5b-7ttvr 6m 297Mi rcs-5d696ff5b-wwlbr 3m 338Mi 18:24:00 DEBUG --- stderr --- 18:24:00 DEBUG 18:24:00 INFO 18:24:00 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:24:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:24:00 INFO [loop_until]: OK (rc = 0) 18:24:00 DEBUG --- stdout --- 18:24:00 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 100m 0% 2262Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1133m 3% 3775Mi 13% 18:24:00 DEBUG --- stderr --- 18:24:00 DEBUG 18:24:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459898" 18:24:58 INFO [http_cmd]: http status code OK 18:24:58 DEBUG --- status code --- 18:24:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:24:58 DEBUG --- http response --- 18:24:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:24:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459899" 18:24:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734459899" 18:24:59 INFO [http_cmd]: http status code OK 18:24:59 INFO [http_cmd]: http status code OK 18:24:59 DEBUG --- status code --- 18:24:59 DEBUG --- status code --- 18:24:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:24:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:24:59 DEBUG --- http response --- 18:24:59 DEBUG --- http response --- 18:24:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:24:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:24:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459899" 18:24:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459899" 18:24:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459899" 18:24:59 INFO [http_cmd]: http status code OK 18:24:59 DEBUG --- status code --- 18:24:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:24:59 DEBUG --- http response --- 18:24:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:24:59 INFO [http_cmd]: http status code OK 18:24:59 DEBUG --- status code --- 18:24:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:24:59 DEBUG --- http response --- 18:24:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:24:59 INFO [http_cmd]: http status code OK 18:24:59 DEBUG --- status code --- 18:24:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:24:59 DEBUG --- http response --- 18:24:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:24:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459899" 18:24:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459899" 18:24:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459899" 18:24:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459899" 18:24:59 INFO [http_cmd]: http status code OK 18:24:59 DEBUG --- status code --- 18:24:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:24:59 INFO [http_cmd]: http status code OK 18:24:59 DEBUG --- http response --- 18:24:59 DEBUG --- status code --- 18:24:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:24:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:24:59 DEBUG --- http response --- 18:24:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:24:59 INFO [http_cmd]: http status code OK 18:24:59 DEBUG --- status code --- 18:24:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:24:59 DEBUG --- http response --- 18:24:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:24:59 INFO [http_cmd]: http status code OK 18:24:59 DEBUG --- status code --- 18:24:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:24:59 DEBUG --- http response --- 18:24:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:24:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459899" 18:24:59 INFO [http_cmd]: http status code OK 18:24:59 DEBUG --- status code --- 18:24:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:24:59 DEBUG --- http response --- 18:24:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:24:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734459899" 18:24:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459899" 18:24:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459899" 18:24:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734459899" 18:24:59 INFO [http_cmd]: http status code OK 18:24:59 DEBUG --- status code --- 18:24:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:24:59 DEBUG --- http response --- 18:24:59 ERROR [http_cmd]: ERROR 18:24:59 INFO [http_cmd]: http status code OK 18:24:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:24:59 ERROR --- status code --- 18:24:59 DEBUG --- status code --- 18:24:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:24:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:24:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459899" 18:24:59 ERROR --- http response --- 18:24:59 DEBUG --- http response --- 18:24:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:24:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459899" 18:24:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:24:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734459899" 18:24:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:24:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459899" 18:24:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459899" 18:24:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459899" 18:24:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459899" 18:24:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459899" 18:24:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734459899" 18:24:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459899" 18:24:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459899" 18:24:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459899" 18:24:59 INFO [http_cmd]: http status code OK 18:24:59 INFO [http_cmd]: http status code OK 18:24:59 DEBUG --- status code --- 18:24:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:24:59 DEBUG --- http response --- 18:24:59 DEBUG --- status code --- 18:24:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:24:59 DEBUG --- http response --- 18:24:59 INFO [http_cmd]: http status code OK 18:24:59 INFO [http_cmd]: http status code OK 18:24:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:24:59 INFO [http_cmd]: http status code OK 18:24:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:24:59 INFO [http_cmd]: http status code OK 18:24:59 DEBUG --- status code --- 18:24:59 DEBUG --- status code --- 18:24:59 INFO [http_cmd]: http status code OK 18:24:59 DEBUG --- status code --- 18:24:59 INFO [http_cmd]: http status code OK 18:24:59 INFO [http_cmd]: http status code OK 18:24:59 INFO [http_cmd]: http status code OK 18:24:59 DEBUG --- status code --- 18:24:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:24:59 INFO [http_cmd]: http status code OK 18:24:59 INFO [http_cmd]: http status code OK 18:24:59 INFO [http_cmd]: http status code OK 18:24:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:24:59 DEBUG --- status code --- 18:24:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:24:59 DEBUG --- status code --- 18:24:59 DEBUG --- status code --- 18:24:59 DEBUG --- status code --- 18:24:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:24:59 DEBUG --- http response --- 18:24:59 DEBUG --- status code --- 18:24:59 DEBUG --- status code --- 18:24:59 DEBUG --- status code --- 18:24:59 DEBUG --- http response --- 18:24:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:24:59 DEBUG --- http response --- 18:24:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:24:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:24:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:24:59 DEBUG --- http response --- 18:24:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:24:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:24:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:24:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:24:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:24:59 DEBUG --- http response --- 18:24:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:24:59 DEBUG --- http response --- 18:24:59 DEBUG --- http response --- 18:24:59 DEBUG --- http response --- 18:24:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:24:59 DEBUG --- http response --- 18:24:59 DEBUG --- http response --- 18:24:59 DEBUG --- http response --- 18:24:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:24:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:24:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:24:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:24:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:24:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:24:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:24:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459899" 18:24:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459899" 18:24:59 INFO [http_cmd]: http status code OK 18:24:59 INFO [http_cmd]: http status code OK 18:24:59 DEBUG --- status code --- 18:24:59 DEBUG --- status code --- 18:24:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:24:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:24:59 DEBUG --- http response --- 18:24:59 DEBUG --- http response --- 18:24:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:24:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:24:59 DEBUG ----- output ----- 18:24:59 DEBUG ----- output ----- 18:24:59 DEBUG { 18:24:59 DEBUG { 18:24:59 DEBUG "status": "success", 18:24:59 DEBUG "status": "success", 18:24:59 DEBUG "data": { 18:24:59 DEBUG "data": { 18:24:59 DEBUG "resultType": "vector", 18:24:59 DEBUG "resultType": "vector", 18:24:59 DEBUG "result": [ 18:24:59 DEBUG "result": [ 18:24:59 DEBUG { 18:24:59 DEBUG { 18:24:59 DEBUG "metric": { 18:24:59 DEBUG "metric": { 18:24:59 DEBUG "pod": "ds-idrepo-0" 18:24:59 DEBUG "pod": "ds-idrepo-0" 18:24:59 DEBUG }, 18:24:59 DEBUG }, 18:24:59 DEBUG "value": [ 18:24:59 DEBUG "value": [ 18:24:59 DEBUG 18:24:59 DEBUG 18:24:59 DEBUG 18:24:59 DEBUG 18:24:59 DEBUG 18:24:59 DEBUG 18:24:59 DEBUG [...] 18:24:59 DEBUG [...] 18:24:59 DEBUG 18:24:59 DEBUG 18:24:59 DEBUG 18:24:59 DEBUG 18:24:59 DEBUG 18:24:59 DEBUG 18:24:59 DEBUG "pod": "ds-cts-0" 18:24:59 DEBUG "pod": "ds-cts-0" 18:24:59 DEBUG }, 18:24:59 DEBUG }, 18:24:59 DEBUG "value": [ 18:24:59 DEBUG "value": [ 18:24:59 DEBUG 1734459899, 18:24:59 DEBUG 1734459899, 18:24:59 DEBUG "0" 18:24:59 DEBUG "0.00689214108910875" 18:24:59 DEBUG ] 18:24:59 DEBUG ] 18:24:59 DEBUG } 18:24:59 DEBUG } 18:24:59 DEBUG ] 18:24:59 DEBUG ] 18:24:59 DEBUG } 18:24:59 DEBUG } 18:24:59 DEBUG } 18:24:59 DEBUG } 18:24:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459899" 18:24:59 INFO [http_cmd]: http status code OK 18:24:59 DEBUG --- status code --- 18:24:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:24:59 DEBUG --- http response --- 18:24:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:24:59 DEBUG ----- output ----- 18:24:59 DEBUG { 18:24:59 DEBUG "status": "success", 18:24:59 DEBUG "data": { 18:24:59 DEBUG "resultType": "vector", 18:24:59 DEBUG "result": [ 18:24:59 DEBUG { 18:24:59 DEBUG "metric": { 18:24:59 DEBUG "instance": "10.142.0.11:9100" 18:24:59 DEBUG }, 18:24:59 DEBUG "value": [ 18:24:59 DEBUG 18:24:59 DEBUG 18:24:59 DEBUG 18:24:59 DEBUG [...] 18:24:59 DEBUG 18:24:59 DEBUG 18:24:59 DEBUG 18:24:59 DEBUG "instance": "10.142.0.51:9100" 18:24:59 DEBUG }, 18:24:59 DEBUG "value": [ 18:24:59 DEBUG 1734459899, 18:24:59 DEBUG "127586.8" 18:24:59 DEBUG ] 18:24:59 DEBUG } 18:24:59 DEBUG ] 18:24:59 DEBUG } 18:24:59 DEBUG } 18:24:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459899" 18:24:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734459899" 18:24:59 INFO [http_cmd]: http status code OK 18:24:59 DEBUG --- status code --- 18:24:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:24:59 DEBUG --- http response --- 18:24:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:24:59 DEBUG ----- output ----- 18:24:59 DEBUG { 18:24:59 DEBUG "status": "success", 18:24:59 DEBUG "data": { 18:24:59 DEBUG "resultType": "vector", 18:24:59 DEBUG "result": [ 18:24:59 DEBUG { 18:24:59 DEBUG "metric": { 18:24:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:24:59 DEBUG }, 18:24:59 DEBUG "value": [ 18:24:59 DEBUG 18:24:59 DEBUG 18:24:59 DEBUG 18:24:59 DEBUG [...] 18:24:59 DEBUG 18:24:59 DEBUG 18:24:59 DEBUG 18:24:59 INFO [http_cmd]: http status code OK 18:24:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:24:59 DEBUG --- status code --- 18:24:59 DEBUG }, 18:24:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:24:59 DEBUG "value": [ 18:24:59 DEBUG --- http response --- 18:24:59 DEBUG 1734459899, 18:24:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:24:59 DEBUG "1.0643555599788086" 18:24:59 DEBUG ----- output ----- 18:24:59 DEBUG ] 18:24:59 DEBUG { 18:24:59 DEBUG } 18:24:59 DEBUG "status": "success", 18:24:59 DEBUG ] 18:24:59 DEBUG "data": { 18:24:59 DEBUG } 18:24:59 DEBUG "resultType": "vector", 18:24:59 DEBUG } 18:24:59 DEBUG "result": [ 18:24:59 DEBUG { 18:24:59 DEBUG "metric": { 18:24:59 DEBUG "pod": "ds-cts-0" 18:24:59 DEBUG }, 18:24:59 DEBUG "value": [ 18:24:59 DEBUG 18:24:59 DEBUG 18:24:59 DEBUG 18:24:59 DEBUG [...] 18:24:59 DEBUG 18:24:59 DEBUG 18:24:59 DEBUG 18:24:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:24:59 DEBUG }, 18:24:59 DEBUG "value": [ 18:24:59 DEBUG 1734459899, 18:24:59 DEBUG "710963200" 18:24:59 DEBUG ] 18:24:59 DEBUG } 18:24:59 DEBUG ] 18:24:59 DEBUG } 18:24:59 DEBUG } 18:24:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459899" 18:24:59 INFO [http_cmd]: http status code OK 18:24:59 DEBUG --- status code --- 18:24:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:24:59 DEBUG --- http response --- 18:24:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:24:59 DEBUG ----- output ----- 18:24:59 DEBUG { 18:24:59 DEBUG "status": "success", 18:24:59 DEBUG "data": { 18:24:59 DEBUG "resultType": "vector", 18:24:59 DEBUG "result": [ 18:24:59 DEBUG { 18:24:59 DEBUG "metric": { 18:24:59 DEBUG "instance": "10.142.0.11:9100" 18:24:59 DEBUG }, 18:24:59 DEBUG "value": [ 18:24:59 DEBUG 18:24:59 DEBUG 18:24:59 DEBUG 18:24:59 DEBUG [...] 18:24:59 DEBUG 18:24:59 DEBUG 18:24:59 DEBUG 18:24:59 DEBUG "instance": "10.142.0.51:9100" 18:24:59 DEBUG }, 18:24:59 DEBUG "value": [ 18:24:59 DEBUG 1734459899, 18:24:59 DEBUG "130236.2" 18:24:59 DEBUG ] 18:24:59 DEBUG } 18:24:59 DEBUG ] 18:24:59 DEBUG } 18:24:59 DEBUG } 18:24:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459899" 18:24:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734459899" 18:24:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459899" 18:24:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459899" 18:24:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459899" 18:24:59 INFO [http_cmd]: http status code OK 18:24:59 DEBUG --- status code --- 18:24:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:24:59 DEBUG --- http response --- 18:24:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:24:59 DEBUG ----- output ----- 18:24:59 DEBUG { 18:24:59 DEBUG "status": "success", 18:24:59 DEBUG "data": { 18:24:59 DEBUG "resultType": "vector", 18:24:59 DEBUG "result": [ 18:24:59 DEBUG { 18:24:59 DEBUG "metric": { 18:24:59 DEBUG "instance": "10.142.0.11:9100" 18:24:59 DEBUG }, 18:24:59 DEBUG "value": [ 18:24:59 DEBUG 18:24:59 DEBUG 18:24:59 DEBUG 18:24:59 DEBUG [...] 18:24:59 DEBUG 18:24:59 INFO [http_cmd]: http status code OK 18:24:59 DEBUG 18:24:59 DEBUG 18:24:59 DEBUG --- status code --- 18:24:59 DEBUG "instance": "10.142.0.51:9100" 18:24:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:24:59 DEBUG }, 18:24:59 DEBUG --- http response --- 18:24:59 DEBUG "value": [ 18:24:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:24:59 DEBUG 1734459899, 18:24:59 DEBUG ----- output ----- 18:24:59 DEBUG "21299.2" 18:24:59 DEBUG { 18:24:59 DEBUG ] 18:24:59 INFO [http_cmd]: http status code OK 18:24:59 DEBUG "status": "success", 18:24:59 DEBUG } 18:24:59 DEBUG --- status code --- 18:24:59 DEBUG "data": { 18:24:59 DEBUG ] 18:24:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:24:59 DEBUG "resultType": "vector", 18:24:59 DEBUG } 18:24:59 DEBUG --- http response --- 18:24:59 DEBUG "result": [ 18:24:59 DEBUG } 18:24:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:24:59 DEBUG { 18:24:59 DEBUG ----- output ----- 18:24:59 DEBUG "metric": { 18:24:59 DEBUG { 18:24:59 INFO [http_cmd]: http status code OK 18:24:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:24:59 DEBUG "status": "success", 18:24:59 DEBUG --- status code --- 18:24:59 DEBUG }, 18:24:59 DEBUG "data": { 18:24:59 DEBUG "resultType": "vector", 18:24:59 DEBUG "value": [ 18:24:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:24:59 DEBUG "result": [ 18:24:59 DEBUG 18:24:59 DEBUG --- http response --- 18:24:59 DEBUG { 18:24:59 DEBUG 18:24:59 DEBUG "metric": { 18:24:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:24:59 DEBUG 18:24:59 DEBUG "instance": "10.142.0.11:9100" 18:24:59 DEBUG ----- output ----- 18:24:59 DEBUG [...] 18:24:59 DEBUG }, 18:24:59 DEBUG { 18:24:59 DEBUG 18:24:59 DEBUG "value": [ 18:24:59 DEBUG "status": "success", 18:24:59 DEBUG 18:24:59 DEBUG 18:24:59 INFO [http_cmd]: http status code OK 18:24:59 DEBUG "data": { 18:24:59 DEBUG 18:24:59 DEBUG 18:24:59 DEBUG --- status code --- 18:24:59 DEBUG "resultType": "vector", 18:24:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:24:59 DEBUG 18:24:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:24:59 DEBUG "result": [ 18:24:59 DEBUG }, 18:24:59 DEBUG [...] 18:24:59 DEBUG --- http response --- 18:24:59 DEBUG { 18:24:59 DEBUG "value": [ 18:24:59 DEBUG 18:24:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:24:59 DEBUG "metric": { 18:24:59 DEBUG 1734459899, 18:24:59 DEBUG 18:24:59 DEBUG ----- output ----- 18:24:59 DEBUG "instance": "10.142.0.11:9100" 18:24:59 DEBUG "2939879424" 18:24:59 DEBUG 18:24:59 DEBUG { 18:24:59 DEBUG }, 18:24:59 DEBUG ] 18:24:59 DEBUG "instance": "10.142.0.51:9100" 18:24:59 DEBUG "status": "success", 18:24:59 DEBUG "value": [ 18:24:59 DEBUG } 18:24:59 DEBUG }, 18:24:59 DEBUG "data": { 18:24:59 DEBUG 18:24:59 DEBUG ] 18:24:59 DEBUG "value": [ 18:24:59 DEBUG "resultType": "vector", 18:24:59 DEBUG 18:24:59 DEBUG } 18:24:59 DEBUG 1734459899, 18:24:59 DEBUG "result": [ 18:24:59 DEBUG 18:24:59 DEBUG } 18:24:59 DEBUG "0.000400000000003331" 18:24:59 DEBUG { 18:24:59 DEBUG [...] 18:24:59 DEBUG ] 18:24:59 DEBUG "metric": { 18:24:59 DEBUG 18:24:59 DEBUG } 18:24:59 DEBUG "pod": "ds-idrepo-0" 18:24:59 DEBUG 18:24:59 DEBUG ] 18:24:59 DEBUG }, 18:24:59 DEBUG 18:24:59 DEBUG } 18:24:59 DEBUG "value": [ 18:24:59 DEBUG "instance": "10.142.0.51:9100" 18:24:59 DEBUG } 18:24:59 DEBUG 18:24:59 DEBUG }, 18:24:59 DEBUG 18:24:59 DEBUG "value": [ 18:24:59 DEBUG 18:24:59 DEBUG 1734459899, 18:24:59 DEBUG [...] 18:24:59 DEBUG "0.000333333333333267" 18:24:59 DEBUG 18:24:59 DEBUG ] 18:24:59 DEBUG 18:24:59 DEBUG } 18:24:59 DEBUG 18:24:59 DEBUG ] 18:24:59 DEBUG "pod": "ds-cts-0" 18:24:59 DEBUG } 18:24:59 DEBUG }, 18:24:59 DEBUG } 18:24:59 DEBUG "value": [ 18:24:59 DEBUG 1734459899, 18:24:59 DEBUG "633.6633663366338" 18:24:59 DEBUG ] 18:24:59 DEBUG } 18:24:59 DEBUG ] 18:24:59 DEBUG } 18:24:59 DEBUG } 18:25:00 INFO 18:25:00 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:25:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:25:00 INFO [loop_until]: OK (rc = 0) 18:25:00 DEBUG --- stdout --- 18:25:00 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 7m 515Mi ds-idrepo-0 6m 710Mi rcs-5d696ff5b-7ttvr 3m 297Mi rcs-5d696ff5b-wwlbr 2m 338Mi 18:25:00 DEBUG --- stderr --- 18:25:00 DEBUG 18:25:00 INFO 18:25:00 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:25:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:25:00 INFO [loop_until]: OK (rc = 0) 18:25:00 DEBUG --- stdout --- 18:25:00 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 116m 0% 2268Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1129m 3% 3775Mi 13% 18:25:00 DEBUG --- stderr --- 18:25:00 DEBUG 18:25:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459958" 18:25:58 INFO [http_cmd]: http status code OK 18:25:58 DEBUG --- status code --- 18:25:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:25:58 DEBUG --- http response --- 18:25:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:25:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734459959" 18:25:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459959" 18:25:59 INFO [http_cmd]: http status code OK 18:25:59 DEBUG --- status code --- 18:25:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:25:59 DEBUG --- http response --- 18:25:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:25:59 INFO [http_cmd]: http status code OK 18:25:59 DEBUG --- status code --- 18:25:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:25:59 DEBUG --- http response --- 18:25:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:25:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459959" 18:25:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459959" 18:25:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459959" 18:25:59 INFO [http_cmd]: http status code OK 18:25:59 DEBUG --- status code --- 18:25:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:25:59 DEBUG --- http response --- 18:25:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:25:59 INFO [http_cmd]: http status code OK 18:25:59 DEBUG --- status code --- 18:25:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:25:59 DEBUG --- http response --- 18:25:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:25:59 INFO [http_cmd]: http status code OK 18:25:59 DEBUG --- status code --- 18:25:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:25:59 DEBUG --- http response --- 18:25:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:25:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459959" 18:25:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459959" 18:25:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459959" 18:25:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459959" 18:25:59 INFO [http_cmd]: http status code OK 18:25:59 DEBUG --- status code --- 18:25:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:25:59 DEBUG --- http response --- 18:25:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:25:59 INFO [http_cmd]: http status code OK 18:25:59 DEBUG --- status code --- 18:25:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:25:59 DEBUG --- http response --- 18:25:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:25:59 INFO [http_cmd]: http status code OK 18:25:59 DEBUG --- status code --- 18:25:59 INFO [http_cmd]: http status code OK 18:25:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:25:59 DEBUG --- status code --- 18:25:59 DEBUG --- http response --- 18:25:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:25:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:25:59 DEBUG --- http response --- 18:25:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:25:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459959" 18:25:59 INFO [http_cmd]: http status code OK 18:25:59 DEBUG --- status code --- 18:25:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:25:59 DEBUG --- http response --- 18:25:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:25:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459959" 18:25:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734459959" 18:25:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459959" 18:25:59 INFO [http_cmd]: http status code OK 18:25:59 DEBUG --- status code --- 18:25:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:25:59 DEBUG --- http response --- 18:25:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:25:59 ERROR [http_cmd]: ERROR 18:25:59 ERROR --- status code --- 18:25:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:25:59 ERROR --- http response --- 18:25:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:25:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:25:59 INFO [http_cmd]: http status code OK 18:25:59 DEBUG --- status code --- 18:25:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:25:59 DEBUG --- http response --- 18:25:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:25:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734459959" 18:25:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459959" 18:25:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459959" 18:25:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734459959" 18:25:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459959" 18:25:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459959" 18:25:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459959" 18:25:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734459959" 18:25:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459959" 18:25:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459959" 18:25:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459959" 18:25:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459959" 18:25:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459959" 18:25:59 INFO [http_cmd]: http status code OK 18:25:59 INFO [http_cmd]: http status code OK 18:25:59 DEBUG --- status code --- 18:25:59 INFO [http_cmd]: http status code OK 18:25:59 DEBUG --- status code --- 18:25:59 INFO [http_cmd]: http status code OK 18:25:59 INFO [http_cmd]: http status code OK 18:25:59 INFO [http_cmd]: http status code OK 18:25:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:25:59 INFO [http_cmd]: http status code OK 18:25:59 INFO [http_cmd]: http status code OK 18:25:59 DEBUG --- status code --- 18:25:59 INFO [http_cmd]: http status code OK 18:25:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:25:59 DEBUG --- status code --- 18:25:59 DEBUG --- status code --- 18:25:59 DEBUG --- status code --- 18:25:59 DEBUG --- http response --- 18:25:59 DEBUG --- status code --- 18:25:59 DEBUG --- status code --- 18:25:59 INFO [http_cmd]: http status code OK 18:25:59 INFO [http_cmd]: http status code OK 18:25:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:25:59 DEBUG --- status code --- 18:25:59 DEBUG --- http response --- 18:25:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:25:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:25:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:25:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:25:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:25:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:25:59 DEBUG --- status code --- 18:25:59 DEBUG --- status code --- 18:25:59 DEBUG --- http response --- 18:25:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:25:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:25:59 DEBUG --- http response --- 18:25:59 DEBUG --- http response --- 18:25:59 DEBUG --- http response --- 18:25:59 DEBUG --- http response --- 18:25:59 DEBUG --- http response --- 18:25:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:25:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:25:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:25:59 DEBUG --- http response --- 18:25:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:25:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:25:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:25:59 INFO [http_cmd]: http status code OK 18:25:59 INFO [http_cmd]: http status code OK 18:25:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:25:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:25:59 DEBUG --- http response --- 18:25:59 DEBUG --- http response --- 18:25:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:25:59 DEBUG --- status code --- 18:25:59 DEBUG --- status code --- 18:25:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:25:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459959" 18:25:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:25:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459959" 18:25:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:25:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:25:59 DEBUG --- http response --- 18:25:59 DEBUG --- http response --- 18:25:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:25:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:25:59 INFO [http_cmd]: http status code OK 18:25:59 DEBUG --- status code --- 18:25:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:25:59 DEBUG --- http response --- 18:25:59 INFO [http_cmd]: http status code OK 18:25:59 DEBUG --- status code --- 18:25:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:25:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:25:59 DEBUG --- http response --- 18:25:59 DEBUG ----- output ----- 18:25:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:25:59 DEBUG { 18:25:59 DEBUG ----- output ----- 18:25:59 DEBUG "status": "success", 18:25:59 DEBUG { 18:25:59 DEBUG "data": { 18:25:59 DEBUG "status": "success", 18:25:59 DEBUG "resultType": "vector", 18:25:59 DEBUG "data": { 18:25:59 DEBUG "result": [ 18:25:59 DEBUG "resultType": "vector", 18:25:59 DEBUG { 18:25:59 DEBUG "result": [ 18:25:59 DEBUG "metric": { 18:25:59 DEBUG { 18:25:59 DEBUG "pod": "ds-cts-0" 18:25:59 DEBUG "metric": { 18:25:59 DEBUG }, 18:25:59 DEBUG "pod": "ds-cts-0" 18:25:59 DEBUG "value": [ 18:25:59 DEBUG }, 18:25:59 DEBUG 18:25:59 DEBUG "value": [ 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG [...] 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG [...] 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:25:59 DEBUG 18:25:59 DEBUG }, 18:25:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:25:59 DEBUG "value": [ 18:25:59 DEBUG }, 18:25:59 DEBUG 1734459959, 18:25:59 DEBUG "value": [ 18:25:59 DEBUG "0" 18:25:59 DEBUG 1734459959, 18:25:59 DEBUG ] 18:25:59 DEBUG "0.002733084268880248" 18:25:59 DEBUG } 18:25:59 DEBUG ] 18:25:59 DEBUG ] 18:25:59 DEBUG } 18:25:59 DEBUG } 18:25:59 DEBUG ] 18:25:59 DEBUG } 18:25:59 DEBUG } 18:25:59 DEBUG } 18:25:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459959" 18:25:59 INFO [http_cmd]: http status code OK 18:25:59 DEBUG --- status code --- 18:25:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:25:59 DEBUG --- http response --- 18:25:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:25:59 DEBUG ----- output ----- 18:25:59 DEBUG { 18:25:59 DEBUG "status": "success", 18:25:59 DEBUG "data": { 18:25:59 DEBUG "resultType": "vector", 18:25:59 DEBUG "result": [ 18:25:59 DEBUG { 18:25:59 DEBUG "metric": { 18:25:59 DEBUG "instance": "10.142.0.11:9100" 18:25:59 DEBUG }, 18:25:59 DEBUG "value": [ 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG [...] 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG "instance": "10.142.0.51:9100" 18:25:59 DEBUG }, 18:25:59 DEBUG "value": [ 18:25:59 DEBUG 1734459959, 18:25:59 DEBUG "126961.76666666668" 18:25:59 DEBUG ] 18:25:59 DEBUG } 18:25:59 DEBUG ] 18:25:59 DEBUG } 18:25:59 DEBUG } 18:25:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734459959" 18:25:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734459959" 18:25:59 INFO [http_cmd]: http status code OK 18:25:59 DEBUG --- status code --- 18:25:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:25:59 INFO [http_cmd]: http status code OK 18:25:59 DEBUG --- http response --- 18:25:59 DEBUG --- status code --- 18:25:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:25:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:25:59 DEBUG ----- output ----- 18:25:59 DEBUG --- http response --- 18:25:59 DEBUG { 18:25:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:25:59 DEBUG "status": "success", 18:25:59 DEBUG ----- output ----- 18:25:59 DEBUG "data": { 18:25:59 DEBUG { 18:25:59 DEBUG "resultType": "vector", 18:25:59 DEBUG "status": "success", 18:25:59 DEBUG "result": [ 18:25:59 DEBUG "data": { 18:25:59 DEBUG { 18:25:59 DEBUG "resultType": "vector", 18:25:59 DEBUG "metric": { 18:25:59 DEBUG "result": [ 18:25:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:25:59 DEBUG { 18:25:59 DEBUG }, 18:25:59 DEBUG "metric": { 18:25:59 DEBUG "value": [ 18:25:59 DEBUG "pod": "ds-cts-0" 18:25:59 DEBUG 18:25:59 DEBUG }, 18:25:59 DEBUG 18:25:59 DEBUG "value": [ 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG [...] 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG [...] 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:25:59 DEBUG 18:25:59 DEBUG }, 18:25:59 DEBUG 18:25:59 DEBUG "value": [ 18:25:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:25:59 DEBUG 1734459959, 18:25:59 DEBUG }, 18:25:59 DEBUG "1.058154623342377" 18:25:59 DEBUG "value": [ 18:25:59 DEBUG ] 18:25:59 DEBUG 1734459959, 18:25:59 DEBUG } 18:25:59 DEBUG "711012352" 18:25:59 DEBUG ] 18:25:59 DEBUG ] 18:25:59 DEBUG } 18:25:59 DEBUG } 18:25:59 DEBUG } 18:25:59 DEBUG ] 18:25:59 DEBUG } 18:25:59 DEBUG } 18:25:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459959" 18:25:59 INFO [http_cmd]: http status code OK 18:25:59 DEBUG --- status code --- 18:25:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:25:59 DEBUG --- http response --- 18:25:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:25:59 DEBUG ----- output ----- 18:25:59 DEBUG { 18:25:59 DEBUG "status": "success", 18:25:59 DEBUG "data": { 18:25:59 DEBUG "resultType": "vector", 18:25:59 DEBUG "result": [ 18:25:59 DEBUG { 18:25:59 DEBUG "metric": { 18:25:59 DEBUG "instance": "10.142.0.11:9100" 18:25:59 DEBUG }, 18:25:59 DEBUG "value": [ 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG [...] 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG "instance": "10.142.0.51:9100" 18:25:59 DEBUG }, 18:25:59 DEBUG "value": [ 18:25:59 DEBUG 1734459959, 18:25:59 DEBUG "124838.76666666666" 18:25:59 DEBUG ] 18:25:59 DEBUG } 18:25:59 DEBUG ] 18:25:59 DEBUG } 18:25:59 DEBUG } 18:25:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459959" 18:25:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734459959" 18:25:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459959" 18:25:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734459959" 18:25:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734459959" 18:25:59 INFO [http_cmd]: http status code OK 18:25:59 DEBUG --- status code --- 18:25:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:25:59 DEBUG --- http response --- 18:25:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:25:59 DEBUG ----- output ----- 18:25:59 DEBUG { 18:25:59 DEBUG "status": "success", 18:25:59 DEBUG "data": { 18:25:59 DEBUG "resultType": "vector", 18:25:59 DEBUG "result": [ 18:25:59 DEBUG { 18:25:59 DEBUG "metric": { 18:25:59 DEBUG "instance": "10.142.0.11:9100" 18:25:59 DEBUG }, 18:25:59 DEBUG "value": [ 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG [...] 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG "instance": "10.142.0.51:9100" 18:25:59 DEBUG }, 18:25:59 DEBUG "value": [ 18:25:59 DEBUG 1734459959, 18:25:59 DEBUG "19387.733333333334" 18:25:59 DEBUG ] 18:25:59 DEBUG } 18:25:59 DEBUG ] 18:25:59 DEBUG } 18:25:59 DEBUG } 18:25:59 INFO [http_cmd]: http status code OK 18:25:59 DEBUG --- status code --- 18:25:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:25:59 DEBUG --- http response --- 18:25:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:25:59 DEBUG ----- output ----- 18:25:59 DEBUG { 18:25:59 DEBUG "status": "success", 18:25:59 DEBUG "data": { 18:25:59 DEBUG "resultType": "vector", 18:25:59 DEBUG "result": [ 18:25:59 DEBUG { 18:25:59 DEBUG "metric": { 18:25:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:25:59 DEBUG }, 18:25:59 DEBUG "value": [ 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG [...] 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:25:59 DEBUG }, 18:25:59 DEBUG "value": [ 18:25:59 DEBUG 1734459959, 18:25:59 DEBUG "2947293184" 18:25:59 DEBUG ] 18:25:59 DEBUG } 18:25:59 DEBUG ] 18:25:59 DEBUG } 18:25:59 DEBUG } 18:25:59 INFO [http_cmd]: http status code OK 18:25:59 DEBUG --- status code --- 18:25:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:25:59 DEBUG --- http response --- 18:25:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:25:59 DEBUG ----- output ----- 18:25:59 DEBUG { 18:25:59 DEBUG "status": "success", 18:25:59 DEBUG "data": { 18:25:59 DEBUG "resultType": "vector", 18:25:59 DEBUG "result": [ 18:25:59 DEBUG { 18:25:59 DEBUG "metric": { 18:25:59 DEBUG "instance": "10.142.0.11:9100" 18:25:59 DEBUG }, 18:25:59 DEBUG "value": [ 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG [...] 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG "instance": "10.142.0.51:9100" 18:25:59 DEBUG }, 18:25:59 DEBUG "value": [ 18:25:59 DEBUG 1734459959, 18:25:59 DEBUG "0.0006666666666666524" 18:25:59 DEBUG ] 18:25:59 DEBUG } 18:25:59 DEBUG ] 18:25:59 DEBUG } 18:25:59 DEBUG } 18:25:59 INFO [http_cmd]: http status code OK 18:25:59 INFO [http_cmd]: http status code OK 18:25:59 DEBUG --- status code --- 18:25:59 DEBUG --- status code --- 18:25:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:25:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:25:59 DEBUG --- http response --- 18:25:59 DEBUG --- http response --- 18:25:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:25:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:25:59 DEBUG ----- output ----- 18:25:59 DEBUG ----- output ----- 18:25:59 DEBUG { 18:25:59 DEBUG { 18:25:59 DEBUG "status": "success", 18:25:59 DEBUG "status": "success", 18:25:59 DEBUG "data": { 18:25:59 DEBUG "data": { 18:25:59 DEBUG "resultType": "vector", 18:25:59 DEBUG "resultType": "vector", 18:25:59 DEBUG "result": [ 18:25:59 DEBUG "result": [ 18:25:59 DEBUG { 18:25:59 DEBUG { 18:25:59 DEBUG "metric": { 18:25:59 DEBUG "metric": { 18:25:59 DEBUG "instance": "10.142.0.11:9100" 18:25:59 DEBUG "pod": "ds-cts-0" 18:25:59 DEBUG }, 18:25:59 DEBUG }, 18:25:59 DEBUG "value": [ 18:25:59 DEBUG "value": [ 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG [...] 18:25:59 DEBUG [...] 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG 18:25:59 DEBUG "instance": "10.142.0.51:9100" 18:25:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:25:59 DEBUG }, 18:25:59 DEBUG }, 18:25:59 DEBUG "value": [ 18:25:59 DEBUG "value": [ 18:25:59 DEBUG 1734459959, 18:25:59 DEBUG 1734459959, 18:25:59 DEBUG "0.00036666666666983856" 18:25:59 DEBUG "1831.9253104422035" 18:25:59 DEBUG ] 18:25:59 DEBUG ] 18:25:59 DEBUG } 18:25:59 DEBUG } 18:25:59 DEBUG ] 18:25:59 DEBUG ] 18:25:59 DEBUG } 18:25:59 DEBUG } 18:25:59 DEBUG } 18:25:59 DEBUG } 18:26:00 INFO 18:26:00 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:26:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:26:00 INFO [loop_until]: OK (rc = 0) 18:26:00 DEBUG --- stdout --- 18:26:00 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 518Mi ds-idrepo-0 7m 710Mi rcs-5d696ff5b-7ttvr 3m 297Mi rcs-5d696ff5b-wwlbr 3m 338Mi 18:26:00 DEBUG --- stderr --- 18:26:00 DEBUG 18:26:00 INFO 18:26:00 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:26:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:26:00 INFO [loop_until]: OK (rc = 0) 18:26:00 DEBUG --- stdout --- 18:26:00 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 111m 0% 2280Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1136m 3% 3782Mi 13% 18:26:00 DEBUG --- stderr --- 18:26:00 DEBUG 18:26:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460018" 18:26:58 INFO [http_cmd]: http status code OK 18:26:58 DEBUG --- status code --- 18:26:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:26:58 DEBUG --- http response --- 18:26:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:26:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734460019" 18:26:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460019" 18:26:59 INFO [http_cmd]: http status code OK 18:26:59 DEBUG --- status code --- 18:26:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:26:59 DEBUG --- http response --- 18:26:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:26:59 INFO [http_cmd]: http status code OK 18:26:59 DEBUG --- status code --- 18:26:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:26:59 DEBUG --- http response --- 18:26:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:26:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460019" 18:26:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460019" 18:26:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460019" 18:26:59 INFO [http_cmd]: http status code OK 18:26:59 DEBUG --- status code --- 18:26:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:26:59 DEBUG --- http response --- 18:26:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:26:59 INFO [http_cmd]: http status code OK 18:26:59 INFO [http_cmd]: http status code OK 18:26:59 DEBUG --- status code --- 18:26:59 DEBUG --- status code --- 18:26:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:26:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:26:59 DEBUG --- http response --- 18:26:59 DEBUG --- http response --- 18:26:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:26:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:26:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460019" 18:26:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460019" 18:26:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460019" 18:26:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460019" 18:26:59 INFO [http_cmd]: http status code OK 18:26:59 DEBUG --- status code --- 18:26:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:26:59 DEBUG --- http response --- 18:26:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:26:59 INFO [http_cmd]: http status code OK 18:26:59 DEBUG --- status code --- 18:26:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:26:59 DEBUG --- http response --- 18:26:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:26:59 INFO [http_cmd]: http status code OK 18:26:59 DEBUG --- status code --- 18:26:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:26:59 INFO [http_cmd]: http status code OK 18:26:59 DEBUG --- http response --- 18:26:59 DEBUG --- status code --- 18:26:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:26:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:26:59 DEBUG --- http response --- 18:26:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:26:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460019" 18:26:59 INFO [http_cmd]: http status code OK 18:26:59 DEBUG --- status code --- 18:26:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:26:59 DEBUG --- http response --- 18:26:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:26:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460019" 18:26:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734460019" 18:26:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460019" 18:26:59 INFO [http_cmd]: http status code OK 18:26:59 DEBUG --- status code --- 18:26:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:26:59 DEBUG --- http response --- 18:26:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:26:59 ERROR [http_cmd]: ERROR 18:26:59 ERROR --- status code --- 18:26:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:26:59 ERROR --- http response --- 18:26:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:26:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:26:59 INFO [http_cmd]: http status code OK 18:26:59 DEBUG --- status code --- 18:26:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:26:59 DEBUG --- http response --- 18:26:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:26:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460019" 18:26:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734460019" 18:26:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460019" 18:26:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460019" 18:26:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460019" 18:26:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734460019" 18:26:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460019" 18:26:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734460019" 18:26:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460019" 18:26:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460019" 18:26:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460019" 18:26:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460019" 18:26:59 INFO [http_cmd]: http status code OK 18:26:59 DEBUG --- status code --- 18:26:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:26:59 DEBUG --- http response --- 18:26:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:26:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460019" 18:26:59 INFO [http_cmd]: http status code OK 18:26:59 DEBUG --- status code --- 18:26:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:26:59 DEBUG --- http response --- 18:26:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:26:59 INFO [http_cmd]: http status code OK 18:26:59 DEBUG --- status code --- 18:26:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:26:59 DEBUG --- http response --- 18:26:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:26:59 INFO [http_cmd]: http status code OK 18:26:59 DEBUG --- status code --- 18:26:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:26:59 DEBUG --- http response --- 18:26:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:26:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460019" 18:26:59 INFO [http_cmd]: http status code OK 18:26:59 INFO [http_cmd]: http status code OK 18:26:59 INFO [http_cmd]: http status code OK 18:26:59 INFO [http_cmd]: http status code OK 18:26:59 INFO [http_cmd]: http status code OK 18:26:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460019" 18:26:59 INFO [http_cmd]: http status code OK 18:26:59 INFO [http_cmd]: http status code OK 18:26:59 DEBUG --- status code --- 18:26:59 DEBUG --- status code --- 18:26:59 DEBUG --- status code --- 18:26:59 DEBUG --- status code --- 18:26:59 DEBUG --- status code --- 18:26:59 INFO [http_cmd]: http status code OK 18:26:59 INFO [http_cmd]: http status code OK 18:26:59 DEBUG --- status code --- 18:26:59 DEBUG --- status code --- 18:26:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:26:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:26:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:26:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:26:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:26:59 DEBUG --- status code --- 18:26:59 DEBUG --- status code --- 18:26:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:26:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:26:59 DEBUG --- http response --- 18:26:59 DEBUG --- http response --- 18:26:59 DEBUG --- http response --- 18:26:59 DEBUG --- http response --- 18:26:59 DEBUG --- http response --- 18:26:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:26:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:26:59 DEBUG --- http response --- 18:26:59 DEBUG --- http response --- 18:26:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:26:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:26:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:26:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:26:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:26:59 DEBUG --- http response --- 18:26:59 DEBUG --- http response --- 18:26:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:26:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:26:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:26:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:26:59 INFO [http_cmd]: http status code OK 18:26:59 DEBUG --- status code --- 18:26:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:26:59 DEBUG --- http response --- 18:26:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:26:59 DEBUG ----- output ----- 18:26:59 DEBUG { 18:26:59 DEBUG "status": "success", 18:26:59 DEBUG "data": { 18:26:59 DEBUG "resultType": "vector", 18:26:59 DEBUG "result": [ 18:26:59 DEBUG { 18:26:59 DEBUG "metric": { 18:26:59 DEBUG "pod": "ds-cts-0" 18:26:59 DEBUG }, 18:26:59 DEBUG "value": [ 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG [...] 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:26:59 DEBUG }, 18:26:59 DEBUG "value": [ 18:26:59 DEBUG 1734460019, 18:26:59 DEBUG "0.0031844124043140286" 18:26:59 DEBUG ] 18:26:59 DEBUG } 18:26:59 DEBUG ] 18:26:59 DEBUG } 18:26:59 DEBUG } 18:26:59 INFO [http_cmd]: http status code OK 18:26:59 DEBUG --- status code --- 18:26:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:26:59 DEBUG --- http response --- 18:26:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:26:59 DEBUG ----- output ----- 18:26:59 DEBUG { 18:26:59 DEBUG "status": "success", 18:26:59 DEBUG "data": { 18:26:59 DEBUG "resultType": "vector", 18:26:59 DEBUG "result": [ 18:26:59 DEBUG { 18:26:59 DEBUG "metric": { 18:26:59 DEBUG "pod": "ds-cts-0" 18:26:59 DEBUG }, 18:26:59 DEBUG "value": [ 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG [...] 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:26:59 DEBUG }, 18:26:59 DEBUG "value": [ 18:26:59 DEBUG 1734460019, 18:26:59 DEBUG "0" 18:26:59 DEBUG ] 18:26:59 DEBUG } 18:26:59 DEBUG ] 18:26:59 DEBUG } 18:26:59 DEBUG } 18:26:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460019" 18:26:59 INFO [http_cmd]: http status code OK 18:26:59 DEBUG --- status code --- 18:26:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:26:59 DEBUG --- http response --- 18:26:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:26:59 DEBUG ----- output ----- 18:26:59 DEBUG { 18:26:59 DEBUG "status": "success", 18:26:59 DEBUG "data": { 18:26:59 DEBUG "resultType": "vector", 18:26:59 DEBUG "result": [ 18:26:59 DEBUG { 18:26:59 DEBUG "metric": { 18:26:59 DEBUG "instance": "10.142.0.11:9100" 18:26:59 DEBUG }, 18:26:59 DEBUG "value": [ 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG [...] 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG "instance": "10.142.0.51:9100" 18:26:59 DEBUG }, 18:26:59 DEBUG "value": [ 18:26:59 DEBUG 1734460019, 18:26:59 DEBUG "127969.49999999997" 18:26:59 DEBUG ] 18:26:59 DEBUG } 18:26:59 DEBUG ] 18:26:59 DEBUG } 18:26:59 DEBUG } 18:26:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734460019" 18:26:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460019" 18:26:59 INFO [http_cmd]: http status code OK 18:26:59 DEBUG --- status code --- 18:26:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:26:59 DEBUG --- http response --- 18:26:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:26:59 DEBUG ----- output ----- 18:26:59 DEBUG { 18:26:59 DEBUG "status": "success", 18:26:59 DEBUG "data": { 18:26:59 DEBUG "resultType": "vector", 18:26:59 DEBUG "result": [ 18:26:59 DEBUG { 18:26:59 DEBUG "metric": { 18:26:59 DEBUG "pod": "ds-cts-0" 18:26:59 DEBUG }, 18:26:59 DEBUG "value": [ 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG [...] 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:26:59 DEBUG }, 18:26:59 DEBUG "value": [ 18:26:59 DEBUG 1734460019, 18:26:59 DEBUG "711041024" 18:26:59 DEBUG ] 18:26:59 DEBUG } 18:26:59 DEBUG ] 18:26:59 DEBUG } 18:26:59 DEBUG } 18:26:59 INFO [http_cmd]: http status code OK 18:26:59 DEBUG --- status code --- 18:26:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:26:59 DEBUG --- http response --- 18:26:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:26:59 DEBUG ----- output ----- 18:26:59 DEBUG { 18:26:59 DEBUG "status": "success", 18:26:59 DEBUG "data": { 18:26:59 DEBUG "resultType": "vector", 18:26:59 DEBUG "result": [ 18:26:59 DEBUG { 18:26:59 DEBUG "metric": { 18:26:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:26:59 DEBUG }, 18:26:59 DEBUG "value": [ 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG [...] 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:26:59 DEBUG }, 18:26:59 DEBUG "value": [ 18:26:59 DEBUG 1734460019, 18:26:59 DEBUG "1.0533697250798146" 18:26:59 DEBUG ] 18:26:59 DEBUG } 18:26:59 DEBUG ] 18:26:59 DEBUG } 18:26:59 DEBUG } 18:26:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460019" 18:26:59 INFO [http_cmd]: http status code OK 18:26:59 DEBUG --- status code --- 18:26:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:26:59 DEBUG --- http response --- 18:26:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:26:59 DEBUG ----- output ----- 18:26:59 DEBUG { 18:26:59 DEBUG "status": "success", 18:26:59 DEBUG "data": { 18:26:59 DEBUG "resultType": "vector", 18:26:59 DEBUG "result": [ 18:26:59 DEBUG { 18:26:59 DEBUG "metric": { 18:26:59 DEBUG "instance": "10.142.0.11:9100" 18:26:59 DEBUG }, 18:26:59 DEBUG "value": [ 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG [...] 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG "instance": "10.142.0.51:9100" 18:26:59 DEBUG }, 18:26:59 DEBUG "value": [ 18:26:59 DEBUG 1734460019, 18:26:59 DEBUG "125793.20000000001" 18:26:59 DEBUG ] 18:26:59 DEBUG } 18:26:59 DEBUG ] 18:26:59 DEBUG } 18:26:59 DEBUG } 18:26:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460019" 18:26:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734460019" 18:26:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460019" 18:26:59 INFO [http_cmd]: http status code OK 18:26:59 DEBUG --- status code --- 18:26:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:26:59 DEBUG --- http response --- 18:26:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:26:59 DEBUG ----- output ----- 18:26:59 DEBUG { 18:26:59 DEBUG "status": "success", 18:26:59 DEBUG "data": { 18:26:59 DEBUG "resultType": "vector", 18:26:59 DEBUG "result": [ 18:26:59 DEBUG { 18:26:59 DEBUG "metric": { 18:26:59 DEBUG "instance": "10.142.0.11:9100" 18:26:59 DEBUG }, 18:26:59 DEBUG "value": [ 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG [...] 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG "instance": "10.142.0.51:9100" 18:26:59 DEBUG }, 18:26:59 DEBUG "value": [ 18:26:59 DEBUG 1734460019, 18:26:59 DEBUG "19933.866666666665" 18:26:59 DEBUG ] 18:26:59 DEBUG } 18:26:59 DEBUG ] 18:26:59 DEBUG } 18:26:59 DEBUG } 18:26:59 INFO [http_cmd]: http status code OK 18:26:59 DEBUG --- status code --- 18:26:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:26:59 DEBUG --- http response --- 18:26:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:26:59 DEBUG ----- output ----- 18:26:59 DEBUG { 18:26:59 DEBUG "status": "success", 18:26:59 DEBUG "data": { 18:26:59 DEBUG "resultType": "vector", 18:26:59 DEBUG "result": [ 18:26:59 DEBUG { 18:26:59 DEBUG "metric": { 18:26:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:26:59 DEBUG }, 18:26:59 DEBUG "value": [ 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG [...] 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:26:59 DEBUG }, 18:26:59 DEBUG "value": [ 18:26:59 DEBUG 1734460019, 18:26:59 DEBUG "2957012992" 18:26:59 DEBUG ] 18:26:59 DEBUG } 18:26:59 DEBUG ] 18:26:59 DEBUG } 18:26:59 DEBUG } 18:26:59 INFO [http_cmd]: http status code OK 18:26:59 DEBUG --- status code --- 18:26:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:26:59 DEBUG --- http response --- 18:26:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:26:59 DEBUG ----- output ----- 18:26:59 DEBUG { 18:26:59 DEBUG "status": "success", 18:26:59 DEBUG "data": { 18:26:59 DEBUG "resultType": "vector", 18:26:59 DEBUG "result": [ 18:26:59 DEBUG { 18:26:59 DEBUG "metric": { 18:26:59 DEBUG "instance": "10.142.0.11:9100" 18:26:59 DEBUG }, 18:26:59 DEBUG "value": [ 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG [...] 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG "instance": "10.142.0.51:9100" 18:26:59 DEBUG }, 18:26:59 DEBUG "value": [ 18:26:59 DEBUG 1734460019, 18:26:59 DEBUG "0.000333333333333267" 18:26:59 DEBUG ] 18:26:59 DEBUG } 18:26:59 DEBUG ] 18:26:59 DEBUG } 18:26:59 DEBUG } 18:26:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460019" 18:26:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460019" 18:26:59 INFO [http_cmd]: http status code OK 18:26:59 DEBUG --- status code --- 18:26:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:26:59 DEBUG --- http response --- 18:26:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:26:59 DEBUG ----- output ----- 18:26:59 DEBUG { 18:26:59 DEBUG "status": "success", 18:26:59 DEBUG "data": { 18:26:59 DEBUG "resultType": "vector", 18:26:59 DEBUG "result": [ 18:26:59 DEBUG { 18:26:59 DEBUG "metric": { 18:26:59 DEBUG "pod": "ds-cts-0" 18:26:59 DEBUG }, 18:26:59 DEBUG "value": [ 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG [...] 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:26:59 DEBUG }, 18:26:59 DEBUG "value": [ 18:26:59 DEBUG 1734460019, 18:26:59 INFO [http_cmd]: http status code OK 18:26:59 DEBUG "1339.0642835520034" 18:26:59 DEBUG --- status code --- 18:26:59 DEBUG ] 18:26:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:26:59 DEBUG } 18:26:59 DEBUG --- http response --- 18:26:59 DEBUG ] 18:26:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:26:59 DEBUG } 18:26:59 DEBUG ----- output ----- 18:26:59 DEBUG } 18:26:59 DEBUG { 18:26:59 DEBUG "status": "success", 18:26:59 DEBUG "data": { 18:26:59 DEBUG "resultType": "vector", 18:26:59 DEBUG "result": [ 18:26:59 DEBUG { 18:26:59 DEBUG "metric": { 18:26:59 DEBUG "instance": "10.142.0.11:9100" 18:26:59 DEBUG }, 18:26:59 DEBUG "value": [ 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG [...] 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG 18:26:59 DEBUG "instance": "10.142.0.51:9100" 18:26:59 DEBUG }, 18:26:59 DEBUG "value": [ 18:26:59 DEBUG 1734460019, 18:26:59 DEBUG "0.00043333333333303397" 18:26:59 DEBUG ] 18:26:59 DEBUG } 18:26:59 DEBUG ] 18:26:59 DEBUG } 18:26:59 DEBUG } 18:27:00 INFO 18:27:00 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:27:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:27:00 INFO [loop_until]: OK (rc = 0) 18:27:00 DEBUG --- stdout --- 18:27:00 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 7m 522Mi ds-idrepo-0 7m 710Mi rcs-5d696ff5b-7ttvr 3m 298Mi rcs-5d696ff5b-wwlbr 2m 338Mi 18:27:00 DEBUG --- stderr --- 18:27:00 DEBUG 18:27:00 INFO 18:27:00 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:27:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:27:00 INFO [loop_until]: OK (rc = 0) 18:27:00 DEBUG --- stdout --- 18:27:00 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 117m 0% 2245Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1126m 3% 3782Mi 13% 18:27:00 DEBUG --- stderr --- 18:27:00 DEBUG 18:27:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460078" 18:27:58 INFO [http_cmd]: http status code OK 18:27:58 DEBUG --- status code --- 18:27:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:27:58 DEBUG --- http response --- 18:27:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:27:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734460079" 18:27:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460079" 18:27:59 INFO [http_cmd]: http status code OK 18:27:59 INFO [http_cmd]: http status code OK 18:27:59 DEBUG --- status code --- 18:27:59 DEBUG --- status code --- 18:27:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:27:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:27:59 DEBUG --- http response --- 18:27:59 DEBUG --- http response --- 18:27:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:27:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:27:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460079" 18:27:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460079" 18:27:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460079" 18:27:59 INFO [http_cmd]: http status code OK 18:27:59 DEBUG --- status code --- 18:27:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:27:59 DEBUG --- http response --- 18:27:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:27:59 INFO [http_cmd]: http status code OK 18:27:59 DEBUG --- status code --- 18:27:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:27:59 DEBUG --- http response --- 18:27:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:27:59 INFO [http_cmd]: http status code OK 18:27:59 DEBUG --- status code --- 18:27:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:27:59 DEBUG --- http response --- 18:27:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:27:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460079" 18:27:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460079" 18:27:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460079" 18:27:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460079" 18:27:59 INFO [http_cmd]: http status code OK 18:27:59 DEBUG --- status code --- 18:27:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:27:59 DEBUG --- http response --- 18:27:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:27:59 INFO [http_cmd]: http status code OK 18:27:59 DEBUG --- status code --- 18:27:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:27:59 DEBUG --- http response --- 18:27:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:27:59 INFO [http_cmd]: http status code OK 18:27:59 DEBUG --- status code --- 18:27:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:27:59 DEBUG --- http response --- 18:27:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:27:59 INFO [http_cmd]: http status code OK 18:27:59 DEBUG --- status code --- 18:27:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:27:59 DEBUG --- http response --- 18:27:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:27:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460079" 18:27:59 INFO [http_cmd]: http status code OK 18:27:59 DEBUG --- status code --- 18:27:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:27:59 DEBUG --- http response --- 18:27:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:27:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460079" 18:27:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734460079" 18:27:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460079" 18:27:59 ERROR [http_cmd]: ERROR 18:27:59 ERROR --- status code --- 18:27:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:27:59 ERROR --- http response --- 18:27:59 INFO [http_cmd]: http status code OK 18:27:59 DEBUG --- status code --- 18:27:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:27:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:27:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:27:59 DEBUG --- http response --- 18:27:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:27:59 INFO [http_cmd]: http status code OK 18:27:59 DEBUG --- status code --- 18:27:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:27:59 DEBUG --- http response --- 18:27:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:27:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460079" 18:27:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460079" 18:27:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734460079" 18:27:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460079" 18:27:59 INFO [http_cmd]: http status code OK 18:27:59 DEBUG --- status code --- 18:27:59 INFO [http_cmd]: http status code OK 18:27:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:27:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460079" 18:27:59 DEBUG --- status code --- 18:27:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:27:59 DEBUG --- http response --- 18:27:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460079" 18:27:59 DEBUG --- http response --- 18:27:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460079" 18:27:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734460079" 18:27:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460079" 18:27:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:27:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734460079" 18:27:59 INFO [http_cmd]: http status code OK 18:27:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460079" 18:27:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460079" 18:27:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460079" 18:27:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:27:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460079" 18:27:59 DEBUG --- status code --- 18:27:59 INFO [http_cmd]: http status code OK 18:27:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460079" 18:27:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:27:59 DEBUG --- status code --- 18:27:59 INFO [http_cmd]: http status code OK 18:27:59 INFO [http_cmd]: http status code OK 18:27:59 DEBUG --- http response --- 18:27:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:27:59 DEBUG --- status code --- 18:27:59 DEBUG --- status code --- 18:27:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:27:59 DEBUG --- http response --- 18:27:59 INFO [http_cmd]: http status code OK 18:27:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:27:59 INFO [http_cmd]: http status code OK 18:27:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:27:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:27:59 DEBUG --- status code --- 18:27:59 DEBUG --- http response --- 18:27:59 DEBUG --- status code --- 18:27:59 DEBUG --- http response --- 18:27:59 INFO [http_cmd]: http status code OK 18:27:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:27:59 INFO [http_cmd]: http status code OK 18:27:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:27:59 INFO [http_cmd]: http status code OK 18:27:59 INFO [http_cmd]: http status code OK 18:27:59 INFO [http_cmd]: http status code OK 18:27:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:27:59 INFO [http_cmd]: http status code OK 18:27:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:27:59 DEBUG --- status code --- 18:27:59 DEBUG --- http response --- 18:27:59 DEBUG --- status code --- 18:27:59 DEBUG --- status code --- 18:27:59 DEBUG --- status code --- 18:27:59 DEBUG --- status code --- 18:27:59 INFO [http_cmd]: http status code OK 18:27:59 DEBUG --- status code --- 18:27:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:27:59 DEBUG --- http response --- 18:27:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:27:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:27:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:27:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:27:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:27:59 DEBUG --- http response --- 18:27:59 DEBUG --- status code --- 18:27:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:27:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:27:59 DEBUG --- http response --- 18:27:59 DEBUG --- http response --- 18:27:59 DEBUG --- http response --- 18:27:59 DEBUG --- http response --- 18:27:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:27:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:27:59 DEBUG --- http response --- 18:27:59 DEBUG ----- output ----- 18:27:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:27:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:27:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:27:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:27:59 DEBUG --- http response --- 18:27:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:27:59 DEBUG { 18:27:59 DEBUG ----- output ----- 18:27:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:27:59 DEBUG "status": "success", 18:27:59 DEBUG { 18:27:59 DEBUG "data": { 18:27:59 DEBUG "status": "success", 18:27:59 DEBUG "resultType": "vector", 18:27:59 DEBUG "data": { 18:27:59 DEBUG "result": [ 18:27:59 DEBUG "resultType": "vector", 18:27:59 DEBUG { 18:27:59 DEBUG "result": [ 18:27:59 DEBUG "metric": { 18:27:59 DEBUG { 18:27:59 DEBUG "pod": "ds-cts-0" 18:27:59 DEBUG "metric": { 18:27:59 DEBUG }, 18:27:59 DEBUG "pod": "ds-cts-0" 18:27:59 DEBUG "value": [ 18:27:59 DEBUG }, 18:27:59 DEBUG 18:27:59 DEBUG "value": [ 18:27:59 DEBUG 18:27:59 DEBUG 18:27:59 DEBUG 18:27:59 DEBUG 18:27:59 DEBUG [...] 18:27:59 DEBUG 18:27:59 DEBUG 18:27:59 DEBUG [...] 18:27:59 DEBUG 18:27:59 DEBUG 18:27:59 DEBUG 18:27:59 DEBUG 18:27:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:27:59 DEBUG 18:27:59 DEBUG }, 18:27:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:27:59 DEBUG "value": [ 18:27:59 DEBUG }, 18:27:59 DEBUG 1734460079, 18:27:59 DEBUG "value": [ 18:27:59 DEBUG "0" 18:27:59 DEBUG 1734460079, 18:27:59 DEBUG ] 18:27:59 DEBUG "0.0025816052397633505" 18:27:59 DEBUG } 18:27:59 DEBUG ] 18:27:59 DEBUG ] 18:27:59 DEBUG } 18:27:59 DEBUG } 18:27:59 DEBUG ] 18:27:59 DEBUG } 18:27:59 DEBUG } 18:27:59 DEBUG } 18:27:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460079" 18:27:59 INFO [http_cmd]: http status code OK 18:27:59 DEBUG --- status code --- 18:27:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:27:59 DEBUG --- http response --- 18:27:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:27:59 DEBUG ----- output ----- 18:27:59 DEBUG { 18:27:59 DEBUG "status": "success", 18:27:59 DEBUG "data": { 18:27:59 DEBUG "resultType": "vector", 18:27:59 DEBUG "result": [ 18:27:59 DEBUG { 18:27:59 DEBUG "metric": { 18:27:59 DEBUG "instance": "10.142.0.11:9100" 18:27:59 DEBUG }, 18:27:59 DEBUG "value": [ 18:27:59 DEBUG 18:27:59 DEBUG 18:27:59 DEBUG 18:27:59 DEBUG [...] 18:27:59 DEBUG 18:27:59 DEBUG 18:27:59 DEBUG 18:27:59 DEBUG "instance": "10.142.0.51:9100" 18:27:59 DEBUG }, 18:27:59 DEBUG "value": [ 18:27:59 DEBUG 1734460079, 18:27:59 DEBUG "127186.3" 18:27:59 DEBUG ] 18:27:59 DEBUG } 18:27:59 DEBUG ] 18:27:59 DEBUG } 18:27:59 DEBUG } 18:27:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460079" 18:27:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734460079" 18:27:59 INFO [http_cmd]: http status code OK 18:27:59 DEBUG --- status code --- 18:27:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:27:59 INFO [http_cmd]: http status code OK 18:27:59 DEBUG --- http response --- 18:27:59 DEBUG --- status code --- 18:27:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:27:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:27:59 DEBUG ----- output ----- 18:27:59 DEBUG --- http response --- 18:27:59 DEBUG { 18:27:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:27:59 DEBUG "status": "success", 18:27:59 DEBUG ----- output ----- 18:27:59 DEBUG "data": { 18:27:59 DEBUG { 18:27:59 DEBUG "resultType": "vector", 18:27:59 DEBUG "status": "success", 18:27:59 DEBUG "result": [ 18:27:59 DEBUG "data": { 18:27:59 DEBUG { 18:27:59 DEBUG "resultType": "vector", 18:27:59 DEBUG "metric": { 18:27:59 DEBUG "result": [ 18:27:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:27:59 DEBUG { 18:27:59 DEBUG }, 18:27:59 DEBUG "metric": { 18:27:59 DEBUG "value": [ 18:27:59 DEBUG "pod": "ds-cts-0" 18:27:59 DEBUG 18:27:59 DEBUG }, 18:27:59 DEBUG 18:27:59 DEBUG "value": [ 18:27:59 DEBUG 18:27:59 DEBUG 18:27:59 DEBUG [...] 18:27:59 DEBUG 18:27:59 DEBUG 18:27:59 DEBUG 18:27:59 DEBUG 18:27:59 DEBUG [...] 18:27:59 DEBUG 18:27:59 DEBUG 18:27:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:27:59 DEBUG 18:27:59 DEBUG }, 18:27:59 DEBUG 18:27:59 DEBUG "value": [ 18:27:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:27:59 DEBUG 1734460079, 18:27:59 DEBUG }, 18:27:59 DEBUG "1.0574517405670665" 18:27:59 DEBUG "value": [ 18:27:59 DEBUG ] 18:27:59 DEBUG 1734460079, 18:27:59 DEBUG } 18:27:59 DEBUG "711053312" 18:27:59 DEBUG ] 18:27:59 DEBUG ] 18:27:59 DEBUG } 18:27:59 DEBUG } 18:27:59 DEBUG } 18:27:59 DEBUG ] 18:27:59 DEBUG } 18:27:59 DEBUG } 18:27:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460079" 18:27:59 INFO [http_cmd]: http status code OK 18:27:59 DEBUG --- status code --- 18:27:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:27:59 DEBUG --- http response --- 18:27:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:27:59 DEBUG ----- output ----- 18:27:59 DEBUG { 18:27:59 DEBUG "status": "success", 18:27:59 DEBUG "data": { 18:27:59 DEBUG "resultType": "vector", 18:27:59 DEBUG "result": [ 18:27:59 DEBUG { 18:27:59 DEBUG "metric": { 18:27:59 DEBUG "instance": "10.142.0.11:9100" 18:27:59 DEBUG }, 18:27:59 DEBUG "value": [ 18:27:59 DEBUG 18:27:59 DEBUG 18:27:59 DEBUG 18:27:59 DEBUG [...] 18:27:59 DEBUG 18:27:59 DEBUG 18:27:59 DEBUG 18:27:59 DEBUG "instance": "10.142.0.51:9100" 18:27:59 DEBUG }, 18:27:59 DEBUG "value": [ 18:27:59 DEBUG 1734460079, 18:27:59 DEBUG "124932.46666666663" 18:27:59 DEBUG ] 18:27:59 DEBUG } 18:27:59 DEBUG ] 18:27:59 DEBUG } 18:27:59 DEBUG } 18:27:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460079" 18:27:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734460079" 18:27:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460079" 18:27:59 INFO [http_cmd]: http status code OK 18:27:59 DEBUG --- status code --- 18:27:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:27:59 DEBUG --- http response --- 18:27:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:27:59 DEBUG ----- output ----- 18:27:59 DEBUG { 18:27:59 DEBUG "status": "success", 18:27:59 DEBUG "data": { 18:27:59 DEBUG "resultType": "vector", 18:27:59 DEBUG "result": [ 18:27:59 DEBUG { 18:27:59 DEBUG "metric": { 18:27:59 DEBUG "instance": "10.142.0.11:9100" 18:27:59 DEBUG }, 18:27:59 DEBUG "value": [ 18:27:59 DEBUG 18:27:59 DEBUG 18:27:59 DEBUG 18:27:59 DEBUG [...] 18:27:59 DEBUG 18:27:59 DEBUG 18:27:59 DEBUG 18:27:59 DEBUG "instance": "10.142.0.51:9100" 18:27:59 DEBUG }, 18:27:59 DEBUG "value": [ 18:27:59 DEBUG 1734460079, 18:27:59 DEBUG "33860.26666666666" 18:27:59 DEBUG ] 18:27:59 DEBUG } 18:27:59 DEBUG ] 18:27:59 DEBUG } 18:27:59 DEBUG } 18:27:59 INFO [http_cmd]: http status code OK 18:27:59 DEBUG --- status code --- 18:27:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:27:59 DEBUG --- http response --- 18:27:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:27:59 DEBUG ----- output ----- 18:27:59 DEBUG { 18:27:59 DEBUG "status": "success", 18:27:59 DEBUG "data": { 18:27:59 DEBUG "resultType": "vector", 18:27:59 DEBUG "result": [ 18:27:59 DEBUG { 18:27:59 DEBUG "metric": { 18:27:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:27:59 DEBUG }, 18:27:59 DEBUG "value": [ 18:27:59 DEBUG 18:27:59 DEBUG 18:27:59 DEBUG 18:27:59 DEBUG [...] 18:27:59 DEBUG 18:27:59 DEBUG 18:27:59 DEBUG 18:27:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:27:59 DEBUG }, 18:27:59 DEBUG "value": [ 18:27:59 DEBUG 1734460079, 18:27:59 DEBUG "2962567168" 18:27:59 DEBUG ] 18:27:59 DEBUG } 18:27:59 DEBUG ] 18:27:59 DEBUG } 18:27:59 DEBUG } 18:27:59 INFO [http_cmd]: http status code OK 18:27:59 DEBUG --- status code --- 18:27:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:27:59 DEBUG --- http response --- 18:27:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:27:59 DEBUG ----- output ----- 18:27:59 DEBUG { 18:27:59 DEBUG "status": "success", 18:27:59 DEBUG "data": { 18:27:59 DEBUG "resultType": "vector", 18:27:59 DEBUG "result": [ 18:27:59 DEBUG { 18:27:59 DEBUG "metric": { 18:27:59 DEBUG "instance": "10.142.0.11:9100" 18:27:59 DEBUG }, 18:27:59 DEBUG "value": [ 18:27:59 DEBUG 18:27:59 DEBUG 18:27:59 DEBUG 18:27:59 DEBUG [...] 18:27:59 DEBUG 18:27:59 DEBUG 18:27:59 DEBUG 18:27:59 DEBUG "instance": "10.142.0.51:9100" 18:27:59 DEBUG }, 18:27:59 DEBUG "value": [ 18:27:59 DEBUG 1734460079, 18:27:59 DEBUG "0.000333333333333267" 18:27:59 DEBUG ] 18:27:59 DEBUG } 18:27:59 DEBUG ] 18:27:59 DEBUG } 18:27:59 DEBUG } 18:27:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460079" 18:27:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460079" 18:27:59 INFO [http_cmd]: http status code OK 18:27:59 DEBUG --- status code --- 18:27:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:27:59 INFO [http_cmd]: http status code OK 18:27:59 DEBUG --- http response --- 18:27:59 DEBUG --- status code --- 18:27:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:27:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:27:59 DEBUG ----- output ----- 18:27:59 DEBUG --- http response --- 18:27:59 DEBUG { 18:27:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:27:59 DEBUG "status": "success", 18:27:59 DEBUG ----- output ----- 18:27:59 DEBUG "data": { 18:27:59 DEBUG { 18:27:59 DEBUG "resultType": "vector", 18:27:59 DEBUG "status": "success", 18:27:59 DEBUG "result": [ 18:27:59 DEBUG "data": { 18:27:59 DEBUG { 18:27:59 DEBUG "resultType": "vector", 18:27:59 DEBUG "metric": { 18:27:59 DEBUG "result": [ 18:27:59 DEBUG "instance": "10.142.0.11:9100" 18:27:59 DEBUG { 18:27:59 DEBUG }, 18:27:59 DEBUG "metric": { 18:27:59 DEBUG "value": [ 18:27:59 DEBUG "pod": "ds-cts-0" 18:27:59 DEBUG 18:27:59 DEBUG }, 18:27:59 DEBUG 18:27:59 DEBUG "value": [ 18:27:59 DEBUG 18:27:59 DEBUG 18:27:59 DEBUG [...] 18:27:59 DEBUG 18:27:59 DEBUG 18:27:59 DEBUG 18:27:59 DEBUG 18:27:59 DEBUG [...] 18:27:59 DEBUG 18:27:59 DEBUG 18:27:59 DEBUG "instance": "10.142.0.51:9100" 18:27:59 DEBUG 18:27:59 DEBUG }, 18:27:59 DEBUG 18:27:59 DEBUG "value": [ 18:27:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:27:59 DEBUG 1734460079, 18:27:59 DEBUG }, 18:27:59 DEBUG "0.00046666666666557904" 18:27:59 DEBUG "value": [ 18:27:59 DEBUG ] 18:27:59 DEBUG 1734460079, 18:27:59 DEBUG } 18:27:59 DEBUG "1537.8691248450195" 18:27:59 DEBUG ] 18:27:59 DEBUG ] 18:27:59 DEBUG } 18:27:59 DEBUG } 18:27:59 DEBUG } 18:27:59 DEBUG ] 18:27:59 DEBUG } 18:27:59 DEBUG } 18:28:00 INFO 18:28:00 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:28:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:28:00 INFO [loop_until]: OK (rc = 0) 18:28:00 DEBUG --- stdout --- 18:28:00 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 527Mi ds-idrepo-0 7m 711Mi rcs-5d696ff5b-7ttvr 5m 298Mi rcs-5d696ff5b-wwlbr 2m 338Mi 18:28:00 DEBUG --- stderr --- 18:28:00 DEBUG 18:28:00 INFO 18:28:00 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:28:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:28:00 INFO [loop_until]: OK (rc = 0) 18:28:00 DEBUG --- stdout --- 18:28:00 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 115m 0% 2273Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1137m 3% 3787Mi 13% 18:28:00 DEBUG --- stderr --- 18:28:00 DEBUG 18:28:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460138" 18:28:58 INFO [http_cmd]: http status code OK 18:28:58 DEBUG --- status code --- 18:28:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:28:58 DEBUG --- http response --- 18:28:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:28:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460139" 18:28:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734460139" 18:28:59 INFO [http_cmd]: http status code OK 18:28:59 DEBUG --- status code --- 18:28:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:28:59 DEBUG --- http response --- 18:28:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:28:59 INFO [http_cmd]: http status code OK 18:28:59 DEBUG --- status code --- 18:28:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:28:59 DEBUG --- http response --- 18:28:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:28:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460139" 18:28:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460139" 18:28:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460139" 18:28:59 INFO [http_cmd]: http status code OK 18:28:59 INFO [http_cmd]: http status code OK 18:28:59 DEBUG --- status code --- 18:28:59 DEBUG --- status code --- 18:28:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:28:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:28:59 DEBUG --- http response --- 18:28:59 DEBUG --- http response --- 18:28:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:28:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:28:59 INFO [http_cmd]: http status code OK 18:28:59 DEBUG --- status code --- 18:28:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:28:59 DEBUG --- http response --- 18:28:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:28:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460139" 18:28:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460139" 18:28:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460139" 18:28:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460139" 18:28:59 INFO [http_cmd]: http status code OK 18:28:59 DEBUG --- status code --- 18:28:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:28:59 DEBUG --- http response --- 18:28:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:28:59 INFO [http_cmd]: http status code OK 18:28:59 DEBUG --- status code --- 18:28:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:28:59 DEBUG --- http response --- 18:28:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:28:59 INFO [http_cmd]: http status code OK 18:28:59 DEBUG --- status code --- 18:28:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:28:59 INFO [http_cmd]: http status code OK 18:28:59 DEBUG --- http response --- 18:28:59 DEBUG --- status code --- 18:28:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:28:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:28:59 DEBUG --- http response --- 18:28:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:28:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460139" 18:28:59 INFO [http_cmd]: http status code OK 18:28:59 DEBUG --- status code --- 18:28:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:28:59 DEBUG --- http response --- 18:28:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:28:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734460139" 18:28:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460139" 18:28:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460139" 18:28:59 INFO [http_cmd]: http status code OK 18:28:59 DEBUG --- status code --- 18:28:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:28:59 DEBUG --- http response --- 18:28:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:28:59 INFO [http_cmd]: http status code OK 18:28:59 DEBUG --- status code --- 18:28:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:28:59 DEBUG --- http response --- 18:28:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:28:59 ERROR [http_cmd]: ERROR 18:28:59 ERROR --- status code --- 18:28:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:28:59 ERROR --- http response --- 18:28:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:28:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:28:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460139" 18:28:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460139" 18:28:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460139" 18:28:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734460139" 18:28:59 INFO [http_cmd]: http status code OK 18:28:59 DEBUG --- status code --- 18:28:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:28:59 DEBUG --- http response --- 18:28:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:28:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460139" 18:28:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460139" 18:28:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734460139" 18:28:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460139" 18:28:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460139" 18:28:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460139" 18:28:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460139" 18:28:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460139" 18:28:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460139" 18:28:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460139" 18:28:59 INFO [http_cmd]: http status code OK 18:28:59 DEBUG --- status code --- 18:28:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:28:59 DEBUG --- http response --- 18:28:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:28:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734460139" 18:28:59 INFO [http_cmd]: http status code OK 18:28:59 INFO [http_cmd]: http status code OK 18:28:59 DEBUG --- status code --- 18:28:59 INFO [http_cmd]: http status code OK 18:28:59 INFO [http_cmd]: http status code OK 18:28:59 INFO [http_cmd]: http status code OK 18:28:59 INFO [http_cmd]: http status code OK 18:28:59 INFO [http_cmd]: http status code OK 18:28:59 DEBUG --- status code --- 18:28:59 INFO [http_cmd]: http status code OK 18:28:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:28:59 DEBUG --- status code --- 18:28:59 DEBUG --- status code --- 18:28:59 DEBUG --- status code --- 18:28:59 DEBUG --- status code --- 18:28:59 DEBUG --- status code --- 18:28:59 INFO [http_cmd]: http status code OK 18:28:59 INFO [http_cmd]: http status code OK 18:28:59 INFO [http_cmd]: http status code OK 18:28:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:28:59 DEBUG --- status code --- 18:28:59 DEBUG --- http response --- 18:28:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:28:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:28:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:28:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:28:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:28:59 INFO [http_cmd]: http status code OK 18:28:59 DEBUG --- status code --- 18:28:59 DEBUG --- status code --- 18:28:59 DEBUG --- status code --- 18:28:59 DEBUG --- http response --- 18:28:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:28:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:28:59 DEBUG --- http response --- 18:28:59 DEBUG --- http response --- 18:28:59 DEBUG --- http response --- 18:28:59 DEBUG --- http response --- 18:28:59 DEBUG --- http response --- 18:28:59 DEBUG --- status code --- 18:28:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:28:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:28:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:28:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:28:59 DEBUG --- http response --- 18:28:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:28:59 INFO [http_cmd]: http status code OK 18:28:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:28:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:28:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:28:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:28:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:28:59 DEBUG --- http response --- 18:28:59 DEBUG --- http response --- 18:28:59 DEBUG --- http response --- 18:28:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:28:59 DEBUG --- status code --- 18:28:59 DEBUG --- http response --- 18:28:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:28:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:28:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:28:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:28:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:28:59 DEBUG ----- output ----- 18:28:59 DEBUG --- http response --- 18:28:59 DEBUG ----- output ----- 18:28:59 DEBUG { 18:28:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:28:59 DEBUG { 18:28:59 DEBUG "status": "success", 18:28:59 DEBUG "status": "success", 18:28:59 DEBUG "data": { 18:28:59 DEBUG "data": { 18:28:59 DEBUG "resultType": "vector", 18:28:59 DEBUG "resultType": "vector", 18:28:59 DEBUG "result": [ 18:28:59 DEBUG "result": [ 18:28:59 DEBUG { 18:28:59 DEBUG { 18:28:59 DEBUG "metric": { 18:28:59 DEBUG "metric": { 18:28:59 DEBUG "pod": "ds-cts-0" 18:28:59 DEBUG "pod": "ds-cts-0" 18:28:59 DEBUG }, 18:28:59 DEBUG }, 18:28:59 DEBUG "value": [ 18:28:59 DEBUG "value": [ 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG [...] 18:28:59 DEBUG [...] 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:28:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:28:59 DEBUG }, 18:28:59 DEBUG }, 18:28:59 DEBUG "value": [ 18:28:59 DEBUG "value": [ 18:28:59 DEBUG 1734460139, 18:28:59 DEBUG 1734460139, 18:28:59 DEBUG "0.0032720594118530234" 18:28:59 DEBUG "0" 18:28:59 DEBUG ] 18:28:59 DEBUG ] 18:28:59 DEBUG } 18:28:59 DEBUG } 18:28:59 DEBUG ] 18:28:59 DEBUG ] 18:28:59 DEBUG } 18:28:59 DEBUG } 18:28:59 DEBUG } 18:28:59 DEBUG } 18:28:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460139" 18:28:59 INFO [http_cmd]: http status code OK 18:28:59 DEBUG --- status code --- 18:28:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:28:59 DEBUG --- http response --- 18:28:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:28:59 DEBUG ----- output ----- 18:28:59 DEBUG { 18:28:59 DEBUG "status": "success", 18:28:59 DEBUG "data": { 18:28:59 DEBUG "resultType": "vector", 18:28:59 DEBUG "result": [ 18:28:59 DEBUG { 18:28:59 DEBUG "metric": { 18:28:59 DEBUG "instance": "10.142.0.11:9100" 18:28:59 DEBUG }, 18:28:59 DEBUG "value": [ 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG [...] 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG "instance": "10.142.0.51:9100" 18:28:59 DEBUG }, 18:28:59 DEBUG "value": [ 18:28:59 DEBUG 1734460139, 18:28:59 DEBUG "124543.50000000001" 18:28:59 DEBUG ] 18:28:59 DEBUG } 18:28:59 DEBUG ] 18:28:59 DEBUG } 18:28:59 DEBUG } 18:28:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734460139" 18:28:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460139" 18:28:59 INFO [http_cmd]: http status code OK 18:28:59 DEBUG --- status code --- 18:28:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:28:59 DEBUG --- http response --- 18:28:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:28:59 DEBUG ----- output ----- 18:28:59 DEBUG { 18:28:59 DEBUG "status": "success", 18:28:59 DEBUG "data": { 18:28:59 DEBUG "resultType": "vector", 18:28:59 DEBUG "result": [ 18:28:59 DEBUG { 18:28:59 DEBUG "metric": { 18:28:59 DEBUG "pod": "ds-cts-0" 18:28:59 DEBUG }, 18:28:59 DEBUG "value": [ 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG [...] 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:28:59 DEBUG }, 18:28:59 DEBUG "value": [ 18:28:59 DEBUG 1734460139, 18:28:59 DEBUG "711106560" 18:28:59 DEBUG ] 18:28:59 DEBUG } 18:28:59 DEBUG ] 18:28:59 DEBUG } 18:28:59 DEBUG } 18:28:59 INFO [http_cmd]: http status code OK 18:28:59 DEBUG --- status code --- 18:28:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:28:59 DEBUG --- http response --- 18:28:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:28:59 DEBUG ----- output ----- 18:28:59 DEBUG { 18:28:59 DEBUG "status": "success", 18:28:59 DEBUG "data": { 18:28:59 DEBUG "resultType": "vector", 18:28:59 DEBUG "result": [ 18:28:59 DEBUG { 18:28:59 DEBUG "metric": { 18:28:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:28:59 DEBUG }, 18:28:59 DEBUG "value": [ 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG [...] 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:28:59 DEBUG }, 18:28:59 DEBUG "value": [ 18:28:59 DEBUG 1734460139, 18:28:59 DEBUG "1.058867474697899" 18:28:59 DEBUG ] 18:28:59 DEBUG } 18:28:59 DEBUG ] 18:28:59 DEBUG } 18:28:59 DEBUG } 18:28:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460139" 18:28:59 INFO [http_cmd]: http status code OK 18:28:59 DEBUG --- status code --- 18:28:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:28:59 DEBUG --- http response --- 18:28:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:28:59 DEBUG ----- output ----- 18:28:59 DEBUG { 18:28:59 DEBUG "status": "success", 18:28:59 DEBUG "data": { 18:28:59 DEBUG "resultType": "vector", 18:28:59 DEBUG "result": [ 18:28:59 DEBUG { 18:28:59 DEBUG "metric": { 18:28:59 DEBUG "instance": "10.142.0.11:9100" 18:28:59 DEBUG }, 18:28:59 DEBUG "value": [ 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG [...] 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG "instance": "10.142.0.51:9100" 18:28:59 DEBUG }, 18:28:59 DEBUG "value": [ 18:28:59 DEBUG 1734460139, 18:28:59 DEBUG "122902.73333333334" 18:28:59 DEBUG ] 18:28:59 DEBUG } 18:28:59 DEBUG ] 18:28:59 DEBUG } 18:28:59 DEBUG } 18:28:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460139" 18:28:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734460139" 18:28:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460139" 18:28:59 INFO [http_cmd]: http status code OK 18:28:59 DEBUG --- status code --- 18:28:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:28:59 DEBUG --- http response --- 18:28:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:28:59 DEBUG ----- output ----- 18:28:59 DEBUG { 18:28:59 DEBUG "status": "success", 18:28:59 DEBUG "data": { 18:28:59 DEBUG "resultType": "vector", 18:28:59 DEBUG "result": [ 18:28:59 DEBUG { 18:28:59 DEBUG "metric": { 18:28:59 DEBUG "instance": "10.142.0.11:9100" 18:28:59 DEBUG }, 18:28:59 DEBUG "value": [ 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG [...] 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG "instance": "10.142.0.51:9100" 18:28:59 DEBUG }, 18:28:59 DEBUG "value": [ 18:28:59 DEBUG 1734460139, 18:28:59 DEBUG "23620.266666666666" 18:28:59 DEBUG ] 18:28:59 DEBUG } 18:28:59 DEBUG ] 18:28:59 DEBUG } 18:28:59 DEBUG } 18:28:59 INFO [http_cmd]: http status code OK 18:28:59 DEBUG --- status code --- 18:28:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:28:59 DEBUG --- http response --- 18:28:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:28:59 DEBUG ----- output ----- 18:28:59 DEBUG { 18:28:59 DEBUG "status": "success", 18:28:59 DEBUG "data": { 18:28:59 DEBUG "resultType": "vector", 18:28:59 DEBUG "result": [ 18:28:59 DEBUG { 18:28:59 DEBUG "metric": { 18:28:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:28:59 DEBUG }, 18:28:59 DEBUG "value": [ 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG [...] 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:28:59 DEBUG }, 18:28:59 DEBUG "value": [ 18:28:59 DEBUG 1734460139, 18:28:59 DEBUG "2975764480" 18:28:59 DEBUG ] 18:28:59 DEBUG } 18:28:59 DEBUG ] 18:28:59 DEBUG } 18:28:59 DEBUG } 18:28:59 INFO [http_cmd]: http status code OK 18:28:59 DEBUG --- status code --- 18:28:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:28:59 DEBUG --- http response --- 18:28:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:28:59 DEBUG ----- output ----- 18:28:59 DEBUG { 18:28:59 DEBUG "status": "success", 18:28:59 DEBUG "data": { 18:28:59 DEBUG "resultType": "vector", 18:28:59 DEBUG "result": [ 18:28:59 DEBUG { 18:28:59 DEBUG "metric": { 18:28:59 DEBUG "instance": "10.142.0.11:9100" 18:28:59 DEBUG }, 18:28:59 DEBUG "value": [ 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG [...] 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG "instance": "10.142.0.51:9100" 18:28:59 DEBUG }, 18:28:59 DEBUG "value": [ 18:28:59 DEBUG 1734460139, 18:28:59 DEBUG "0.000333333333333267" 18:28:59 DEBUG ] 18:28:59 DEBUG } 18:28:59 DEBUG ] 18:28:59 DEBUG } 18:28:59 DEBUG } 18:28:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460139" 18:28:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460139" 18:28:59 INFO [http_cmd]: http status code OK 18:28:59 DEBUG --- status code --- 18:28:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:28:59 INFO [http_cmd]: http status code OK 18:28:59 DEBUG --- http response --- 18:28:59 DEBUG --- status code --- 18:28:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:28:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:28:59 DEBUG ----- output ----- 18:28:59 DEBUG --- http response --- 18:28:59 DEBUG { 18:28:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:28:59 DEBUG "status": "success", 18:28:59 DEBUG ----- output ----- 18:28:59 DEBUG "data": { 18:28:59 DEBUG { 18:28:59 DEBUG "resultType": "vector", 18:28:59 DEBUG "status": "success", 18:28:59 DEBUG "result": [ 18:28:59 DEBUG "data": { 18:28:59 DEBUG { 18:28:59 DEBUG "resultType": "vector", 18:28:59 DEBUG "metric": { 18:28:59 DEBUG "result": [ 18:28:59 DEBUG "instance": "10.142.0.11:9100" 18:28:59 DEBUG { 18:28:59 DEBUG }, 18:28:59 DEBUG "metric": { 18:28:59 DEBUG "value": [ 18:28:59 DEBUG "pod": "ds-cts-0" 18:28:59 DEBUG 18:28:59 DEBUG }, 18:28:59 DEBUG 18:28:59 DEBUG "value": [ 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG [...] 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG [...] 18:28:59 DEBUG 18:28:59 DEBUG 18:28:59 DEBUG "instance": "10.142.0.51:9100" 18:28:59 DEBUG 18:28:59 DEBUG }, 18:28:59 DEBUG 18:28:59 DEBUG "value": [ 18:28:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:28:59 DEBUG 1734460139, 18:28:59 DEBUG }, 18:28:59 DEBUG "0.00036666666666747" 18:28:59 DEBUG "value": [ 18:28:59 DEBUG ] 18:28:59 DEBUG 1734460139, 18:28:59 DEBUG } 18:28:59 DEBUG "1160.987185193614" 18:28:59 DEBUG ] 18:28:59 DEBUG ] 18:28:59 DEBUG } 18:28:59 DEBUG } 18:28:59 DEBUG } 18:28:59 DEBUG ] 18:28:59 DEBUG } 18:28:59 DEBUG } 18:29:00 INFO 18:29:00 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:29:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:29:00 INFO [loop_until]: OK (rc = 0) 18:29:00 DEBUG --- stdout --- 18:29:00 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 529Mi ds-idrepo-0 8m 710Mi rcs-5d696ff5b-7ttvr 3m 298Mi rcs-5d696ff5b-wwlbr 2m 338Mi 18:29:00 DEBUG --- stderr --- 18:29:00 DEBUG 18:29:00 INFO 18:29:00 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:29:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:29:00 INFO [loop_until]: OK (rc = 0) 18:29:00 DEBUG --- stdout --- 18:29:00 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 117m 0% 2275Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1127m 3% 3791Mi 13% 18:29:00 DEBUG --- stderr --- 18:29:00 DEBUG 18:29:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460198" 18:29:58 INFO [http_cmd]: http status code OK 18:29:58 DEBUG --- status code --- 18:29:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:29:58 DEBUG --- http response --- 18:29:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:29:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460199" 18:29:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734460199" 18:29:59 INFO [http_cmd]: http status code OK 18:29:59 DEBUG --- status code --- 18:29:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:29:59 DEBUG --- http response --- 18:29:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:29:59 INFO [http_cmd]: http status code OK 18:29:59 DEBUG --- status code --- 18:29:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:29:59 DEBUG --- http response --- 18:29:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:29:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460199" 18:29:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460199" 18:29:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460199" 18:29:59 INFO [http_cmd]: http status code OK 18:29:59 INFO [http_cmd]: http status code OK 18:29:59 DEBUG --- status code --- 18:29:59 DEBUG --- status code --- 18:29:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:29:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:29:59 DEBUG --- http response --- 18:29:59 DEBUG --- http response --- 18:29:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:29:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:29:59 INFO [http_cmd]: http status code OK 18:29:59 DEBUG --- status code --- 18:29:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:29:59 DEBUG --- http response --- 18:29:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:29:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460199" 18:29:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460199" 18:29:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460199" 18:29:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460199" 18:29:59 INFO [http_cmd]: http status code OK 18:29:59 DEBUG --- status code --- 18:29:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:29:59 DEBUG --- http response --- 18:29:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:29:59 INFO [http_cmd]: http status code OK 18:29:59 DEBUG --- status code --- 18:29:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:29:59 DEBUG --- http response --- 18:29:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:29:59 INFO [http_cmd]: http status code OK 18:29:59 DEBUG --- status code --- 18:29:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:29:59 DEBUG --- http response --- 18:29:59 INFO [http_cmd]: http status code OK 18:29:59 DEBUG --- status code --- 18:29:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:29:59 DEBUG --- http response --- 18:29:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:29:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:29:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460199" 18:29:59 INFO [http_cmd]: http status code OK 18:29:59 DEBUG --- status code --- 18:29:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:29:59 DEBUG --- http response --- 18:29:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:29:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460199" 18:29:59 INFO [http_cmd]: http status code OK 18:29:59 DEBUG --- status code --- 18:29:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:29:59 DEBUG --- http response --- 18:29:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:29:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734460199" 18:29:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460199" 18:29:59 INFO [http_cmd]: http status code OK 18:29:59 DEBUG --- status code --- 18:29:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:29:59 DEBUG --- http response --- 18:29:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:29:59 ERROR [http_cmd]: ERROR 18:29:59 ERROR --- status code --- 18:29:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:29:59 ERROR --- http response --- 18:29:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:29:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:29:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460199" 18:29:59 INFO [http_cmd]: http status code OK 18:29:59 DEBUG --- status code --- 18:29:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:29:59 DEBUG --- http response --- 18:29:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:29:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460199" 18:29:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460199" 18:29:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734460199" 18:29:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460199" 18:29:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460199" 18:29:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460199" 18:29:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460199" 18:29:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734460199" 18:29:59 INFO [http_cmd]: http status code OK 18:29:59 DEBUG --- status code --- 18:29:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:29:59 DEBUG --- http response --- 18:29:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:29:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734460199" 18:29:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460199" 18:29:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460199" 18:29:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460199" 18:29:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460199" 18:29:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460199" 18:29:59 INFO [http_cmd]: http status code OK 18:29:59 INFO [http_cmd]: http status code OK 18:29:59 DEBUG --- status code --- 18:29:59 DEBUG --- status code --- 18:29:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:29:59 INFO [http_cmd]: http status code OK 18:29:59 INFO [http_cmd]: http status code OK 18:29:59 INFO [http_cmd]: http status code OK 18:29:59 INFO [http_cmd]: http status code OK 18:29:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:29:59 INFO [http_cmd]: http status code OK 18:29:59 DEBUG --- http response --- 18:29:59 INFO [http_cmd]: http status code OK 18:29:59 DEBUG --- status code --- 18:29:59 INFO [http_cmd]: http status code OK 18:29:59 DEBUG --- status code --- 18:29:59 INFO [http_cmd]: http status code OK 18:29:59 INFO [http_cmd]: http status code OK 18:29:59 DEBUG --- status code --- 18:29:59 DEBUG --- status code --- 18:29:59 DEBUG --- http response --- 18:29:59 DEBUG --- status code --- 18:29:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:29:59 DEBUG --- status code --- 18:29:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:29:59 DEBUG --- status code --- 18:29:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:29:59 DEBUG --- http response --- 18:29:59 DEBUG --- status code --- 18:29:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:29:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:29:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:29:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:29:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:29:59 DEBUG --- http response --- 18:29:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:29:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:29:59 DEBUG --- status code --- 18:29:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:29:59 INFO [http_cmd]: http status code OK 18:29:59 DEBUG --- http response --- 18:29:59 INFO [http_cmd]: http status code OK 18:29:59 DEBUG --- http response --- 18:29:59 DEBUG --- http response --- 18:29:59 DEBUG --- http response --- 18:29:59 DEBUG --- http response --- 18:29:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:29:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:29:59 DEBUG --- http response --- 18:29:59 DEBUG --- http response --- 18:29:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:29:59 DEBUG --- status code --- 18:29:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:29:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:29:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:29:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:29:59 DEBUG --- status code --- 18:29:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:29:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:29:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:29:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:29:59 DEBUG ----- output ----- 18:29:59 DEBUG --- http response --- 18:29:59 DEBUG --- http response --- 18:29:59 DEBUG { 18:29:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:29:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:29:59 DEBUG "status": "success", 18:29:59 DEBUG ----- output ----- 18:29:59 DEBUG "data": { 18:29:59 DEBUG { 18:29:59 DEBUG "resultType": "vector", 18:29:59 DEBUG "status": "success", 18:29:59 DEBUG "result": [ 18:29:59 DEBUG "data": { 18:29:59 DEBUG { 18:29:59 DEBUG "resultType": "vector", 18:29:59 DEBUG "metric": { 18:29:59 DEBUG "result": [ 18:29:59 DEBUG "pod": "ds-cts-0" 18:29:59 DEBUG { 18:29:59 DEBUG }, 18:29:59 DEBUG "metric": { 18:29:59 DEBUG "value": [ 18:29:59 DEBUG "pod": "ds-cts-0" 18:29:59 DEBUG 18:29:59 DEBUG }, 18:29:59 DEBUG 18:29:59 DEBUG "value": [ 18:29:59 DEBUG 18:29:59 DEBUG 18:29:59 DEBUG [...] 18:29:59 DEBUG 18:29:59 DEBUG 18:29:59 DEBUG 18:29:59 DEBUG 18:29:59 DEBUG [...] 18:29:59 DEBUG 18:29:59 DEBUG 18:29:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:29:59 DEBUG 18:29:59 DEBUG }, 18:29:59 DEBUG 18:29:59 DEBUG "value": [ 18:29:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:29:59 DEBUG 1734460199, 18:29:59 DEBUG }, 18:29:59 DEBUG "0.010377595209426447" 18:29:59 DEBUG "value": [ 18:29:59 DEBUG ] 18:29:59 DEBUG 1734460199, 18:29:59 DEBUG } 18:29:59 DEBUG "0" 18:29:59 DEBUG ] 18:29:59 DEBUG ] 18:29:59 DEBUG } 18:29:59 DEBUG } 18:29:59 DEBUG } 18:29:59 DEBUG ] 18:29:59 DEBUG } 18:29:59 DEBUG } 18:29:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460199" 18:29:59 INFO [http_cmd]: http status code OK 18:29:59 DEBUG --- status code --- 18:29:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:29:59 DEBUG --- http response --- 18:29:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:29:59 DEBUG ----- output ----- 18:29:59 DEBUG { 18:29:59 DEBUG "status": "success", 18:29:59 DEBUG "data": { 18:29:59 DEBUG "resultType": "vector", 18:29:59 DEBUG "result": [ 18:29:59 DEBUG { 18:29:59 DEBUG "metric": { 18:29:59 DEBUG "instance": "10.142.0.11:9100" 18:29:59 DEBUG }, 18:29:59 DEBUG "value": [ 18:29:59 DEBUG 18:29:59 DEBUG 18:29:59 DEBUG 18:29:59 DEBUG [...] 18:29:59 DEBUG 18:29:59 DEBUG 18:29:59 DEBUG 18:29:59 DEBUG "instance": "10.142.0.51:9100" 18:29:59 DEBUG }, 18:29:59 DEBUG "value": [ 18:29:59 DEBUG 1734460199, 18:29:59 DEBUG "135270.96666666665" 18:29:59 DEBUG ] 18:29:59 DEBUG } 18:29:59 DEBUG ] 18:29:59 DEBUG } 18:29:59 DEBUG } 18:29:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460199" 18:29:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734460199" 18:29:59 INFO [http_cmd]: http status code OK 18:29:59 DEBUG --- status code --- 18:29:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:29:59 DEBUG --- http response --- 18:29:59 INFO [http_cmd]: http status code OK 18:29:59 DEBUG --- status code --- 18:29:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:29:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:29:59 DEBUG --- http response --- 18:29:59 DEBUG ----- output ----- 18:29:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:29:59 DEBUG { 18:29:59 DEBUG ----- output ----- 18:29:59 DEBUG "status": "success", 18:29:59 DEBUG { 18:29:59 DEBUG "data": { 18:29:59 DEBUG "status": "success", 18:29:59 DEBUG "resultType": "vector", 18:29:59 DEBUG "data": { 18:29:59 DEBUG "result": [ 18:29:59 DEBUG "resultType": "vector", 18:29:59 DEBUG { 18:29:59 DEBUG "result": [ 18:29:59 DEBUG "metric": { 18:29:59 DEBUG { 18:29:59 DEBUG "pod": "ds-cts-0" 18:29:59 DEBUG "metric": { 18:29:59 DEBUG }, 18:29:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:29:59 DEBUG "value": [ 18:29:59 DEBUG }, 18:29:59 DEBUG 18:29:59 DEBUG "value": [ 18:29:59 DEBUG 18:29:59 DEBUG 18:29:59 DEBUG 18:29:59 DEBUG 18:29:59 DEBUG [...] 18:29:59 DEBUG 18:29:59 DEBUG 18:29:59 DEBUG [...] 18:29:59 DEBUG 18:29:59 DEBUG 18:29:59 DEBUG 18:29:59 DEBUG 18:29:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:29:59 DEBUG 18:29:59 DEBUG }, 18:29:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:29:59 DEBUG "value": [ 18:29:59 DEBUG }, 18:29:59 DEBUG 1734460199, 18:29:59 DEBUG "value": [ 18:29:59 DEBUG "711315456" 18:29:59 DEBUG 1734460199, 18:29:59 DEBUG ] 18:29:59 DEBUG "1.0604491347508533" 18:29:59 DEBUG } 18:29:59 DEBUG ] 18:29:59 DEBUG ] 18:29:59 DEBUG } 18:29:59 DEBUG } 18:29:59 DEBUG ] 18:29:59 DEBUG } 18:29:59 DEBUG } 18:29:59 DEBUG } 18:29:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460199" 18:29:59 INFO [http_cmd]: http status code OK 18:29:59 DEBUG --- status code --- 18:29:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:29:59 DEBUG --- http response --- 18:29:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:29:59 DEBUG ----- output ----- 18:29:59 DEBUG { 18:29:59 DEBUG "status": "success", 18:29:59 DEBUG "data": { 18:29:59 DEBUG "resultType": "vector", 18:29:59 DEBUG "result": [ 18:29:59 DEBUG { 18:29:59 DEBUG "metric": { 18:29:59 DEBUG "instance": "10.142.0.11:9100" 18:29:59 DEBUG }, 18:29:59 DEBUG "value": [ 18:29:59 DEBUG 18:29:59 DEBUG 18:29:59 DEBUG 18:29:59 DEBUG [...] 18:29:59 DEBUG 18:29:59 DEBUG 18:29:59 DEBUG 18:29:59 DEBUG "instance": "10.142.0.51:9100" 18:29:59 DEBUG }, 18:29:59 DEBUG "value": [ 18:29:59 DEBUG 1734460199, 18:29:59 DEBUG "133690.99999999997" 18:29:59 DEBUG ] 18:29:59 DEBUG } 18:29:59 DEBUG ] 18:29:59 DEBUG } 18:29:59 DEBUG } 18:29:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460199" 18:29:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734460199" 18:29:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460199" 18:29:59 INFO [http_cmd]: http status code OK 18:29:59 DEBUG --- status code --- 18:29:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:29:59 DEBUG --- http response --- 18:29:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:29:59 DEBUG ----- output ----- 18:29:59 DEBUG { 18:29:59 DEBUG "status": "success", 18:29:59 DEBUG "data": { 18:29:59 DEBUG "resultType": "vector", 18:29:59 DEBUG "result": [ 18:29:59 DEBUG { 18:29:59 DEBUG "metric": { 18:29:59 DEBUG "instance": "10.142.0.11:9100" 18:29:59 DEBUG }, 18:29:59 DEBUG "value": [ 18:29:59 DEBUG 18:29:59 DEBUG 18:29:59 DEBUG 18:29:59 DEBUG [...] 18:29:59 DEBUG 18:29:59 DEBUG 18:29:59 DEBUG 18:29:59 DEBUG "instance": "10.142.0.51:9100" 18:29:59 DEBUG }, 18:29:59 DEBUG "value": [ 18:29:59 DEBUG 1734460199, 18:29:59 DEBUG "26760.533333333333" 18:29:59 DEBUG ] 18:29:59 DEBUG } 18:29:59 DEBUG ] 18:29:59 DEBUG } 18:29:59 DEBUG } 18:29:59 INFO [http_cmd]: http status code OK 18:29:59 DEBUG --- status code --- 18:29:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:29:59 DEBUG --- http response --- 18:29:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:29:59 DEBUG ----- output ----- 18:29:59 DEBUG { 18:29:59 DEBUG "status": "success", 18:29:59 DEBUG "data": { 18:29:59 DEBUG "resultType": "vector", 18:29:59 DEBUG "result": [ 18:29:59 DEBUG { 18:29:59 DEBUG "metric": { 18:29:59 DEBUG "instance": "10.142.0.11:9100" 18:29:59 DEBUG }, 18:29:59 DEBUG "value": [ 18:29:59 DEBUG 18:29:59 DEBUG 18:29:59 DEBUG 18:29:59 DEBUG [...] 18:29:59 DEBUG 18:29:59 DEBUG 18:29:59 DEBUG 18:29:59 DEBUG "instance": "10.142.0.51:9100" 18:29:59 DEBUG }, 18:29:59 DEBUG "value": [ 18:29:59 DEBUG 1734460199, 18:29:59 DEBUG "0.0003333333333333854" 18:29:59 DEBUG ] 18:29:59 DEBUG } 18:29:59 DEBUG ] 18:29:59 DEBUG } 18:29:59 DEBUG } 18:29:59 INFO [http_cmd]: http status code OK 18:29:59 DEBUG --- status code --- 18:29:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:29:59 DEBUG --- http response --- 18:29:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:29:59 DEBUG ----- output ----- 18:29:59 DEBUG { 18:29:59 DEBUG "status": "success", 18:29:59 DEBUG "data": { 18:29:59 DEBUG "resultType": "vector", 18:29:59 DEBUG "result": [ 18:29:59 DEBUG { 18:29:59 DEBUG "metric": { 18:29:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:29:59 DEBUG }, 18:29:59 DEBUG "value": [ 18:29:59 DEBUG 18:29:59 DEBUG 18:29:59 DEBUG 18:29:59 DEBUG [...] 18:29:59 DEBUG 18:29:59 DEBUG 18:29:59 DEBUG 18:29:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:29:59 DEBUG }, 18:29:59 DEBUG "value": [ 18:29:59 DEBUG 1734460199, 18:29:59 DEBUG "2984624128" 18:29:59 DEBUG ] 18:29:59 DEBUG } 18:29:59 DEBUG ] 18:29:59 DEBUG } 18:29:59 DEBUG } 18:29:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460199" 18:29:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460199" 18:29:59 INFO [http_cmd]: http status code OK 18:29:59 DEBUG --- status code --- 18:29:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:29:59 DEBUG --- http response --- 18:29:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:29:59 DEBUG ----- output ----- 18:29:59 DEBUG { 18:29:59 DEBUG "status": "success", 18:29:59 DEBUG "data": { 18:29:59 DEBUG "resultType": "vector", 18:29:59 DEBUG "result": [ 18:29:59 DEBUG { 18:29:59 DEBUG "metric": { 18:29:59 DEBUG "pod": "ds-cts-0" 18:29:59 DEBUG }, 18:29:59 DEBUG "value": [ 18:29:59 DEBUG 18:29:59 DEBUG 18:29:59 DEBUG 18:29:59 DEBUG [...] 18:29:59 DEBUG 18:29:59 DEBUG 18:29:59 DEBUG 18:29:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:29:59 DEBUG }, 18:29:59 DEBUG "value": [ 18:29:59 DEBUG 1734460199, 18:29:59 DEBUG "1633.8444294634912" 18:29:59 DEBUG ] 18:29:59 DEBUG } 18:29:59 DEBUG ] 18:29:59 DEBUG } 18:29:59 DEBUG } 18:29:59 INFO [http_cmd]: http status code OK 18:29:59 DEBUG --- status code --- 18:29:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:29:59 DEBUG --- http response --- 18:29:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:29:59 DEBUG ----- output ----- 18:29:59 DEBUG { 18:29:59 DEBUG "status": "success", 18:29:59 DEBUG "data": { 18:29:59 DEBUG "resultType": "vector", 18:29:59 DEBUG "result": [ 18:29:59 DEBUG { 18:29:59 DEBUG "metric": { 18:29:59 DEBUG "instance": "10.142.0.11:9100" 18:29:59 DEBUG }, 18:29:59 DEBUG "value": [ 18:29:59 DEBUG 18:29:59 DEBUG 18:29:59 DEBUG 18:29:59 DEBUG [...] 18:29:59 DEBUG 18:29:59 DEBUG 18:29:59 DEBUG 18:29:59 DEBUG "instance": "10.142.0.51:9100" 18:29:59 DEBUG }, 18:29:59 DEBUG "value": [ 18:29:59 DEBUG 1734460199, 18:29:59 DEBUG "0.00040000000000001514" 18:29:59 DEBUG ] 18:29:59 DEBUG } 18:29:59 DEBUG ] 18:29:59 DEBUG } 18:29:59 DEBUG } 18:30:00 INFO 18:30:00 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:30:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:30:00 INFO [loop_until]: OK (rc = 0) 18:30:00 DEBUG --- stdout --- 18:30:00 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 8m 534Mi ds-idrepo-0 6m 710Mi rcs-5d696ff5b-7ttvr 3m 298Mi rcs-5d696ff5b-wwlbr 9m 338Mi 18:30:00 DEBUG --- stderr --- 18:30:00 DEBUG 18:30:00 INFO 18:30:00 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:30:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:30:00 INFO [loop_until]: OK (rc = 0) 18:30:00 DEBUG --- stdout --- 18:30:00 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 121m 0% 2285Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1133m 3% 3794Mi 13% 18:30:00 DEBUG --- stderr --- 18:30:00 DEBUG 18:30:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460258" 18:30:58 INFO [http_cmd]: http status code OK 18:30:58 DEBUG --- status code --- 18:30:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:30:58 DEBUG --- http response --- 18:30:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:30:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460259" 18:30:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734460259" 18:30:59 INFO [http_cmd]: http status code OK 18:30:59 DEBUG --- status code --- 18:30:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:30:59 DEBUG --- http response --- 18:30:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:30:59 INFO [http_cmd]: http status code OK 18:30:59 DEBUG --- status code --- 18:30:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:30:59 DEBUG --- http response --- 18:30:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:30:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460259" 18:30:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460259" 18:30:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460259" 18:30:59 INFO [http_cmd]: http status code OK 18:30:59 INFO [http_cmd]: http status code OK 18:30:59 INFO [http_cmd]: http status code OK 18:30:59 DEBUG --- status code --- 18:30:59 DEBUG --- status code --- 18:30:59 DEBUG --- status code --- 18:30:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:30:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:30:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:30:59 DEBUG --- http response --- 18:30:59 DEBUG --- http response --- 18:30:59 DEBUG --- http response --- 18:30:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:30:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:30:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:30:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460259" 18:30:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460259" 18:30:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460259" 18:30:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460259" 18:30:59 INFO [http_cmd]: http status code OK 18:30:59 DEBUG --- status code --- 18:30:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:30:59 INFO [http_cmd]: http status code OK 18:30:59 DEBUG --- http response --- 18:30:59 DEBUG --- status code --- 18:30:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:30:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:30:59 DEBUG --- http response --- 18:30:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:30:59 INFO [http_cmd]: http status code OK 18:30:59 DEBUG --- status code --- 18:30:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:30:59 DEBUG --- http response --- 18:30:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:30:59 INFO [http_cmd]: http status code OK 18:30:59 DEBUG --- status code --- 18:30:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:30:59 DEBUG --- http response --- 18:30:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:30:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460259" 18:30:59 INFO [http_cmd]: http status code OK 18:30:59 DEBUG --- status code --- 18:30:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:30:59 DEBUG --- http response --- 18:30:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:30:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460259" 18:30:59 INFO [http_cmd]: http status code OK 18:30:59 DEBUG --- status code --- 18:30:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:30:59 DEBUG --- http response --- 18:30:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:30:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460259" 18:30:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734460259" 18:30:59 ERROR [http_cmd]: ERROR 18:30:59 ERROR --- status code --- 18:30:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:30:59 ERROR --- http response --- 18:30:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:30:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:30:59 INFO [http_cmd]: http status code OK 18:30:59 DEBUG --- status code --- 18:30:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:30:59 DEBUG --- http response --- 18:30:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:30:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460259" 18:30:59 INFO [http_cmd]: http status code OK 18:30:59 DEBUG --- status code --- 18:30:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:30:59 DEBUG --- http response --- 18:30:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:30:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460259" 18:30:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460259" 18:30:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460259" 18:30:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460259" 18:30:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460259" 18:30:59 INFO [http_cmd]: http status code OK 18:30:59 DEBUG --- status code --- 18:30:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:30:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734460259" 18:30:59 DEBUG --- http response --- 18:30:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734460259" 18:30:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:30:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460259" 18:30:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460259" 18:30:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460259" 18:30:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734460259" 18:30:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460259" 18:30:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460259" 18:30:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460259" 18:30:59 INFO [http_cmd]: http status code OK 18:30:59 INFO [http_cmd]: http status code OK 18:30:59 INFO [http_cmd]: http status code OK 18:30:59 INFO [http_cmd]: http status code OK 18:30:59 DEBUG --- status code --- 18:30:59 DEBUG --- status code --- 18:30:59 DEBUG --- status code --- 18:30:59 INFO [http_cmd]: http status code OK 18:30:59 INFO [http_cmd]: http status code OK 18:30:59 INFO [http_cmd]: http status code OK 18:30:59 DEBUG --- status code --- 18:30:59 INFO [http_cmd]: http status code OK 18:30:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:30:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:30:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:30:59 DEBUG --- status code --- 18:30:59 DEBUG --- status code --- 18:30:59 DEBUG --- status code --- 18:30:59 INFO [http_cmd]: http status code OK 18:30:59 INFO [http_cmd]: http status code OK 18:30:59 DEBUG --- status code --- 18:30:59 DEBUG --- status code --- 18:30:59 DEBUG --- http response --- 18:30:59 DEBUG --- http response --- 18:30:59 DEBUG --- http response --- 18:30:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:30:59 DEBUG --- http response --- 18:30:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:30:59 DEBUG --- status code --- 18:30:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:30:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:30:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:30:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:30:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:30:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:30:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:30:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:30:59 DEBUG --- http response --- 18:30:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:30:59 DEBUG --- http response --- 18:30:59 DEBUG --- http response --- 18:30:59 DEBUG --- http response --- 18:30:59 INFO [http_cmd]: http status code OK 18:30:59 INFO [http_cmd]: http status code OK 18:30:59 DEBUG --- http response --- 18:30:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:30:59 DEBUG --- http response --- 18:30:59 INFO [http_cmd]: http status code OK 18:30:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:30:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:30:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:30:59 DEBUG --- status code --- 18:30:59 DEBUG --- status code --- 18:30:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:30:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:30:59 DEBUG --- status code --- 18:30:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:30:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:30:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:30:59 DEBUG --- http response --- 18:30:59 DEBUG --- http response --- 18:30:59 DEBUG --- http response --- 18:30:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:30:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:30:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:30:59 DEBUG ----- output ----- 18:30:59 DEBUG ----- output ----- 18:30:59 DEBUG { 18:30:59 DEBUG { 18:30:59 DEBUG "status": "success", 18:30:59 DEBUG "status": "success", 18:30:59 DEBUG "data": { 18:30:59 DEBUG "data": { 18:30:59 DEBUG "resultType": "vector", 18:30:59 DEBUG "resultType": "vector", 18:30:59 DEBUG "result": [ 18:30:59 DEBUG "result": [ 18:30:59 DEBUG { 18:30:59 DEBUG { 18:30:59 DEBUG "metric": { 18:30:59 DEBUG "metric": { 18:30:59 DEBUG "pod": "ds-cts-0" 18:30:59 DEBUG "pod": "ds-cts-0" 18:30:59 DEBUG }, 18:30:59 DEBUG }, 18:30:59 DEBUG "value": [ 18:30:59 DEBUG "value": [ 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 DEBUG [...] 18:30:59 DEBUG [...] 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:30:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:30:59 DEBUG }, 18:30:59 DEBUG }, 18:30:59 DEBUG "value": [ 18:30:59 DEBUG "value": [ 18:30:59 DEBUG 1734460259, 18:30:59 DEBUG 1734460259, 18:30:59 DEBUG "0.003825357370010013" 18:30:59 DEBUG "0" 18:30:59 DEBUG ] 18:30:59 DEBUG ] 18:30:59 DEBUG } 18:30:59 DEBUG } 18:30:59 DEBUG ] 18:30:59 DEBUG ] 18:30:59 DEBUG } 18:30:59 DEBUG } 18:30:59 DEBUG } 18:30:59 DEBUG } 18:30:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460259" 18:30:59 INFO [http_cmd]: http status code OK 18:30:59 DEBUG --- status code --- 18:30:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:30:59 DEBUG --- http response --- 18:30:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:30:59 DEBUG ----- output ----- 18:30:59 DEBUG { 18:30:59 DEBUG "status": "success", 18:30:59 DEBUG "data": { 18:30:59 DEBUG "resultType": "vector", 18:30:59 DEBUG "result": [ 18:30:59 DEBUG { 18:30:59 DEBUG "metric": { 18:30:59 DEBUG "instance": "10.142.0.11:9100" 18:30:59 DEBUG }, 18:30:59 DEBUG "value": [ 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 DEBUG [...] 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 DEBUG "instance": "10.142.0.51:9100" 18:30:59 DEBUG }, 18:30:59 DEBUG "value": [ 18:30:59 DEBUG 1734460259, 18:30:59 DEBUG "132388.23333333334" 18:30:59 DEBUG ] 18:30:59 DEBUG } 18:30:59 DEBUG ] 18:30:59 DEBUG } 18:30:59 DEBUG } 18:30:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734460259" 18:30:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460259" 18:30:59 INFO [http_cmd]: http status code OK 18:30:59 DEBUG --- status code --- 18:30:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:30:59 DEBUG --- http response --- 18:30:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:30:59 DEBUG ----- output ----- 18:30:59 DEBUG { 18:30:59 DEBUG "status": "success", 18:30:59 DEBUG "data": { 18:30:59 DEBUG "resultType": "vector", 18:30:59 DEBUG "result": [ 18:30:59 DEBUG { 18:30:59 DEBUG "metric": { 18:30:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:30:59 DEBUG }, 18:30:59 DEBUG "value": [ 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 DEBUG [...] 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 INFO [http_cmd]: http status code OK 18:30:59 DEBUG 18:30:59 DEBUG --- status code --- 18:30:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:30:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:30:59 DEBUG }, 18:30:59 DEBUG --- http response --- 18:30:59 DEBUG "value": [ 18:30:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:30:59 DEBUG 1734460259, 18:30:59 DEBUG ----- output ----- 18:30:59 DEBUG "1.0616257939527416" 18:30:59 DEBUG { 18:30:59 DEBUG ] 18:30:59 DEBUG "status": "success", 18:30:59 DEBUG } 18:30:59 DEBUG "data": { 18:30:59 DEBUG ] 18:30:59 DEBUG "resultType": "vector", 18:30:59 DEBUG } 18:30:59 DEBUG "result": [ 18:30:59 DEBUG } 18:30:59 DEBUG { 18:30:59 DEBUG "metric": { 18:30:59 DEBUG "pod": "ds-cts-0" 18:30:59 DEBUG }, 18:30:59 DEBUG "value": [ 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 DEBUG [...] 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:30:59 DEBUG }, 18:30:59 DEBUG "value": [ 18:30:59 DEBUG 1734460259, 18:30:59 DEBUG "711376896" 18:30:59 DEBUG ] 18:30:59 DEBUG } 18:30:59 DEBUG ] 18:30:59 DEBUG } 18:30:59 DEBUG } 18:30:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460259" 18:30:59 INFO [http_cmd]: http status code OK 18:30:59 DEBUG --- status code --- 18:30:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:30:59 DEBUG --- http response --- 18:30:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:30:59 DEBUG ----- output ----- 18:30:59 DEBUG { 18:30:59 DEBUG "status": "success", 18:30:59 DEBUG "data": { 18:30:59 DEBUG "resultType": "vector", 18:30:59 DEBUG "result": [ 18:30:59 DEBUG { 18:30:59 DEBUG "metric": { 18:30:59 DEBUG "instance": "10.142.0.11:9100" 18:30:59 DEBUG }, 18:30:59 DEBUG "value": [ 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 DEBUG [...] 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 DEBUG "instance": "10.142.0.51:9100" 18:30:59 DEBUG }, 18:30:59 DEBUG "value": [ 18:30:59 DEBUG 1734460259, 18:30:59 DEBUG "130628.26666666669" 18:30:59 DEBUG ] 18:30:59 DEBUG } 18:30:59 DEBUG ] 18:30:59 DEBUG } 18:30:59 DEBUG } 18:30:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460259" 18:30:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460259" 18:30:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734460259" 18:30:59 INFO [http_cmd]: http status code OK 18:30:59 DEBUG --- status code --- 18:30:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:30:59 DEBUG --- http response --- 18:30:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:30:59 DEBUG ----- output ----- 18:30:59 DEBUG { 18:30:59 DEBUG "status": "success", 18:30:59 DEBUG "data": { 18:30:59 DEBUG "resultType": "vector", 18:30:59 DEBUG "result": [ 18:30:59 DEBUG { 18:30:59 DEBUG "metric": { 18:30:59 DEBUG "instance": "10.142.0.11:9100" 18:30:59 DEBUG }, 18:30:59 DEBUG "value": [ 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 DEBUG [...] 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 DEBUG "instance": "10.142.0.51:9100" 18:30:59 DEBUG }, 18:30:59 DEBUG "value": [ 18:30:59 DEBUG 1734460259, 18:30:59 DEBUG "20480" 18:30:59 DEBUG ] 18:30:59 DEBUG } 18:30:59 DEBUG ] 18:30:59 DEBUG } 18:30:59 DEBUG } 18:30:59 INFO [http_cmd]: http status code OK 18:30:59 DEBUG --- status code --- 18:30:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:30:59 DEBUG --- http response --- 18:30:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:30:59 DEBUG ----- output ----- 18:30:59 DEBUG { 18:30:59 DEBUG "status": "success", 18:30:59 DEBUG "data": { 18:30:59 DEBUG "resultType": "vector", 18:30:59 DEBUG "result": [ 18:30:59 DEBUG { 18:30:59 DEBUG "metric": { 18:30:59 DEBUG "instance": "10.142.0.11:9100" 18:30:59 DEBUG }, 18:30:59 DEBUG "value": [ 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 DEBUG [...] 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 DEBUG "instance": "10.142.0.51:9100" 18:30:59 DEBUG }, 18:30:59 DEBUG "value": [ 18:30:59 DEBUG 1734460259, 18:30:59 DEBUG "0.000333333333333267" 18:30:59 DEBUG ] 18:30:59 DEBUG } 18:30:59 DEBUG ] 18:30:59 DEBUG } 18:30:59 DEBUG } 18:30:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460259" 18:30:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460259" 18:30:59 INFO [http_cmd]: http status code OK 18:30:59 DEBUG --- status code --- 18:30:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:30:59 DEBUG --- http response --- 18:30:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:30:59 DEBUG ----- output ----- 18:30:59 DEBUG { 18:30:59 DEBUG "status": "success", 18:30:59 DEBUG "data": { 18:30:59 DEBUG "resultType": "vector", 18:30:59 DEBUG "result": [ 18:30:59 DEBUG { 18:30:59 DEBUG "metric": { 18:30:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:30:59 DEBUG }, 18:30:59 DEBUG "value": [ 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 DEBUG [...] 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:30:59 DEBUG }, 18:30:59 DEBUG "value": [ 18:30:59 DEBUG 1734460259, 18:30:59 DEBUG "2992144384" 18:30:59 DEBUG ] 18:30:59 DEBUG } 18:30:59 DEBUG ] 18:30:59 DEBUG } 18:30:59 DEBUG } 18:30:59 INFO [http_cmd]: http status code OK 18:30:59 DEBUG --- status code --- 18:30:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:30:59 DEBUG --- http response --- 18:30:59 INFO [http_cmd]: http status code OK 18:30:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:30:59 DEBUG --- status code --- 18:30:59 DEBUG ----- output ----- 18:30:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:30:59 DEBUG { 18:30:59 DEBUG --- http response --- 18:30:59 DEBUG "status": "success", 18:30:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:30:59 DEBUG "data": { 18:30:59 DEBUG ----- output ----- 18:30:59 DEBUG "resultType": "vector", 18:30:59 DEBUG { 18:30:59 DEBUG "result": [ 18:30:59 DEBUG "status": "success", 18:30:59 DEBUG { 18:30:59 DEBUG "data": { 18:30:59 DEBUG "metric": { 18:30:59 DEBUG "resultType": "vector", 18:30:59 DEBUG "pod": "ds-cts-0" 18:30:59 DEBUG "result": [ 18:30:59 DEBUG }, 18:30:59 DEBUG { 18:30:59 DEBUG "value": [ 18:30:59 DEBUG "metric": { 18:30:59 DEBUG 18:30:59 DEBUG "instance": "10.142.0.11:9100" 18:30:59 DEBUG 18:30:59 DEBUG }, 18:30:59 DEBUG 18:30:59 DEBUG "value": [ 18:30:59 DEBUG [...] 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 DEBUG 18:30:59 DEBUG [...] 18:30:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:30:59 DEBUG 18:30:59 DEBUG }, 18:30:59 DEBUG 18:30:59 DEBUG "value": [ 18:30:59 DEBUG 18:30:59 DEBUG 1734460259, 18:30:59 DEBUG "instance": "10.142.0.51:9100" 18:30:59 DEBUG "527.5291390302016" 18:30:59 DEBUG }, 18:30:59 DEBUG ] 18:30:59 DEBUG "value": [ 18:30:59 DEBUG } 18:30:59 DEBUG 1734460259, 18:30:59 DEBUG ] 18:30:59 DEBUG "0.00043333333333303397" 18:30:59 DEBUG } 18:30:59 DEBUG ] 18:30:59 DEBUG } 18:30:59 DEBUG } 18:30:59 DEBUG ] 18:30:59 DEBUG } 18:30:59 DEBUG } 18:31:00 INFO 18:31:00 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:31:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:31:00 INFO [loop_until]: OK (rc = 0) 18:31:00 DEBUG --- stdout --- 18:31:00 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 7m 536Mi ds-idrepo-0 9m 710Mi rcs-5d696ff5b-7ttvr 3m 298Mi rcs-5d696ff5b-wwlbr 2m 338Mi 18:31:00 DEBUG --- stderr --- 18:31:00 DEBUG 18:31:00 INFO 18:31:00 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:31:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:31:00 INFO [loop_until]: OK (rc = 0) 18:31:00 DEBUG --- stdout --- 18:31:00 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 111m 0% 2282Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1134m 3% 3799Mi 13% 18:31:00 DEBUG --- stderr --- 18:31:00 DEBUG 18:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460318" 18:31:58 INFO [http_cmd]: http status code OK 18:31:58 DEBUG --- status code --- 18:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:31:58 DEBUG --- http response --- 18:31:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:31:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460319" 18:31:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734460319" 18:31:59 INFO [http_cmd]: http status code OK 18:31:59 DEBUG --- status code --- 18:31:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:31:59 DEBUG --- http response --- 18:31:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:31:59 INFO [http_cmd]: http status code OK 18:31:59 DEBUG --- status code --- 18:31:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:31:59 DEBUG --- http response --- 18:31:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:31:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460319" 18:31:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460319" 18:31:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460319" 18:31:59 INFO [http_cmd]: http status code OK 18:31:59 DEBUG --- status code --- 18:31:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:31:59 DEBUG --- http response --- 18:31:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:31:59 INFO [http_cmd]: http status code OK 18:31:59 DEBUG --- status code --- 18:31:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:31:59 DEBUG --- http response --- 18:31:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:31:59 INFO [http_cmd]: http status code OK 18:31:59 DEBUG --- status code --- 18:31:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:31:59 DEBUG --- http response --- 18:31:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:31:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460319" 18:31:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460319" 18:31:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460319" 18:31:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460319" 18:31:59 INFO [http_cmd]: http status code OK 18:31:59 DEBUG --- status code --- 18:31:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:31:59 DEBUG --- http response --- 18:31:59 INFO [http_cmd]: http status code OK 18:31:59 DEBUG --- status code --- 18:31:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:31:59 DEBUG --- http response --- 18:31:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:31:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:31:59 INFO [http_cmd]: http status code OK 18:31:59 INFO [http_cmd]: http status code OK 18:31:59 DEBUG --- status code --- 18:31:59 DEBUG --- status code --- 18:31:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:31:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:31:59 DEBUG --- http response --- 18:31:59 DEBUG --- http response --- 18:31:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:31:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:31:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460319" 18:31:59 INFO [http_cmd]: http status code OK 18:31:59 DEBUG --- status code --- 18:31:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:31:59 DEBUG --- http response --- 18:31:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:31:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460319" 18:31:59 INFO [http_cmd]: http status code OK 18:31:59 DEBUG --- status code --- 18:31:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:31:59 DEBUG --- http response --- 18:31:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:31:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734460319" 18:31:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460319" 18:31:59 INFO [http_cmd]: http status code OK 18:31:59 DEBUG --- status code --- 18:31:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:31:59 ERROR [http_cmd]: ERROR 18:31:59 DEBUG --- http response --- 18:31:59 ERROR --- status code --- 18:31:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:31:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:31:59 ERROR --- http response --- 18:31:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:31:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:31:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460319" 18:31:59 INFO [http_cmd]: http status code OK 18:31:59 DEBUG --- status code --- 18:31:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:31:59 DEBUG --- http response --- 18:31:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:31:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460319" 18:31:59 INFO [http_cmd]: http status code OK 18:31:59 DEBUG --- status code --- 18:31:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:31:59 DEBUG --- http response --- 18:31:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:31:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460319" 18:31:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460319" 18:31:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460319" 18:31:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734460319" 18:31:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460319" 18:31:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460319" 18:31:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734460319" 18:31:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460319" 18:31:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734460319" 18:31:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460319" 18:31:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460319" 18:31:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460319" 18:31:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460319" 18:31:59 INFO [http_cmd]: http status code OK 18:31:59 DEBUG --- status code --- 18:31:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:31:59 DEBUG --- http response --- 18:31:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:31:59 INFO [http_cmd]: http status code OK 18:31:59 DEBUG --- status code --- 18:31:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:31:59 DEBUG --- http response --- 18:31:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:31:59 INFO [http_cmd]: http status code OK 18:31:59 DEBUG --- status code --- 18:31:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:31:59 DEBUG --- http response --- 18:31:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:31:59 INFO [http_cmd]: http status code OK 18:31:59 DEBUG --- status code --- 18:31:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:31:59 DEBUG --- http response --- 18:31:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:31:59 INFO [http_cmd]: http status code OK 18:31:59 DEBUG --- status code --- 18:31:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:31:59 DEBUG --- http response --- 18:31:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:31:59 INFO [http_cmd]: http status code OK 18:31:59 DEBUG --- status code --- 18:31:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:31:59 DEBUG --- http response --- 18:31:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:31:59 INFO [http_cmd]: http status code OK 18:31:59 INFO [http_cmd]: http status code OK 18:31:59 INFO [http_cmd]: http status code OK 18:31:59 INFO [http_cmd]: http status code OK 18:31:59 DEBUG --- status code --- 18:31:59 INFO [http_cmd]: http status code OK 18:31:59 DEBUG --- status code --- 18:31:59 DEBUG --- status code --- 18:31:59 DEBUG --- status code --- 18:31:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:31:59 DEBUG --- status code --- 18:31:59 INFO [http_cmd]: http status code OK 18:31:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:31:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:31:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:31:59 DEBUG --- http response --- 18:31:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:31:59 DEBUG --- status code --- 18:31:59 DEBUG --- http response --- 18:31:59 DEBUG --- http response --- 18:31:59 DEBUG --- http response --- 18:31:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:31:59 DEBUG --- http response --- 18:31:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:31:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:31:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:31:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:31:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:31:59 DEBUG --- http response --- 18:31:59 DEBUG ----- output ----- 18:31:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:31:59 DEBUG { 18:31:59 DEBUG ----- output ----- 18:31:59 DEBUG "status": "success", 18:31:59 DEBUG { 18:31:59 INFO [http_cmd]: http status code OK 18:31:59 DEBUG "data": { 18:31:59 DEBUG "status": "success", 18:31:59 DEBUG --- status code --- 18:31:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460319" 18:31:59 DEBUG "resultType": "vector", 18:31:59 DEBUG "data": { 18:31:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:31:59 DEBUG "result": [ 18:31:59 DEBUG "resultType": "vector", 18:31:59 DEBUG --- http response --- 18:31:59 DEBUG { 18:31:59 DEBUG "result": [ 18:31:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:31:59 DEBUG "metric": { 18:31:59 DEBUG { 18:31:59 DEBUG "pod": "ds-cts-0" 18:31:59 DEBUG "metric": { 18:31:59 DEBUG }, 18:31:59 DEBUG "pod": "ds-cts-0" 18:31:59 DEBUG "value": [ 18:31:59 DEBUG }, 18:31:59 DEBUG 18:31:59 DEBUG "value": [ 18:31:59 DEBUG 18:31:59 DEBUG 18:31:59 DEBUG 18:31:59 DEBUG 18:31:59 DEBUG [...] 18:31:59 DEBUG 18:31:59 DEBUG 18:31:59 DEBUG [...] 18:31:59 DEBUG 18:31:59 DEBUG 18:31:59 DEBUG 18:31:59 DEBUG 18:31:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:31:59 DEBUG 18:31:59 DEBUG }, 18:31:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:31:59 DEBUG "value": [ 18:31:59 DEBUG }, 18:31:59 DEBUG 1734460319, 18:31:59 DEBUG "value": [ 18:31:59 DEBUG "0.0050027290900328085" 18:31:59 DEBUG 1734460319, 18:31:59 DEBUG ] 18:31:59 DEBUG "0" 18:31:59 DEBUG } 18:31:59 DEBUG ] 18:31:59 DEBUG ] 18:31:59 DEBUG } 18:31:59 DEBUG } 18:31:59 DEBUG ] 18:31:59 DEBUG } 18:31:59 DEBUG } 18:31:59 DEBUG } 18:31:59 INFO [http_cmd]: http status code OK 18:31:59 DEBUG --- status code --- 18:31:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:31:59 DEBUG --- http response --- 18:31:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:31:59 DEBUG ----- output ----- 18:31:59 DEBUG { 18:31:59 DEBUG "status": "success", 18:31:59 DEBUG "data": { 18:31:59 DEBUG "resultType": "vector", 18:31:59 DEBUG "result": [ 18:31:59 DEBUG { 18:31:59 DEBUG "metric": { 18:31:59 DEBUG "instance": "10.142.0.11:9100" 18:31:59 DEBUG }, 18:31:59 DEBUG "value": [ 18:31:59 DEBUG 18:31:59 DEBUG 18:31:59 DEBUG 18:31:59 DEBUG [...] 18:31:59 DEBUG 18:31:59 DEBUG 18:31:59 DEBUG 18:31:59 DEBUG "instance": "10.142.0.51:9100" 18:31:59 DEBUG }, 18:31:59 DEBUG "value": [ 18:31:59 DEBUG 1734460319, 18:31:59 DEBUG "130089.73333333335" 18:31:59 DEBUG ] 18:31:59 DEBUG } 18:31:59 DEBUG ] 18:31:59 DEBUG } 18:31:59 DEBUG } 18:31:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460319" 18:31:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734460319" 18:31:59 INFO [http_cmd]: http status code OK 18:31:59 DEBUG --- status code --- 18:31:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:31:59 DEBUG --- http response --- 18:31:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:31:59 DEBUG ----- output ----- 18:31:59 DEBUG { 18:31:59 DEBUG "status": "success", 18:31:59 DEBUG "data": { 18:31:59 DEBUG "resultType": "vector", 18:31:59 DEBUG "result": [ 18:31:59 DEBUG { 18:31:59 DEBUG "metric": { 18:31:59 DEBUG "pod": "ds-cts-0" 18:31:59 DEBUG }, 18:31:59 DEBUG "value": [ 18:31:59 DEBUG 18:31:59 DEBUG 18:31:59 DEBUG 18:31:59 DEBUG [...] 18:31:59 DEBUG 18:31:59 DEBUG 18:31:59 DEBUG 18:31:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:31:59 DEBUG }, 18:31:59 DEBUG "value": [ 18:31:59 DEBUG 1734460319, 18:31:59 DEBUG "711430144" 18:31:59 DEBUG ] 18:31:59 DEBUG } 18:31:59 DEBUG ] 18:31:59 DEBUG } 18:31:59 DEBUG } 18:31:59 INFO [http_cmd]: http status code OK 18:31:59 DEBUG --- status code --- 18:31:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:31:59 DEBUG --- http response --- 18:31:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:31:59 DEBUG ----- output ----- 18:31:59 DEBUG { 18:31:59 DEBUG "status": "success", 18:31:59 DEBUG "data": { 18:31:59 DEBUG "resultType": "vector", 18:31:59 DEBUG "result": [ 18:31:59 DEBUG { 18:31:59 DEBUG "metric": { 18:31:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:31:59 DEBUG }, 18:31:59 DEBUG "value": [ 18:31:59 DEBUG 18:31:59 DEBUG 18:31:59 DEBUG 18:31:59 DEBUG [...] 18:31:59 DEBUG 18:31:59 DEBUG 18:31:59 DEBUG 18:31:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:31:59 DEBUG }, 18:31:59 DEBUG "value": [ 18:31:59 DEBUG 1734460319, 18:31:59 DEBUG "1.0574877479239018" 18:31:59 DEBUG ] 18:31:59 DEBUG } 18:31:59 DEBUG ] 18:31:59 DEBUG } 18:31:59 DEBUG } 18:31:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460319" 18:31:59 INFO [http_cmd]: http status code OK 18:31:59 DEBUG --- status code --- 18:31:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:31:59 DEBUG --- http response --- 18:31:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:31:59 DEBUG ----- output ----- 18:31:59 DEBUG { 18:31:59 DEBUG "status": "success", 18:31:59 DEBUG "data": { 18:31:59 DEBUG "resultType": "vector", 18:31:59 DEBUG "result": [ 18:31:59 DEBUG { 18:31:59 DEBUG "metric": { 18:31:59 DEBUG "instance": "10.142.0.11:9100" 18:31:59 DEBUG }, 18:31:59 DEBUG "value": [ 18:31:59 DEBUG 18:31:59 DEBUG 18:31:59 DEBUG 18:31:59 DEBUG [...] 18:31:59 DEBUG 18:31:59 DEBUG 18:31:59 DEBUG 18:31:59 DEBUG "instance": "10.142.0.51:9100" 18:31:59 DEBUG }, 18:31:59 DEBUG "value": [ 18:31:59 DEBUG 1734460319, 18:31:59 DEBUG "130270.93333333335" 18:31:59 DEBUG ] 18:31:59 DEBUG } 18:31:59 DEBUG ] 18:31:59 DEBUG } 18:31:59 DEBUG } 18:31:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460319" 18:31:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460319" 18:31:59 INFO [http_cmd]: http status code OK 18:31:59 DEBUG --- status code --- 18:31:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:31:59 DEBUG --- http response --- 18:31:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:31:59 DEBUG ----- output ----- 18:31:59 DEBUG { 18:31:59 DEBUG "status": "success", 18:31:59 DEBUG "data": { 18:31:59 DEBUG "resultType": "vector", 18:31:59 DEBUG "result": [ 18:31:59 DEBUG { 18:31:59 DEBUG "metric": { 18:31:59 DEBUG "instance": "10.142.0.11:9100" 18:31:59 DEBUG }, 18:31:59 DEBUG "value": [ 18:31:59 DEBUG 18:31:59 DEBUG 18:31:59 DEBUG 18:31:59 DEBUG [...] 18:31:59 DEBUG 18:31:59 DEBUG 18:31:59 DEBUG 18:31:59 DEBUG "instance": "10.142.0.51:9100" 18:31:59 DEBUG }, 18:31:59 DEBUG "value": [ 18:31:59 DEBUG 1734460319, 18:31:59 DEBUG "28808.533333333333" 18:31:59 DEBUG ] 18:31:59 DEBUG } 18:31:59 DEBUG ] 18:31:59 DEBUG } 18:31:59 DEBUG } 18:31:59 INFO [http_cmd]: http status code OK 18:31:59 DEBUG --- status code --- 18:31:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:31:59 DEBUG --- http response --- 18:31:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:31:59 DEBUG ----- output ----- 18:31:59 DEBUG { 18:31:59 DEBUG "status": "success", 18:31:59 DEBUG "data": { 18:31:59 DEBUG "resultType": "vector", 18:31:59 DEBUG "result": [ 18:31:59 DEBUG { 18:31:59 DEBUG "metric": { 18:31:59 DEBUG "instance": "10.142.0.11:9100" 18:31:59 DEBUG }, 18:31:59 DEBUG "value": [ 18:31:59 DEBUG 18:31:59 DEBUG 18:31:59 DEBUG 18:31:59 DEBUG [...] 18:31:59 DEBUG 18:31:59 DEBUG 18:31:59 DEBUG 18:31:59 DEBUG "instance": "10.142.0.51:9100" 18:31:59 DEBUG }, 18:31:59 DEBUG "value": [ 18:31:59 DEBUG 1734460319, 18:31:59 DEBUG "0.0010000000000000378" 18:31:59 DEBUG ] 18:31:59 DEBUG } 18:31:59 DEBUG ] 18:31:59 DEBUG } 18:31:59 DEBUG } 18:31:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734460319" 18:31:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460319" 18:31:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460319" 18:31:59 INFO [http_cmd]: http status code OK 18:31:59 DEBUG --- status code --- 18:31:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:31:59 DEBUG --- http response --- 18:31:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:31:59 DEBUG ----- output ----- 18:31:59 DEBUG { 18:31:59 DEBUG "status": "success", 18:31:59 DEBUG "data": { 18:31:59 DEBUG "resultType": "vector", 18:31:59 DEBUG "result": [ 18:31:59 DEBUG { 18:31:59 DEBUG "metric": { 18:31:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:31:59 DEBUG }, 18:31:59 DEBUG "value": [ 18:31:59 DEBUG 18:31:59 DEBUG 18:31:59 DEBUG 18:31:59 DEBUG [...] 18:31:59 DEBUG 18:31:59 DEBUG 18:31:59 DEBUG 18:31:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:31:59 DEBUG }, 18:31:59 DEBUG "value": [ 18:31:59 DEBUG 1734460319, 18:31:59 DEBUG "3001970688" 18:31:59 DEBUG ] 18:31:59 DEBUG } 18:31:59 DEBUG ] 18:31:59 DEBUG } 18:31:59 DEBUG } 18:31:59 INFO [http_cmd]: http status code OK 18:31:59 DEBUG --- status code --- 18:31:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:31:59 INFO [http_cmd]: http status code OK 18:31:59 DEBUG --- http response --- 18:31:59 DEBUG --- status code --- 18:31:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:31:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:31:59 DEBUG ----- output ----- 18:31:59 DEBUG --- http response --- 18:31:59 DEBUG { 18:31:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:31:59 DEBUG "status": "success", 18:31:59 DEBUG ----- output ----- 18:31:59 DEBUG "data": { 18:31:59 DEBUG { 18:31:59 DEBUG "resultType": "vector", 18:31:59 DEBUG "status": "success", 18:31:59 DEBUG "result": [ 18:31:59 DEBUG "data": { 18:31:59 DEBUG { 18:31:59 DEBUG "resultType": "vector", 18:31:59 DEBUG "metric": { 18:31:59 DEBUG "result": [ 18:31:59 DEBUG "pod": "ds-cts-0" 18:31:59 DEBUG { 18:31:59 DEBUG }, 18:31:59 DEBUG "metric": { 18:31:59 DEBUG "value": [ 18:31:59 DEBUG "instance": "10.142.0.11:9100" 18:31:59 DEBUG 18:31:59 DEBUG }, 18:31:59 DEBUG 18:31:59 DEBUG "value": [ 18:31:59 DEBUG 18:31:59 DEBUG 18:31:59 DEBUG [...] 18:31:59 DEBUG 18:31:59 DEBUG 18:31:59 DEBUG 18:31:59 DEBUG 18:31:59 DEBUG [...] 18:31:59 DEBUG 18:31:59 DEBUG 18:31:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:31:59 DEBUG 18:31:59 DEBUG }, 18:31:59 DEBUG 18:31:59 DEBUG "value": [ 18:31:59 DEBUG "instance": "10.142.0.51:9100" 18:31:59 DEBUG 1734460319, 18:31:59 DEBUG }, 18:31:59 DEBUG "1211.6066765657533" 18:31:59 DEBUG "value": [ 18:31:59 DEBUG ] 18:31:59 DEBUG 1734460319, 18:31:59 DEBUG } 18:31:59 DEBUG "0.0012999999999981544" 18:31:59 DEBUG ] 18:31:59 DEBUG ] 18:31:59 DEBUG } 18:31:59 DEBUG } 18:31:59 DEBUG } 18:31:59 DEBUG ] 18:31:59 DEBUG } 18:31:59 DEBUG } 18:32:00 INFO 18:32:00 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:32:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:32:00 INFO [loop_until]: OK (rc = 0) 18:32:00 DEBUG --- stdout --- 18:32:00 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 539Mi ds-idrepo-0 6m 711Mi rcs-5d696ff5b-7ttvr 4m 298Mi rcs-5d696ff5b-wwlbr 2m 338Mi 18:32:00 DEBUG --- stderr --- 18:32:00 DEBUG 18:32:00 INFO 18:32:00 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:32:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:32:01 INFO [loop_until]: OK (rc = 0) 18:32:01 DEBUG --- stdout --- 18:32:01 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 109m 0% 2283Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1141m 3% 3803Mi 13% 18:32:01 DEBUG --- stderr --- 18:32:01 DEBUG 18:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460378" 18:32:58 INFO [http_cmd]: http status code OK 18:32:58 DEBUG --- status code --- 18:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:32:58 DEBUG --- http response --- 18:32:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:32:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734460379" 18:32:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460379" 18:32:59 INFO [http_cmd]: http status code OK 18:32:59 DEBUG --- status code --- 18:32:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:32:59 INFO [http_cmd]: http status code OK 18:32:59 DEBUG --- http response --- 18:32:59 DEBUG --- status code --- 18:32:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:32:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:32:59 DEBUG --- http response --- 18:32:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:32:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460379" 18:32:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460379" 18:32:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460379" 18:32:59 INFO [http_cmd]: http status code OK 18:32:59 DEBUG --- status code --- 18:32:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:32:59 INFO [http_cmd]: http status code OK 18:32:59 DEBUG --- http response --- 18:32:59 DEBUG --- status code --- 18:32:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:32:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:32:59 DEBUG --- http response --- 18:32:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:32:59 INFO [http_cmd]: http status code OK 18:32:59 DEBUG --- status code --- 18:32:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:32:59 DEBUG --- http response --- 18:32:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:32:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460379" 18:32:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460379" 18:32:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460379" 18:32:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460379" 18:32:59 INFO [http_cmd]: http status code OK 18:32:59 DEBUG --- status code --- 18:32:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:32:59 DEBUG --- http response --- 18:32:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:32:59 INFO [http_cmd]: http status code OK 18:32:59 DEBUG --- status code --- 18:32:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:32:59 DEBUG --- http response --- 18:32:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:32:59 INFO [http_cmd]: http status code OK 18:32:59 DEBUG --- status code --- 18:32:59 INFO [http_cmd]: http status code OK 18:32:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:32:59 DEBUG --- status code --- 18:32:59 DEBUG --- http response --- 18:32:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:32:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:32:59 DEBUG --- http response --- 18:32:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:32:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460379" 18:32:59 INFO [http_cmd]: http status code OK 18:32:59 DEBUG --- status code --- 18:32:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:32:59 DEBUG --- http response --- 18:32:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:32:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460379" 18:32:59 INFO [http_cmd]: http status code OK 18:32:59 DEBUG --- status code --- 18:32:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:32:59 DEBUG --- http response --- 18:32:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:32:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734460379" 18:32:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460379" 18:32:59 ERROR [http_cmd]: ERROR 18:32:59 ERROR --- status code --- 18:32:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:32:59 ERROR --- http response --- 18:32:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:32:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:32:59 INFO [http_cmd]: http status code OK 18:32:59 DEBUG --- status code --- 18:32:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:32:59 DEBUG --- http response --- 18:32:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:32:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460379" 18:32:59 INFO [http_cmd]: http status code OK 18:32:59 DEBUG --- status code --- 18:32:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:32:59 DEBUG --- http response --- 18:32:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:32:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460379" 18:32:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460379" 18:32:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460379" 18:32:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734460379" 18:32:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460379" 18:32:59 INFO [http_cmd]: http status code OK 18:32:59 DEBUG --- status code --- 18:32:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:32:59 DEBUG --- http response --- 18:32:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:32:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734460379" 18:32:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460379" 18:32:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460379" 18:32:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734460379" 18:32:59 INFO [http_cmd]: http status code OK 18:32:59 DEBUG --- status code --- 18:32:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:32:59 DEBUG --- http response --- 18:32:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:32:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460379" 18:32:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460379" 18:32:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460379" 18:32:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460379" 18:32:59 INFO [http_cmd]: http status code OK 18:32:59 DEBUG --- status code --- 18:32:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:32:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460379" 18:32:59 DEBUG --- http response --- 18:32:59 INFO [http_cmd]: http status code OK 18:32:59 INFO [http_cmd]: http status code OK 18:32:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460379" 18:32:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:32:59 INFO [http_cmd]: http status code OK 18:32:59 INFO [http_cmd]: http status code OK 18:32:59 DEBUG --- status code --- 18:32:59 INFO [http_cmd]: http status code OK 18:32:59 DEBUG --- status code --- 18:32:59 INFO [http_cmd]: http status code OK 18:32:59 INFO [http_cmd]: http status code OK 18:32:59 INFO [http_cmd]: http status code OK 18:32:59 DEBUG --- status code --- 18:32:59 DEBUG --- status code --- 18:32:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:32:59 DEBUG --- status code --- 18:32:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:32:59 DEBUG --- status code --- 18:32:59 INFO [http_cmd]: http status code OK 18:32:59 DEBUG --- status code --- 18:32:59 INFO [http_cmd]: http status code OK 18:32:59 DEBUG --- status code --- 18:32:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:32:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:32:59 DEBUG --- http response --- 18:32:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:32:59 DEBUG --- http response --- 18:32:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:32:59 DEBUG --- status code --- 18:32:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:32:59 DEBUG --- status code --- 18:32:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:32:59 INFO [http_cmd]: http status code OK 18:32:59 DEBUG --- http response --- 18:32:59 DEBUG --- http response --- 18:32:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:32:59 DEBUG --- http response --- 18:32:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:32:59 DEBUG --- http response --- 18:32:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:32:59 DEBUG --- http response --- 18:32:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:32:59 DEBUG --- http response --- 18:32:59 DEBUG --- status code --- 18:32:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:32:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:32:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:32:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:32:59 DEBUG --- http response --- 18:32:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:32:59 DEBUG --- http response --- 18:32:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:32:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:32:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:32:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:32:59 DEBUG --- http response --- 18:32:59 INFO [http_cmd]: http status code OK 18:32:59 DEBUG ----- output ----- 18:32:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:32:59 DEBUG --- status code --- 18:32:59 DEBUG { 18:32:59 DEBUG ----- output ----- 18:32:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:32:59 DEBUG "status": "success", 18:32:59 DEBUG { 18:32:59 DEBUG --- http response --- 18:32:59 DEBUG "data": { 18:32:59 DEBUG "status": "success", 18:32:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:32:59 DEBUG "resultType": "vector", 18:32:59 DEBUG "data": { 18:32:59 DEBUG ----- output ----- 18:32:59 DEBUG "result": [ 18:32:59 DEBUG "resultType": "vector", 18:32:59 DEBUG { 18:32:59 DEBUG { 18:32:59 DEBUG "result": [ 18:32:59 DEBUG "status": "success", 18:32:59 DEBUG "metric": { 18:32:59 DEBUG { 18:32:59 DEBUG "data": { 18:32:59 DEBUG "pod": "ds-cts-0" 18:32:59 DEBUG "metric": { 18:32:59 DEBUG "resultType": "vector", 18:32:59 DEBUG }, 18:32:59 DEBUG "pod": "ds-cts-0" 18:32:59 DEBUG "result": [ 18:32:59 DEBUG "value": [ 18:32:59 DEBUG }, 18:32:59 DEBUG { 18:32:59 DEBUG 18:32:59 DEBUG "value": [ 18:32:59 DEBUG "metric": { 18:32:59 DEBUG 18:32:59 DEBUG 18:32:59 DEBUG "instance": "10.142.0.11:9100" 18:32:59 DEBUG 18:32:59 DEBUG 18:32:59 DEBUG }, 18:32:59 DEBUG [...] 18:32:59 DEBUG 18:32:59 DEBUG "value": [ 18:32:59 DEBUG 18:32:59 DEBUG [...] 18:32:59 DEBUG 18:32:59 DEBUG 18:32:59 DEBUG 18:32:59 DEBUG 18:32:59 DEBUG 18:32:59 DEBUG 18:32:59 DEBUG 18:32:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:32:59 DEBUG 18:32:59 DEBUG [...] 18:32:59 DEBUG }, 18:32:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:32:59 DEBUG 18:32:59 DEBUG "value": [ 18:32:59 DEBUG }, 18:32:59 DEBUG 18:32:59 DEBUG 1734460379, 18:32:59 DEBUG "value": [ 18:32:59 DEBUG 18:32:59 DEBUG "0.004659863797030462" 18:32:59 DEBUG 1734460379, 18:32:59 DEBUG "instance": "10.142.0.51:9100" 18:32:59 DEBUG ] 18:32:59 DEBUG "0" 18:32:59 DEBUG }, 18:32:59 DEBUG } 18:32:59 DEBUG ] 18:32:59 DEBUG "value": [ 18:32:59 DEBUG 1734460379, 18:32:59 DEBUG } 18:32:59 DEBUG ] 18:32:59 DEBUG "129055.06666666667" 18:32:59 DEBUG ] 18:32:59 DEBUG } 18:32:59 DEBUG ] 18:32:59 DEBUG } 18:32:59 DEBUG } 18:32:59 DEBUG } 18:32:59 DEBUG ] 18:32:59 DEBUG } 18:32:59 DEBUG } 18:32:59 DEBUG } 18:32:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460379" 18:32:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734460379" 18:32:59 INFO [http_cmd]: http status code OK 18:32:59 DEBUG --- status code --- 18:32:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:32:59 DEBUG --- http response --- 18:32:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:32:59 DEBUG ----- output ----- 18:32:59 DEBUG { 18:32:59 DEBUG "status": "success", 18:32:59 DEBUG "data": { 18:32:59 DEBUG "resultType": "vector", 18:32:59 DEBUG "result": [ 18:32:59 DEBUG { 18:32:59 DEBUG "metric": { 18:32:59 DEBUG "pod": "ds-cts-0" 18:32:59 DEBUG }, 18:32:59 DEBUG "value": [ 18:32:59 DEBUG 18:32:59 DEBUG 18:32:59 DEBUG 18:32:59 DEBUG [...] 18:32:59 DEBUG 18:32:59 DEBUG 18:32:59 DEBUG 18:32:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:32:59 DEBUG }, 18:32:59 DEBUG "value": [ 18:32:59 DEBUG 1734460379, 18:32:59 DEBUG "712032256" 18:32:59 DEBUG ] 18:32:59 DEBUG } 18:32:59 DEBUG ] 18:32:59 DEBUG } 18:32:59 DEBUG } 18:32:59 INFO [http_cmd]: http status code OK 18:32:59 DEBUG --- status code --- 18:32:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:32:59 DEBUG --- http response --- 18:32:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:32:59 DEBUG ----- output ----- 18:32:59 DEBUG { 18:32:59 DEBUG "status": "success", 18:32:59 DEBUG "data": { 18:32:59 DEBUG "resultType": "vector", 18:32:59 DEBUG "result": [ 18:32:59 DEBUG { 18:32:59 DEBUG "metric": { 18:32:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:32:59 DEBUG }, 18:32:59 DEBUG "value": [ 18:32:59 DEBUG 18:32:59 DEBUG 18:32:59 DEBUG 18:32:59 DEBUG [...] 18:32:59 DEBUG 18:32:59 DEBUG 18:32:59 DEBUG 18:32:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:32:59 DEBUG }, 18:32:59 DEBUG "value": [ 18:32:59 DEBUG 1734460379, 18:32:59 DEBUG "1.0562832388036614" 18:32:59 DEBUG ] 18:32:59 DEBUG } 18:32:59 DEBUG ] 18:32:59 DEBUG } 18:32:59 DEBUG } 18:32:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460379" 18:32:59 INFO [http_cmd]: http status code OK 18:32:59 DEBUG --- status code --- 18:32:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:32:59 DEBUG --- http response --- 18:32:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:32:59 DEBUG ----- output ----- 18:32:59 DEBUG { 18:32:59 DEBUG "status": "success", 18:32:59 DEBUG "data": { 18:32:59 DEBUG "resultType": "vector", 18:32:59 DEBUG "result": [ 18:32:59 DEBUG { 18:32:59 DEBUG "metric": { 18:32:59 DEBUG "instance": "10.142.0.11:9100" 18:32:59 DEBUG }, 18:32:59 DEBUG "value": [ 18:32:59 DEBUG 18:32:59 DEBUG 18:32:59 DEBUG 18:32:59 DEBUG [...] 18:32:59 DEBUG 18:32:59 DEBUG 18:32:59 DEBUG 18:32:59 DEBUG "instance": "10.142.0.51:9100" 18:32:59 DEBUG }, 18:32:59 DEBUG "value": [ 18:32:59 DEBUG 1734460379, 18:32:59 DEBUG "128054.83333333333" 18:32:59 DEBUG ] 18:32:59 DEBUG } 18:32:59 DEBUG ] 18:32:59 DEBUG } 18:32:59 DEBUG } 18:32:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460379" 18:32:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460379" 18:32:59 INFO [http_cmd]: http status code OK 18:32:59 DEBUG --- status code --- 18:32:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:32:59 DEBUG --- http response --- 18:32:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:32:59 DEBUG ----- output ----- 18:32:59 DEBUG { 18:32:59 DEBUG "status": "success", 18:32:59 DEBUG "data": { 18:32:59 DEBUG "resultType": "vector", 18:32:59 DEBUG "result": [ 18:32:59 DEBUG { 18:32:59 DEBUG "metric": { 18:32:59 DEBUG "instance": "10.142.0.11:9100" 18:32:59 DEBUG }, 18:32:59 DEBUG "value": [ 18:32:59 DEBUG 18:32:59 DEBUG 18:32:59 DEBUG 18:32:59 DEBUG [...] 18:32:59 DEBUG 18:32:59 DEBUG 18:32:59 DEBUG 18:32:59 DEBUG "instance": "10.142.0.51:9100" 18:32:59 DEBUG }, 18:32:59 DEBUG "value": [ 18:32:59 DEBUG 1734460379, 18:32:59 DEBUG "31812.266666666666" 18:32:59 DEBUG ] 18:32:59 DEBUG } 18:32:59 DEBUG ] 18:32:59 DEBUG } 18:32:59 DEBUG } 18:32:59 INFO [http_cmd]: http status code OK 18:32:59 DEBUG --- status code --- 18:32:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:32:59 DEBUG --- http response --- 18:32:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:32:59 DEBUG ----- output ----- 18:32:59 DEBUG { 18:32:59 DEBUG "status": "success", 18:32:59 DEBUG "data": { 18:32:59 DEBUG "resultType": "vector", 18:32:59 DEBUG "result": [ 18:32:59 DEBUG { 18:32:59 DEBUG "metric": { 18:32:59 DEBUG "instance": "10.142.0.11:9100" 18:32:59 DEBUG }, 18:32:59 DEBUG "value": [ 18:32:59 DEBUG 18:32:59 DEBUG 18:32:59 DEBUG 18:32:59 DEBUG [...] 18:32:59 DEBUG 18:32:59 DEBUG 18:32:59 DEBUG 18:32:59 DEBUG "instance": "10.142.0.51:9100" 18:32:59 DEBUG }, 18:32:59 DEBUG "value": [ 18:32:59 DEBUG 1734460379, 18:32:59 DEBUG "0.000333333333333267" 18:32:59 DEBUG ] 18:32:59 DEBUG } 18:32:59 DEBUG ] 18:32:59 DEBUG } 18:32:59 DEBUG } 18:32:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734460379" 18:32:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460379" 18:32:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460379" 18:32:59 INFO [http_cmd]: http status code OK 18:32:59 DEBUG --- status code --- 18:32:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:32:59 DEBUG --- http response --- 18:32:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:32:59 DEBUG ----- output ----- 18:32:59 DEBUG { 18:32:59 DEBUG "status": "success", 18:32:59 DEBUG "data": { 18:32:59 DEBUG "resultType": "vector", 18:32:59 DEBUG "result": [ 18:32:59 DEBUG { 18:32:59 DEBUG "metric": { 18:32:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:32:59 DEBUG }, 18:32:59 DEBUG "value": [ 18:32:59 DEBUG 18:32:59 DEBUG 18:32:59 DEBUG 18:32:59 DEBUG [...] 18:32:59 DEBUG 18:32:59 DEBUG 18:32:59 DEBUG 18:32:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:32:59 DEBUG }, 18:32:59 DEBUG "value": [ 18:32:59 DEBUG 1734460379, 18:32:59 DEBUG "3011047424" 18:32:59 DEBUG ] 18:32:59 DEBUG } 18:32:59 DEBUG ] 18:32:59 DEBUG } 18:32:59 DEBUG } 18:32:59 INFO [http_cmd]: http status code OK 18:32:59 DEBUG --- status code --- 18:32:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:32:59 DEBUG --- http response --- 18:32:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:32:59 DEBUG ----- output ----- 18:32:59 DEBUG { 18:32:59 DEBUG "status": "success", 18:32:59 DEBUG "data": { 18:32:59 DEBUG "resultType": "vector", 18:32:59 DEBUG "result": [ 18:32:59 DEBUG { 18:32:59 DEBUG "metric": { 18:32:59 DEBUG "pod": "ds-cts-0" 18:32:59 DEBUG }, 18:32:59 DEBUG "value": [ 18:32:59 DEBUG 18:32:59 DEBUG 18:32:59 DEBUG 18:32:59 DEBUG [...] 18:32:59 DEBUG 18:32:59 DEBUG 18:32:59 DEBUG 18:32:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:32:59 DEBUG }, 18:32:59 DEBUG "value": [ 18:32:59 DEBUG 1734460379, 18:32:59 DEBUG "1484.9443962788814" 18:32:59 DEBUG ] 18:32:59 DEBUG } 18:32:59 DEBUG ] 18:32:59 DEBUG } 18:32:59 INFO [http_cmd]: http status code OK 18:32:59 DEBUG } 18:32:59 DEBUG --- status code --- 18:32:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:32:59 DEBUG --- http response --- 18:32:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:32:59 DEBUG ----- output ----- 18:32:59 DEBUG { 18:32:59 DEBUG "status": "success", 18:32:59 DEBUG "data": { 18:32:59 DEBUG "resultType": "vector", 18:32:59 DEBUG "result": [ 18:32:59 DEBUG { 18:32:59 DEBUG "metric": { 18:32:59 DEBUG "instance": "10.142.0.11:9100" 18:32:59 DEBUG }, 18:32:59 DEBUG "value": [ 18:32:59 DEBUG 18:32:59 DEBUG 18:32:59 DEBUG 18:32:59 DEBUG [...] 18:32:59 DEBUG 18:32:59 DEBUG 18:32:59 DEBUG 18:32:59 DEBUG "instance": "10.142.0.51:9100" 18:32:59 DEBUG }, 18:32:59 DEBUG "value": [ 18:32:59 DEBUG 1734460379, 18:32:59 DEBUG "0.0005666666666660566" 18:32:59 DEBUG ] 18:32:59 DEBUG } 18:32:59 DEBUG ] 18:32:59 DEBUG } 18:32:59 DEBUG } 18:33:00 INFO 18:33:00 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:33:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:33:00 INFO [loop_until]: OK (rc = 0) 18:33:00 DEBUG --- stdout --- 18:33:00 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 542Mi ds-idrepo-0 7m 711Mi rcs-5d696ff5b-7ttvr 5m 298Mi rcs-5d696ff5b-wwlbr 3m 339Mi 18:33:00 DEBUG --- stderr --- 18:33:00 DEBUG 18:33:01 INFO 18:33:01 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:33:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:33:01 INFO [loop_until]: OK (rc = 0) 18:33:01 DEBUG --- stdout --- 18:33:01 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 110m 0% 2284Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1143m 3% 3807Mi 13% 18:33:01 DEBUG --- stderr --- 18:33:01 DEBUG 18:33:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460438" 18:33:58 INFO [http_cmd]: http status code OK 18:33:58 DEBUG --- status code --- 18:33:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:33:58 DEBUG --- http response --- 18:33:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:33:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460439" 18:33:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734460439" 18:33:59 INFO [http_cmd]: http status code OK 18:33:59 INFO [http_cmd]: http status code OK 18:33:59 DEBUG --- status code --- 18:33:59 DEBUG --- status code --- 18:33:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:33:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:33:59 DEBUG --- http response --- 18:33:59 DEBUG --- http response --- 18:33:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:33:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:33:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460439" 18:33:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460439" 18:33:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460439" 18:33:59 INFO [http_cmd]: http status code OK 18:33:59 DEBUG --- status code --- 18:33:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:33:59 DEBUG --- http response --- 18:33:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:33:59 INFO [http_cmd]: http status code OK 18:33:59 INFO [http_cmd]: http status code OK 18:33:59 DEBUG --- status code --- 18:33:59 DEBUG --- status code --- 18:33:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:33:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:33:59 DEBUG --- http response --- 18:33:59 DEBUG --- http response --- 18:33:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:33:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:33:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460439" 18:33:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460439" 18:33:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460439" 18:33:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460439" 18:33:59 INFO [http_cmd]: http status code OK 18:33:59 INFO [http_cmd]: http status code OK 18:33:59 DEBUG --- status code --- 18:33:59 INFO [http_cmd]: http status code OK 18:33:59 INFO [http_cmd]: http status code OK 18:33:59 DEBUG --- status code --- 18:33:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:33:59 DEBUG --- status code --- 18:33:59 DEBUG --- status code --- 18:33:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:33:59 DEBUG --- http response --- 18:33:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:33:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:33:59 DEBUG --- http response --- 18:33:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:33:59 DEBUG --- http response --- 18:33:59 DEBUG --- http response --- 18:33:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:33:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:33:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:33:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460439" 18:33:59 INFO [http_cmd]: http status code OK 18:33:59 DEBUG --- status code --- 18:33:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:33:59 DEBUG --- http response --- 18:33:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:33:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460439" 18:33:59 INFO [http_cmd]: http status code OK 18:33:59 DEBUG --- status code --- 18:33:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:33:59 DEBUG --- http response --- 18:33:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:33:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734460439" 18:33:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460439" 18:33:59 ERROR [http_cmd]: ERROR 18:33:59 INFO [http_cmd]: http status code OK 18:33:59 ERROR --- status code --- 18:33:59 DEBUG --- status code --- 18:33:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:33:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:33:59 ERROR --- http response --- 18:33:59 DEBUG --- http response --- 18:33:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:33:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:33:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:33:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460439" 18:33:59 INFO [http_cmd]: http status code OK 18:33:59 DEBUG --- status code --- 18:33:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:33:59 DEBUG --- http response --- 18:33:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:33:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460439" 18:33:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460439" 18:33:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460439" 18:33:59 INFO [http_cmd]: http status code OK 18:33:59 DEBUG --- status code --- 18:33:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:33:59 INFO [http_cmd]: http status code OK 18:33:59 DEBUG --- http response --- 18:33:59 DEBUG --- status code --- 18:33:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:33:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:33:59 DEBUG --- http response --- 18:33:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:33:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734460439" 18:33:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460439" 18:33:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460439" 18:33:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460439" 18:33:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460439" 18:33:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460439" 18:33:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460439" 18:33:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734460439" 18:33:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734460439" 18:33:59 INFO [http_cmd]: http status code OK 18:33:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460439" 18:33:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460439" 18:33:59 DEBUG --- status code --- 18:33:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460439" 18:33:59 INFO [http_cmd]: http status code OK 18:33:59 DEBUG --- status code --- 18:33:59 INFO [http_cmd]: http status code OK 18:33:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:33:59 INFO [http_cmd]: http status code OK 18:33:59 INFO [http_cmd]: http status code OK 18:33:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:33:59 INFO [http_cmd]: http status code OK 18:33:59 INFO [http_cmd]: http status code OK 18:33:59 DEBUG --- status code --- 18:33:59 INFO [http_cmd]: http status code OK 18:33:59 DEBUG --- http response --- 18:33:59 DEBUG --- status code --- 18:33:59 DEBUG --- status code --- 18:33:59 DEBUG --- http response --- 18:33:59 DEBUG --- status code --- 18:33:59 DEBUG --- status code --- 18:33:59 INFO [http_cmd]: http status code OK 18:33:59 INFO [http_cmd]: http status code OK 18:33:59 DEBUG --- status code --- 18:33:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:33:59 DEBUG --- http response --- 18:33:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:33:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:33:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:33:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:33:59 INFO [http_cmd]: http status code OK 18:33:59 INFO [http_cmd]: http status code OK 18:33:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:33:59 DEBUG --- status code --- 18:33:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:33:59 DEBUG --- status code --- 18:33:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:33:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:33:59 DEBUG --- http response --- 18:33:59 DEBUG --- http response --- 18:33:59 DEBUG --- status code --- 18:33:59 DEBUG --- status code --- 18:33:59 DEBUG --- http response --- 18:33:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:33:59 DEBUG --- http response --- 18:33:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:33:59 DEBUG --- http response --- 18:33:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:33:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:33:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:33:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:33:59 INFO [http_cmd]: http status code OK 18:33:59 DEBUG --- status code --- 18:33:59 DEBUG --- http response --- 18:33:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:33:59 DEBUG --- http response --- 18:33:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:33:59 DEBUG --- http response --- 18:33:59 DEBUG --- http response --- 18:33:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:33:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:33:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:33:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:33:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:33:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:33:59 DEBUG --- http response --- 18:33:59 DEBUG ----- output ----- 18:33:59 DEBUG ----- output ----- 18:33:59 DEBUG { 18:33:59 DEBUG { 18:33:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:33:59 DEBUG "status": "success", 18:33:59 DEBUG "status": "success", 18:33:59 DEBUG ----- output ----- 18:33:59 DEBUG "data": { 18:33:59 DEBUG "data": { 18:33:59 DEBUG { 18:33:59 DEBUG "resultType": "vector", 18:33:59 DEBUG "resultType": "vector", 18:33:59 DEBUG "status": "success", 18:33:59 DEBUG "result": [ 18:33:59 DEBUG "result": [ 18:33:59 DEBUG "data": { 18:33:59 DEBUG { 18:33:59 DEBUG { 18:33:59 DEBUG "resultType": "vector", 18:33:59 DEBUG "metric": { 18:33:59 DEBUG "metric": { 18:33:59 DEBUG "result": [ 18:33:59 DEBUG "pod": "ds-cts-0" 18:33:59 DEBUG "pod": "ds-cts-0" 18:33:59 DEBUG { 18:33:59 DEBUG }, 18:33:59 DEBUG }, 18:33:59 DEBUG "metric": { 18:33:59 DEBUG "value": [ 18:33:59 DEBUG "value": [ 18:33:59 DEBUG "instance": "10.142.0.11:9100" 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG }, 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG "value": [ 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG [...] 18:33:59 DEBUG [...] 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG [...] 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:33:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:33:59 DEBUG 18:33:59 DEBUG }, 18:33:59 DEBUG }, 18:33:59 DEBUG 18:33:59 DEBUG "value": [ 18:33:59 DEBUG "value": [ 18:33:59 DEBUG "instance": "10.142.0.51:9100" 18:33:59 DEBUG 1734460439, 18:33:59 DEBUG 1734460439, 18:33:59 DEBUG }, 18:33:59 DEBUG "0.0027803511758036385" 18:33:59 DEBUG "0" 18:33:59 DEBUG "value": [ 18:33:59 DEBUG ] 18:33:59 DEBUG ] 18:33:59 DEBUG 1734460439, 18:33:59 DEBUG } 18:33:59 DEBUG } 18:33:59 DEBUG "128520.06666666668" 18:33:59 DEBUG ] 18:33:59 DEBUG ] 18:33:59 DEBUG ] 18:33:59 DEBUG } 18:33:59 DEBUG } 18:33:59 DEBUG } 18:33:59 DEBUG } 18:33:59 DEBUG } 18:33:59 DEBUG ] 18:33:59 DEBUG } 18:33:59 DEBUG } 18:33:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460439" 18:33:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734460439" 18:33:59 INFO [http_cmd]: http status code OK 18:33:59 DEBUG --- status code --- 18:33:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:33:59 DEBUG --- http response --- 18:33:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:33:59 DEBUG ----- output ----- 18:33:59 DEBUG { 18:33:59 DEBUG "status": "success", 18:33:59 DEBUG "data": { 18:33:59 DEBUG "resultType": "vector", 18:33:59 DEBUG "result": [ 18:33:59 DEBUG { 18:33:59 DEBUG "metric": { 18:33:59 DEBUG "pod": "ds-cts-0" 18:33:59 DEBUG }, 18:33:59 DEBUG "value": [ 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG [...] 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:33:59 DEBUG }, 18:33:59 DEBUG "value": [ 18:33:59 DEBUG 1734460439, 18:33:59 DEBUG "712032256" 18:33:59 DEBUG ] 18:33:59 DEBUG } 18:33:59 DEBUG ] 18:33:59 DEBUG } 18:33:59 DEBUG } 18:33:59 INFO [http_cmd]: http status code OK 18:33:59 DEBUG --- status code --- 18:33:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:33:59 DEBUG --- http response --- 18:33:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:33:59 DEBUG ----- output ----- 18:33:59 DEBUG { 18:33:59 DEBUG "status": "success", 18:33:59 DEBUG "data": { 18:33:59 DEBUG "resultType": "vector", 18:33:59 DEBUG "result": [ 18:33:59 DEBUG { 18:33:59 DEBUG "metric": { 18:33:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:33:59 DEBUG }, 18:33:59 DEBUG "value": [ 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG [...] 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:33:59 DEBUG }, 18:33:59 DEBUG "value": [ 18:33:59 DEBUG 1734460439, 18:33:59 DEBUG "1.0626913310455086" 18:33:59 DEBUG ] 18:33:59 DEBUG } 18:33:59 DEBUG ] 18:33:59 DEBUG } 18:33:59 DEBUG } 18:33:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460439" 18:33:59 INFO [http_cmd]: http status code OK 18:33:59 DEBUG --- status code --- 18:33:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:33:59 DEBUG --- http response --- 18:33:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:33:59 DEBUG ----- output ----- 18:33:59 DEBUG { 18:33:59 DEBUG "status": "success", 18:33:59 DEBUG "data": { 18:33:59 DEBUG "resultType": "vector", 18:33:59 DEBUG "result": [ 18:33:59 DEBUG { 18:33:59 DEBUG "metric": { 18:33:59 DEBUG "instance": "10.142.0.11:9100" 18:33:59 DEBUG }, 18:33:59 DEBUG "value": [ 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG [...] 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG "instance": "10.142.0.51:9100" 18:33:59 DEBUG }, 18:33:59 DEBUG "value": [ 18:33:59 DEBUG 1734460439, 18:33:59 DEBUG "126371.7666666667" 18:33:59 DEBUG ] 18:33:59 DEBUG } 18:33:59 DEBUG ] 18:33:59 DEBUG } 18:33:59 DEBUG } 18:33:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460439" 18:33:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460439" 18:33:59 INFO [http_cmd]: http status code OK 18:33:59 DEBUG --- status code --- 18:33:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:33:59 DEBUG --- http response --- 18:33:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:33:59 DEBUG ----- output ----- 18:33:59 DEBUG { 18:33:59 DEBUG "status": "success", 18:33:59 DEBUG "data": { 18:33:59 DEBUG "resultType": "vector", 18:33:59 DEBUG "result": [ 18:33:59 DEBUG { 18:33:59 DEBUG "metric": { 18:33:59 DEBUG "instance": "10.142.0.11:9100" 18:33:59 DEBUG }, 18:33:59 DEBUG "value": [ 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG [...] 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG "instance": "10.142.0.51:9100" 18:33:59 DEBUG }, 18:33:59 DEBUG "value": [ 18:33:59 DEBUG 1734460439, 18:33:59 DEBUG "28535.466666666667" 18:33:59 DEBUG ] 18:33:59 DEBUG } 18:33:59 DEBUG ] 18:33:59 DEBUG } 18:33:59 DEBUG } 18:33:59 INFO [http_cmd]: http status code OK 18:33:59 DEBUG --- status code --- 18:33:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:33:59 DEBUG --- http response --- 18:33:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:33:59 DEBUG ----- output ----- 18:33:59 DEBUG { 18:33:59 DEBUG "status": "success", 18:33:59 DEBUG "data": { 18:33:59 DEBUG "resultType": "vector", 18:33:59 DEBUG "result": [ 18:33:59 DEBUG { 18:33:59 DEBUG "metric": { 18:33:59 DEBUG "instance": "10.142.0.11:9100" 18:33:59 DEBUG }, 18:33:59 DEBUG "value": [ 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG [...] 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG "instance": "10.142.0.51:9100" 18:33:59 DEBUG }, 18:33:59 DEBUG "value": [ 18:33:59 DEBUG 1734460439, 18:33:59 DEBUG "0.0003333333333333854" 18:33:59 DEBUG ] 18:33:59 DEBUG } 18:33:59 DEBUG ] 18:33:59 DEBUG } 18:33:59 DEBUG } 18:33:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734460439" 18:33:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460439" 18:33:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460439" 18:33:59 INFO [http_cmd]: http status code OK 18:33:59 DEBUG --- status code --- 18:33:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:33:59 DEBUG --- http response --- 18:33:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:33:59 DEBUG ----- output ----- 18:33:59 DEBUG { 18:33:59 DEBUG "status": "success", 18:33:59 DEBUG "data": { 18:33:59 DEBUG "resultType": "vector", 18:33:59 DEBUG "result": [ 18:33:59 DEBUG { 18:33:59 DEBUG "metric": { 18:33:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:33:59 DEBUG }, 18:33:59 DEBUG "value": [ 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG [...] 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:33:59 DEBUG }, 18:33:59 DEBUG "value": [ 18:33:59 DEBUG 1734460439, 18:33:59 DEBUG "3013169152" 18:33:59 DEBUG ] 18:33:59 DEBUG } 18:33:59 DEBUG ] 18:33:59 DEBUG } 18:33:59 DEBUG } 18:33:59 INFO [http_cmd]: http status code OK 18:33:59 DEBUG --- status code --- 18:33:59 INFO [http_cmd]: http status code OK 18:33:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:33:59 DEBUG --- status code --- 18:33:59 DEBUG --- http response --- 18:33:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:33:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:33:59 DEBUG --- http response --- 18:33:59 DEBUG ----- output ----- 18:33:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:33:59 DEBUG { 18:33:59 DEBUG ----- output ----- 18:33:59 DEBUG "status": "success", 18:33:59 DEBUG { 18:33:59 DEBUG "data": { 18:33:59 DEBUG "status": "success", 18:33:59 DEBUG "resultType": "vector", 18:33:59 DEBUG "data": { 18:33:59 DEBUG "result": [ 18:33:59 DEBUG "resultType": "vector", 18:33:59 DEBUG { 18:33:59 DEBUG "result": [ 18:33:59 DEBUG "metric": { 18:33:59 DEBUG { 18:33:59 DEBUG "instance": "10.142.0.11:9100" 18:33:59 DEBUG "metric": { 18:33:59 DEBUG }, 18:33:59 DEBUG "pod": "ds-cts-0" 18:33:59 DEBUG "value": [ 18:33:59 DEBUG }, 18:33:59 DEBUG 18:33:59 DEBUG "value": [ 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG [...] 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG [...] 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG 18:33:59 DEBUG "instance": "10.142.0.51:9100" 18:33:59 DEBUG 18:33:59 DEBUG }, 18:33:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:33:59 DEBUG "value": [ 18:33:59 DEBUG }, 18:33:59 DEBUG 1734460439, 18:33:59 DEBUG "value": [ 18:33:59 DEBUG "0.0005333333333344588" 18:33:59 DEBUG 1734460439, 18:33:59 DEBUG ] 18:33:59 DEBUG "1327.9756999713209" 18:33:59 DEBUG } 18:33:59 DEBUG ] 18:33:59 DEBUG ] 18:33:59 DEBUG } 18:33:59 DEBUG } 18:33:59 DEBUG ] 18:33:59 DEBUG } 18:33:59 DEBUG } 18:33:59 DEBUG } 18:34:00 INFO 18:34:00 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:34:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:34:01 INFO [loop_until]: OK (rc = 0) 18:34:01 DEBUG --- stdout --- 18:34:01 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 542Mi ds-idrepo-0 6m 711Mi rcs-5d696ff5b-7ttvr 5m 299Mi rcs-5d696ff5b-wwlbr 2m 339Mi 18:34:01 DEBUG --- stderr --- 18:34:01 DEBUG 18:34:01 INFO 18:34:01 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:34:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:34:01 INFO [loop_until]: OK (rc = 0) 18:34:01 DEBUG --- stdout --- 18:34:01 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 107m 0% 2284Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1136m 3% 3808Mi 13% 18:34:01 DEBUG --- stderr --- 18:34:01 DEBUG 18:34:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460498" 18:34:58 INFO [http_cmd]: http status code OK 18:34:58 DEBUG --- status code --- 18:34:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:34:58 DEBUG --- http response --- 18:34:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:34:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460499" 18:34:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734460499" 18:34:59 INFO [http_cmd]: http status code OK 18:34:59 DEBUG --- status code --- 18:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:34:59 DEBUG --- http response --- 18:34:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:34:59 INFO [http_cmd]: http status code OK 18:34:59 DEBUG --- status code --- 18:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:34:59 DEBUG --- http response --- 18:34:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:34:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460499" 18:34:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460499" 18:34:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460499" 18:34:59 INFO [http_cmd]: http status code OK 18:34:59 DEBUG --- status code --- 18:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:34:59 DEBUG --- http response --- 18:34:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:34:59 INFO [http_cmd]: http status code OK 18:34:59 DEBUG --- status code --- 18:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:34:59 DEBUG --- http response --- 18:34:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:34:59 INFO [http_cmd]: http status code OK 18:34:59 DEBUG --- status code --- 18:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:34:59 DEBUG --- http response --- 18:34:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:34:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460499" 18:34:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460499" 18:34:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460499" 18:34:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460499" 18:34:59 INFO [http_cmd]: http status code OK 18:34:59 DEBUG --- status code --- 18:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:34:59 DEBUG --- http response --- 18:34:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:34:59 INFO [http_cmd]: http status code OK 18:34:59 DEBUG --- status code --- 18:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:34:59 DEBUG --- http response --- 18:34:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:34:59 INFO [http_cmd]: http status code OK 18:34:59 DEBUG --- status code --- 18:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:34:59 DEBUG --- http response --- 18:34:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:34:59 INFO [http_cmd]: http status code OK 18:34:59 DEBUG --- status code --- 18:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:34:59 DEBUG --- http response --- 18:34:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:34:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460499" 18:34:59 INFO [http_cmd]: http status code OK 18:34:59 DEBUG --- status code --- 18:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:34:59 DEBUG --- http response --- 18:34:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:34:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460499" 18:34:59 INFO [http_cmd]: http status code OK 18:34:59 DEBUG --- status code --- 18:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:34:59 DEBUG --- http response --- 18:34:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:34:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734460499" 18:34:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460499" 18:34:59 INFO [http_cmd]: http status code OK 18:34:59 DEBUG --- status code --- 18:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:34:59 DEBUG --- http response --- 18:34:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:34:59 ERROR [http_cmd]: ERROR 18:34:59 ERROR --- status code --- 18:34:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:34:59 ERROR --- http response --- 18:34:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:34:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:34:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460499" 18:34:59 INFO [http_cmd]: http status code OK 18:34:59 DEBUG --- status code --- 18:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:34:59 DEBUG --- http response --- 18:34:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:34:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460499" 18:34:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460499" 18:34:59 INFO [http_cmd]: http status code OK 18:34:59 INFO [http_cmd]: http status code OK 18:34:59 DEBUG --- status code --- 18:34:59 DEBUG --- status code --- 18:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:34:59 DEBUG --- http response --- 18:34:59 DEBUG --- http response --- 18:34:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:34:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:34:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460499" 18:34:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460499" 18:34:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460499" 18:34:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460499" 18:34:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460499" 18:34:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460499" 18:34:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734460499" 18:34:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734460499" 18:34:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460499" 18:34:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734460499" 18:34:59 INFO [http_cmd]: http status code OK 18:34:59 INFO [http_cmd]: http status code OK 18:34:59 INFO [http_cmd]: http status code OK 18:34:59 DEBUG --- status code --- 18:34:59 INFO [http_cmd]: http status code OK 18:34:59 DEBUG --- status code --- 18:34:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460499" 18:34:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460499" 18:34:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460499" 18:34:59 INFO [http_cmd]: http status code OK 18:34:59 DEBUG --- status code --- 18:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:34:59 DEBUG --- status code --- 18:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:34:59 INFO [http_cmd]: http status code OK 18:34:59 INFO [http_cmd]: http status code OK 18:34:59 DEBUG --- status code --- 18:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:34:59 DEBUG --- http response --- 18:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:34:59 DEBUG --- http response --- 18:34:59 DEBUG --- status code --- 18:34:59 DEBUG --- status code --- 18:34:59 INFO [http_cmd]: http status code OK 18:34:59 DEBUG --- http response --- 18:34:59 DEBUG --- http response --- 18:34:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:34:59 DEBUG --- http response --- 18:34:59 INFO [http_cmd]: http status code OK 18:34:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:34:59 DEBUG --- status code --- 18:34:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:34:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:34:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:34:59 DEBUG --- status code --- 18:34:59 DEBUG --- http response --- 18:34:59 DEBUG --- http response --- 18:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:34:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:34:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:34:59 DEBUG --- http response --- 18:34:59 DEBUG --- http response --- 18:34:59 INFO [http_cmd]: http status code OK 18:34:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:34:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:34:59 INFO [http_cmd]: http status code OK 18:34:59 DEBUG --- status code --- 18:34:59 DEBUG --- status code --- 18:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:34:59 INFO [http_cmd]: http status code OK 18:34:59 DEBUG --- http response --- 18:34:59 INFO [http_cmd]: http status code OK 18:34:59 DEBUG --- http response --- 18:34:59 DEBUG --- status code --- 18:34:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:34:59 DEBUG --- status code --- 18:34:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:34:59 DEBUG ----- output ----- 18:34:59 DEBUG --- http response --- 18:34:59 DEBUG --- http response --- 18:34:59 DEBUG { 18:34:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:34:59 DEBUG "status": "success", 18:34:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:34:59 DEBUG ----- output ----- 18:34:59 DEBUG "data": { 18:34:59 DEBUG ----- output ----- 18:34:59 DEBUG { 18:34:59 DEBUG "resultType": "vector", 18:34:59 DEBUG { 18:34:59 DEBUG "status": "success", 18:34:59 DEBUG "result": [ 18:34:59 DEBUG "status": "success", 18:34:59 DEBUG "data": { 18:34:59 DEBUG { 18:34:59 DEBUG "data": { 18:34:59 DEBUG "resultType": "vector", 18:34:59 DEBUG "metric": { 18:34:59 DEBUG "resultType": "vector", 18:34:59 DEBUG "result": [ 18:34:59 DEBUG "pod": "ds-cts-0" 18:34:59 DEBUG "result": [ 18:34:59 DEBUG { 18:34:59 DEBUG }, 18:34:59 DEBUG { 18:34:59 DEBUG "metric": { 18:34:59 DEBUG "value": [ 18:34:59 DEBUG "metric": { 18:34:59 DEBUG "instance": "10.142.0.11:9100" 18:34:59 DEBUG 18:34:59 DEBUG "pod": "ds-cts-0" 18:34:59 DEBUG }, 18:34:59 DEBUG 18:34:59 DEBUG }, 18:34:59 DEBUG "value": [ 18:34:59 DEBUG 18:34:59 DEBUG "value": [ 18:34:59 DEBUG 18:34:59 DEBUG [...] 18:34:59 DEBUG 18:34:59 DEBUG 18:34:59 DEBUG 18:34:59 DEBUG 18:34:59 DEBUG 18:34:59 DEBUG 18:34:59 DEBUG 18:34:59 DEBUG [...] 18:34:59 DEBUG 18:34:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:34:59 DEBUG 18:34:59 DEBUG [...] 18:34:59 DEBUG }, 18:34:59 DEBUG 18:34:59 DEBUG 18:34:59 DEBUG "value": [ 18:34:59 DEBUG 18:34:59 DEBUG 18:34:59 DEBUG 1734460499, 18:34:59 DEBUG "instance": "10.142.0.51:9100" 18:34:59 DEBUG 18:34:59 DEBUG "0.0038775395975319754" 18:34:59 DEBUG }, 18:34:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:34:59 DEBUG ] 18:34:59 DEBUG "value": [ 18:34:59 DEBUG }, 18:34:59 DEBUG } 18:34:59 DEBUG 1734460499, 18:34:59 DEBUG "value": [ 18:34:59 DEBUG ] 18:34:59 DEBUG "129249.63333333335" 18:34:59 DEBUG 1734460499, 18:34:59 DEBUG } 18:34:59 DEBUG ] 18:34:59 DEBUG } 18:34:59 DEBUG } 18:34:59 DEBUG "0" 18:34:59 DEBUG ] 18:34:59 DEBUG ] 18:34:59 DEBUG } 18:34:59 DEBUG } 18:34:59 DEBUG } 18:34:59 DEBUG ] 18:34:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460499" 18:34:59 DEBUG } 18:34:59 DEBUG } 18:34:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734460499" 18:34:59 INFO [http_cmd]: http status code OK 18:34:59 DEBUG --- status code --- 18:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:34:59 DEBUG --- http response --- 18:34:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:34:59 DEBUG ----- output ----- 18:34:59 DEBUG { 18:34:59 DEBUG "status": "success", 18:34:59 DEBUG "data": { 18:34:59 DEBUG "resultType": "vector", 18:34:59 DEBUG "result": [ 18:34:59 DEBUG { 18:34:59 DEBUG "metric": { 18:34:59 DEBUG "pod": "ds-cts-0" 18:34:59 DEBUG }, 18:34:59 DEBUG "value": [ 18:34:59 DEBUG 18:34:59 DEBUG 18:34:59 DEBUG 18:34:59 DEBUG [...] 18:34:59 DEBUG 18:34:59 DEBUG 18:34:59 DEBUG 18:34:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:34:59 DEBUG }, 18:34:59 DEBUG "value": [ 18:34:59 DEBUG 1734460499, 18:34:59 DEBUG "712093696" 18:34:59 DEBUG ] 18:34:59 DEBUG } 18:34:59 DEBUG ] 18:34:59 DEBUG } 18:34:59 DEBUG } 18:34:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460499" 18:34:59 INFO [http_cmd]: http status code OK 18:34:59 DEBUG --- status code --- 18:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:34:59 DEBUG --- http response --- 18:34:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:34:59 DEBUG ----- output ----- 18:34:59 DEBUG { 18:34:59 DEBUG "status": "success", 18:34:59 DEBUG "data": { 18:34:59 DEBUG "resultType": "vector", 18:34:59 DEBUG "result": [ 18:34:59 DEBUG { 18:34:59 DEBUG "metric": { 18:34:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:34:59 DEBUG }, 18:34:59 DEBUG "value": [ 18:34:59 DEBUG 18:34:59 DEBUG 18:34:59 DEBUG 18:34:59 DEBUG [...] 18:34:59 DEBUG 18:34:59 DEBUG 18:34:59 DEBUG 18:34:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:34:59 DEBUG }, 18:34:59 DEBUG "value": [ 18:34:59 DEBUG 1734460499, 18:34:59 DEBUG "1.0564893481826039" 18:34:59 DEBUG ] 18:34:59 DEBUG } 18:34:59 DEBUG ] 18:34:59 DEBUG } 18:34:59 DEBUG } 18:34:59 INFO [http_cmd]: http status code OK 18:34:59 DEBUG --- status code --- 18:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:34:59 DEBUG --- http response --- 18:34:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:34:59 DEBUG ----- output ----- 18:34:59 DEBUG { 18:34:59 DEBUG "status": "success", 18:34:59 DEBUG "data": { 18:34:59 DEBUG "resultType": "vector", 18:34:59 DEBUG "result": [ 18:34:59 DEBUG { 18:34:59 DEBUG "metric": { 18:34:59 DEBUG "instance": "10.142.0.11:9100" 18:34:59 DEBUG }, 18:34:59 DEBUG "value": [ 18:34:59 DEBUG 18:34:59 DEBUG 18:34:59 DEBUG 18:34:59 DEBUG [...] 18:34:59 DEBUG 18:34:59 DEBUG 18:34:59 DEBUG 18:34:59 DEBUG "instance": "10.142.0.51:9100" 18:34:59 DEBUG }, 18:34:59 DEBUG "value": [ 18:34:59 DEBUG 1734460499, 18:34:59 DEBUG "126987.66666666667" 18:34:59 DEBUG ] 18:34:59 DEBUG } 18:34:59 DEBUG ] 18:34:59 DEBUG } 18:34:59 DEBUG } 18:34:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460499" 18:34:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460499" 18:34:59 INFO [http_cmd]: http status code OK 18:34:59 DEBUG --- status code --- 18:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:34:59 DEBUG --- http response --- 18:34:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:34:59 DEBUG ----- output ----- 18:34:59 DEBUG { 18:34:59 DEBUG "status": "success", 18:34:59 DEBUG "data": { 18:34:59 DEBUG "resultType": "vector", 18:34:59 DEBUG "result": [ 18:34:59 DEBUG { 18:34:59 DEBUG "metric": { 18:34:59 DEBUG "instance": "10.142.0.11:9100" 18:34:59 DEBUG }, 18:34:59 DEBUG "value": [ 18:34:59 DEBUG 18:34:59 DEBUG 18:34:59 DEBUG 18:34:59 DEBUG [...] 18:34:59 DEBUG 18:34:59 DEBUG 18:34:59 DEBUG 18:34:59 DEBUG "instance": "10.142.0.51:9100" 18:34:59 DEBUG }, 18:34:59 DEBUG "value": [ 18:34:59 DEBUG 1734460499, 18:34:59 DEBUG "19660.8" 18:34:59 DEBUG ] 18:34:59 DEBUG } 18:34:59 DEBUG ] 18:34:59 DEBUG } 18:34:59 DEBUG } 18:34:59 INFO [http_cmd]: http status code OK 18:34:59 DEBUG --- status code --- 18:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:34:59 DEBUG --- http response --- 18:34:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:34:59 DEBUG ----- output ----- 18:34:59 DEBUG { 18:34:59 DEBUG "status": "success", 18:34:59 DEBUG "data": { 18:34:59 DEBUG "resultType": "vector", 18:34:59 DEBUG "result": [ 18:34:59 DEBUG { 18:34:59 DEBUG "metric": { 18:34:59 DEBUG "instance": "10.142.0.11:9100" 18:34:59 DEBUG }, 18:34:59 DEBUG "value": [ 18:34:59 DEBUG 18:34:59 DEBUG 18:34:59 DEBUG 18:34:59 DEBUG [...] 18:34:59 DEBUG 18:34:59 DEBUG 18:34:59 DEBUG 18:34:59 DEBUG "instance": "10.142.0.51:9100" 18:34:59 DEBUG }, 18:34:59 DEBUG "value": [ 18:34:59 DEBUG 1734460499, 18:34:59 DEBUG "0.000666666666666534" 18:34:59 DEBUG ] 18:34:59 DEBUG } 18:34:59 DEBUG ] 18:34:59 DEBUG } 18:34:59 DEBUG } 18:34:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734460499" 18:34:59 INFO [http_cmd]: http status code OK 18:34:59 DEBUG --- status code --- 18:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:34:59 DEBUG --- http response --- 18:34:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:34:59 DEBUG ----- output ----- 18:34:59 DEBUG { 18:34:59 DEBUG "status": "success", 18:34:59 DEBUG "data": { 18:34:59 DEBUG "resultType": "vector", 18:34:59 DEBUG "result": [ 18:34:59 DEBUG { 18:34:59 DEBUG "metric": { 18:34:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:34:59 DEBUG }, 18:34:59 DEBUG "value": [ 18:34:59 DEBUG 18:34:59 DEBUG 18:34:59 DEBUG 18:34:59 DEBUG [...] 18:34:59 DEBUG 18:34:59 DEBUG 18:34:59 DEBUG 18:34:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:34:59 DEBUG }, 18:34:59 DEBUG "value": [ 18:34:59 DEBUG 1734460499, 18:34:59 DEBUG "3024035840" 18:34:59 DEBUG ] 18:34:59 DEBUG } 18:34:59 DEBUG ] 18:34:59 DEBUG } 18:34:59 DEBUG } 18:34:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460499" 18:34:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460499" 18:34:59 INFO [http_cmd]: http status code OK 18:34:59 DEBUG --- status code --- 18:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:34:59 DEBUG --- http response --- 18:34:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:34:59 DEBUG ----- output ----- 18:34:59 DEBUG { 18:34:59 DEBUG "status": "success", 18:34:59 DEBUG "data": { 18:34:59 DEBUG "resultType": "vector", 18:34:59 DEBUG "result": [ 18:34:59 DEBUG { 18:34:59 DEBUG "metric": { 18:34:59 DEBUG "instance": "10.142.0.11:9100" 18:34:59 DEBUG }, 18:34:59 DEBUG "value": [ 18:34:59 DEBUG 18:34:59 DEBUG 18:34:59 DEBUG 18:34:59 DEBUG [...] 18:34:59 DEBUG 18:34:59 DEBUG 18:34:59 DEBUG 18:34:59 DEBUG "instance": "10.142.0.51:9100" 18:34:59 DEBUG }, 18:34:59 DEBUG "value": [ 18:34:59 DEBUG 1734460499, 18:34:59 DEBUG "0.0005666666666660566" 18:34:59 DEBUG ] 18:34:59 DEBUG } 18:34:59 DEBUG ] 18:34:59 DEBUG } 18:34:59 DEBUG } 18:34:59 INFO [http_cmd]: http status code OK 18:34:59 DEBUG --- status code --- 18:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:34:59 DEBUG --- http response --- 18:34:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:34:59 DEBUG ----- output ----- 18:34:59 DEBUG { 18:34:59 DEBUG "status": "success", 18:34:59 DEBUG "data": { 18:34:59 DEBUG "resultType": "vector", 18:34:59 DEBUG "result": [ 18:34:59 DEBUG { 18:34:59 DEBUG "metric": { 18:34:59 DEBUG "pod": "ds-cts-0" 18:34:59 DEBUG }, 18:34:59 DEBUG "value": [ 18:34:59 DEBUG 18:34:59 DEBUG 18:34:59 DEBUG 18:34:59 DEBUG [...] 18:34:59 DEBUG 18:34:59 DEBUG 18:34:59 DEBUG 18:34:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:34:59 DEBUG }, 18:34:59 DEBUG "value": [ 18:34:59 DEBUG 1734460499, 18:34:59 DEBUG "479.16240166115875" 18:34:59 DEBUG ] 18:34:59 DEBUG } 18:34:59 DEBUG ] 18:34:59 DEBUG } 18:34:59 DEBUG } 18:35:01 INFO 18:35:01 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:35:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:35:01 INFO [loop_until]: OK (rc = 0) 18:35:01 DEBUG --- stdout --- 18:35:01 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 5m 542Mi ds-idrepo-0 7m 713Mi rcs-5d696ff5b-7ttvr 4m 298Mi rcs-5d696ff5b-wwlbr 3m 339Mi 18:35:01 DEBUG --- stderr --- 18:35:01 DEBUG 18:35:01 INFO 18:35:01 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:35:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:35:01 INFO [loop_until]: OK (rc = 0) 18:35:01 DEBUG --- stdout --- 18:35:01 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 104m 0% 2288Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1132m 3% 3813Mi 13% 18:35:01 DEBUG --- stderr --- 18:35:01 DEBUG 18:35:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460558" 18:35:58 INFO [http_cmd]: http status code OK 18:35:58 DEBUG --- status code --- 18:35:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:35:58 DEBUG --- http response --- 18:35:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460559" 18:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734460559" 18:35:59 INFO [http_cmd]: http status code OK 18:35:59 DEBUG --- status code --- 18:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:35:59 DEBUG --- http response --- 18:35:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:35:59 INFO [http_cmd]: http status code OK 18:35:59 DEBUG --- status code --- 18:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:35:59 DEBUG --- http response --- 18:35:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460559" 18:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460559" 18:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460559" 18:35:59 INFO [http_cmd]: http status code OK 18:35:59 DEBUG --- status code --- 18:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:35:59 DEBUG --- http response --- 18:35:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:35:59 INFO [http_cmd]: http status code OK 18:35:59 DEBUG --- status code --- 18:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:35:59 DEBUG --- http response --- 18:35:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:35:59 INFO [http_cmd]: http status code OK 18:35:59 DEBUG --- status code --- 18:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:35:59 DEBUG --- http response --- 18:35:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460559" 18:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460559" 18:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460559" 18:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460559" 18:35:59 INFO [http_cmd]: http status code OK 18:35:59 INFO [http_cmd]: http status code OK 18:35:59 DEBUG --- status code --- 18:35:59 INFO [http_cmd]: http status code OK 18:35:59 DEBUG --- status code --- 18:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:35:59 DEBUG --- status code --- 18:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:35:59 DEBUG --- http response --- 18:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:35:59 DEBUG --- http response --- 18:35:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:35:59 INFO [http_cmd]: http status code OK 18:35:59 DEBUG --- http response --- 18:35:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:35:59 DEBUG --- status code --- 18:35:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:35:59 DEBUG --- http response --- 18:35:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460559" 18:35:59 INFO [http_cmd]: http status code OK 18:35:59 DEBUG --- status code --- 18:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:35:59 DEBUG --- http response --- 18:35:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460559" 18:35:59 INFO [http_cmd]: http status code OK 18:35:59 DEBUG --- status code --- 18:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:35:59 DEBUG --- http response --- 18:35:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460559" 18:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734460559" 18:35:59 INFO [http_cmd]: http status code OK 18:35:59 DEBUG --- status code --- 18:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:35:59 DEBUG --- http response --- 18:35:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:35:59 ERROR [http_cmd]: ERROR 18:35:59 ERROR --- status code --- 18:35:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:35:59 ERROR --- http response --- 18:35:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:35:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460559" 18:35:59 INFO [http_cmd]: http status code OK 18:35:59 DEBUG --- status code --- 18:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:35:59 DEBUG --- http response --- 18:35:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460559" 18:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460559" 18:35:59 INFO [http_cmd]: http status code OK 18:35:59 DEBUG --- status code --- 18:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:35:59 INFO [http_cmd]: http status code OK 18:35:59 DEBUG --- http response --- 18:35:59 DEBUG --- status code --- 18:35:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:35:59 DEBUG --- http response --- 18:35:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460559" 18:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460559" 18:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460559" 18:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460559" 18:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460559" 18:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734460559" 18:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460559" 18:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460559" 18:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734460559" 18:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734460559" 18:35:59 INFO [http_cmd]: http status code OK 18:35:59 DEBUG --- status code --- 18:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:35:59 DEBUG --- http response --- 18:35:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460559" 18:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460559" 18:35:59 INFO [http_cmd]: http status code OK 18:35:59 DEBUG --- status code --- 18:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:35:59 DEBUG --- http response --- 18:35:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460559" 18:35:59 INFO [http_cmd]: http status code OK 18:35:59 INFO [http_cmd]: http status code OK 18:35:59 INFO [http_cmd]: http status code OK 18:35:59 INFO [http_cmd]: http status code OK 18:35:59 INFO [http_cmd]: http status code OK 18:35:59 DEBUG --- status code --- 18:35:59 INFO [http_cmd]: http status code OK 18:35:59 INFO [http_cmd]: http status code OK 18:35:59 INFO [http_cmd]: http status code OK 18:35:59 INFO [http_cmd]: http status code OK 18:35:59 DEBUG --- status code --- 18:35:59 DEBUG --- status code --- 18:35:59 DEBUG --- status code --- 18:35:59 DEBUG --- status code --- 18:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:35:59 DEBUG --- status code --- 18:35:59 DEBUG --- status code --- 18:35:59 INFO [http_cmd]: http status code OK 18:35:59 DEBUG --- status code --- 18:35:59 DEBUG --- status code --- 18:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:35:59 DEBUG --- http response --- 18:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:35:59 DEBUG --- status code --- 18:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460559" 18:35:59 DEBUG --- http response --- 18:35:59 DEBUG --- http response --- 18:35:59 DEBUG --- http response --- 18:35:59 DEBUG --- http response --- 18:35:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:35:59 DEBUG --- http response --- 18:35:59 DEBUG --- http response --- 18:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:35:59 DEBUG --- http response --- 18:35:59 DEBUG --- http response --- 18:35:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:35:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:35:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:35:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:35:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:35:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:35:59 DEBUG --- http response --- 18:35:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:35:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:35:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:35:59 INFO [http_cmd]: http status code OK 18:35:59 DEBUG ----- output ----- 18:35:59 DEBUG ----- output ----- 18:35:59 DEBUG --- status code --- 18:35:59 DEBUG { 18:35:59 DEBUG { 18:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:35:59 DEBUG "status": "success", 18:35:59 DEBUG "status": "success", 18:35:59 DEBUG --- http response --- 18:35:59 DEBUG "data": { 18:35:59 DEBUG "data": { 18:35:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:35:59 DEBUG "resultType": "vector", 18:35:59 DEBUG "resultType": "vector", 18:35:59 DEBUG ----- output ----- 18:35:59 DEBUG "result": [ 18:35:59 DEBUG "result": [ 18:35:59 DEBUG { 18:35:59 DEBUG { 18:35:59 DEBUG { 18:35:59 DEBUG "status": "success", 18:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734460559" 18:35:59 DEBUG "metric": { 18:35:59 DEBUG "metric": { 18:35:59 DEBUG "data": { 18:35:59 DEBUG "pod": "ds-cts-0" 18:35:59 DEBUG "pod": "ds-cts-0" 18:35:59 DEBUG "resultType": "vector", 18:35:59 DEBUG }, 18:35:59 DEBUG }, 18:35:59 DEBUG "result": [ 18:35:59 DEBUG "value": [ 18:35:59 DEBUG "value": [ 18:35:59 DEBUG { 18:35:59 DEBUG 18:35:59 DEBUG 18:35:59 DEBUG "metric": { 18:35:59 DEBUG 18:35:59 DEBUG 18:35:59 DEBUG "instance": "10.142.0.11:9100" 18:35:59 DEBUG 18:35:59 DEBUG 18:35:59 DEBUG }, 18:35:59 INFO [http_cmd]: http status code OK 18:35:59 DEBUG [...] 18:35:59 DEBUG [...] 18:35:59 DEBUG "value": [ 18:35:59 DEBUG --- status code --- 18:35:59 DEBUG 18:35:59 DEBUG 18:35:59 DEBUG 18:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:35:59 DEBUG 18:35:59 DEBUG 18:35:59 DEBUG 18:35:59 DEBUG --- http response --- 18:35:59 DEBUG 18:35:59 DEBUG 18:35:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:35:59 DEBUG }, 18:35:59 DEBUG "value": [ 18:35:59 DEBUG 1734460559, 18:35:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:35:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:35:59 DEBUG 18:35:59 DEBUG "0.0027484859847319403" 18:35:59 DEBUG ----- output ----- 18:35:59 DEBUG }, 18:35:59 DEBUG [...] 18:35:59 DEBUG ] 18:35:59 DEBUG { 18:35:59 DEBUG "value": [ 18:35:59 DEBUG 18:35:59 DEBUG } 18:35:59 DEBUG "status": "success", 18:35:59 DEBUG 1734460559, 18:35:59 DEBUG 18:35:59 DEBUG ] 18:35:59 DEBUG "data": { 18:35:59 DEBUG "0" 18:35:59 DEBUG 18:35:59 DEBUG } 18:35:59 DEBUG "resultType": "vector", 18:35:59 DEBUG ] 18:35:59 DEBUG "instance": "10.142.0.51:9100" 18:35:59 DEBUG } 18:35:59 DEBUG "result": [ 18:35:59 DEBUG } 18:35:59 DEBUG }, 18:35:59 DEBUG { 18:35:59 DEBUG ] 18:35:59 DEBUG "value": [ 18:35:59 DEBUG "metric": { 18:35:59 DEBUG } 18:35:59 DEBUG 1734460559, 18:35:59 DEBUG "pod": "ds-cts-0" 18:35:59 DEBUG } 18:35:59 DEBUG "129174.26666666666" 18:35:59 DEBUG }, 18:35:59 DEBUG ] 18:35:59 DEBUG "value": [ 18:35:59 DEBUG } 18:35:59 DEBUG 18:35:59 DEBUG ] 18:35:59 DEBUG 18:35:59 DEBUG } 18:35:59 DEBUG 18:35:59 DEBUG } 18:35:59 DEBUG [...] 18:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460559" 18:35:59 DEBUG 18:35:59 DEBUG 18:35:59 DEBUG 18:35:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:35:59 DEBUG }, 18:35:59 DEBUG "value": [ 18:35:59 DEBUG 1734460559, 18:35:59 DEBUG "712110080" 18:35:59 DEBUG ] 18:35:59 DEBUG } 18:35:59 DEBUG ] 18:35:59 DEBUG } 18:35:59 DEBUG } 18:35:59 INFO [http_cmd]: http status code OK 18:35:59 DEBUG --- status code --- 18:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:35:59 DEBUG --- http response --- 18:35:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:35:59 DEBUG ----- output ----- 18:35:59 DEBUG { 18:35:59 DEBUG "status": "success", 18:35:59 DEBUG "data": { 18:35:59 DEBUG "resultType": "vector", 18:35:59 DEBUG "result": [ 18:35:59 DEBUG { 18:35:59 DEBUG "metric": { 18:35:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:35:59 DEBUG }, 18:35:59 DEBUG "value": [ 18:35:59 DEBUG 18:35:59 DEBUG 18:35:59 DEBUG 18:35:59 DEBUG [...] 18:35:59 DEBUG 18:35:59 DEBUG 18:35:59 DEBUG 18:35:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:35:59 DEBUG }, 18:35:59 DEBUG "value": [ 18:35:59 DEBUG 1734460559, 18:35:59 DEBUG "1.0645070949361826" 18:35:59 DEBUG ] 18:35:59 DEBUG } 18:35:59 DEBUG ] 18:35:59 DEBUG } 18:35:59 DEBUG } 18:35:59 INFO [http_cmd]: http status code OK 18:35:59 DEBUG --- status code --- 18:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:35:59 DEBUG --- http response --- 18:35:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:35:59 DEBUG ----- output ----- 18:35:59 DEBUG { 18:35:59 DEBUG "status": "success", 18:35:59 DEBUG "data": { 18:35:59 DEBUG "resultType": "vector", 18:35:59 DEBUG "result": [ 18:35:59 DEBUG { 18:35:59 DEBUG "metric": { 18:35:59 DEBUG "instance": "10.142.0.11:9100" 18:35:59 DEBUG }, 18:35:59 DEBUG "value": [ 18:35:59 DEBUG 18:35:59 DEBUG 18:35:59 DEBUG 18:35:59 DEBUG [...] 18:35:59 DEBUG 18:35:59 DEBUG 18:35:59 DEBUG 18:35:59 DEBUG "instance": "10.142.0.51:9100" 18:35:59 DEBUG }, 18:35:59 DEBUG "value": [ 18:35:59 DEBUG 1734460559, 18:35:59 DEBUG "127506.8" 18:35:59 DEBUG ] 18:35:59 DEBUG } 18:35:59 DEBUG ] 18:35:59 DEBUG } 18:35:59 DEBUG } 18:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460559" 18:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460559" 18:35:59 INFO [http_cmd]: http status code OK 18:35:59 DEBUG --- status code --- 18:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:35:59 DEBUG --- http response --- 18:35:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:35:59 DEBUG ----- output ----- 18:35:59 DEBUG { 18:35:59 DEBUG "status": "success", 18:35:59 DEBUG "data": { 18:35:59 DEBUG "resultType": "vector", 18:35:59 DEBUG "result": [ 18:35:59 DEBUG { 18:35:59 DEBUG "metric": { 18:35:59 DEBUG "instance": "10.142.0.11:9100" 18:35:59 DEBUG }, 18:35:59 DEBUG "value": [ 18:35:59 DEBUG 18:35:59 DEBUG 18:35:59 DEBUG 18:35:59 DEBUG [...] 18:35:59 DEBUG 18:35:59 DEBUG 18:35:59 DEBUG 18:35:59 DEBUG "instance": "10.142.0.51:9100" 18:35:59 DEBUG }, 18:35:59 DEBUG "value": [ 18:35:59 DEBUG 1734460559, 18:35:59 DEBUG "30310.399999999998" 18:35:59 DEBUG ] 18:35:59 DEBUG } 18:35:59 DEBUG ] 18:35:59 DEBUG } 18:35:59 DEBUG } 18:35:59 INFO [http_cmd]: http status code OK 18:35:59 DEBUG --- status code --- 18:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:35:59 DEBUG --- http response --- 18:35:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:35:59 DEBUG ----- output ----- 18:35:59 DEBUG { 18:35:59 DEBUG "status": "success", 18:35:59 DEBUG "data": { 18:35:59 DEBUG "resultType": "vector", 18:35:59 DEBUG "result": [ 18:35:59 DEBUG { 18:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734460559" 18:35:59 DEBUG "metric": { 18:35:59 DEBUG "instance": "10.142.0.11:9100" 18:35:59 DEBUG }, 18:35:59 DEBUG "value": [ 18:35:59 DEBUG 18:35:59 DEBUG 18:35:59 DEBUG 18:35:59 DEBUG [...] 18:35:59 DEBUG 18:35:59 DEBUG 18:35:59 DEBUG 18:35:59 DEBUG "instance": "10.142.0.51:9100" 18:35:59 DEBUG }, 18:35:59 DEBUG "value": [ 18:35:59 DEBUG 1734460559, 18:35:59 DEBUG "0.0003333333333333854" 18:35:59 DEBUG ] 18:35:59 DEBUG } 18:35:59 DEBUG ] 18:35:59 DEBUG } 18:35:59 DEBUG } 18:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460559" 18:35:59 INFO [http_cmd]: http status code OK 18:35:59 DEBUG --- status code --- 18:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:35:59 DEBUG --- http response --- 18:35:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460559" 18:35:59 DEBUG ----- output ----- 18:35:59 DEBUG { 18:35:59 DEBUG "status": "success", 18:35:59 DEBUG "data": { 18:35:59 DEBUG "resultType": "vector", 18:35:59 DEBUG "result": [ 18:35:59 DEBUG { 18:35:59 DEBUG "metric": { 18:35:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:35:59 DEBUG }, 18:35:59 DEBUG "value": [ 18:35:59 DEBUG 18:35:59 DEBUG 18:35:59 DEBUG 18:35:59 DEBUG [...] 18:35:59 DEBUG 18:35:59 DEBUG 18:35:59 DEBUG 18:35:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:35:59 DEBUG }, 18:35:59 DEBUG "value": [ 18:35:59 DEBUG 1734460559, 18:35:59 DEBUG "3032317952" 18:35:59 DEBUG ] 18:35:59 DEBUG } 18:35:59 DEBUG ] 18:35:59 DEBUG } 18:35:59 DEBUG } 18:35:59 INFO [http_cmd]: http status code OK 18:35:59 DEBUG --- status code --- 18:35:59 INFO [http_cmd]: http status code OK 18:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:35:59 DEBUG --- status code --- 18:35:59 DEBUG --- http response --- 18:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:35:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:35:59 DEBUG --- http response --- 18:35:59 DEBUG ----- output ----- 18:35:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:35:59 DEBUG { 18:35:59 DEBUG ----- output ----- 18:35:59 DEBUG "status": "success", 18:35:59 DEBUG { 18:35:59 DEBUG "data": { 18:35:59 DEBUG "status": "success", 18:35:59 DEBUG "resultType": "vector", 18:35:59 DEBUG "data": { 18:35:59 DEBUG "result": [ 18:35:59 DEBUG "resultType": "vector", 18:35:59 DEBUG { 18:35:59 DEBUG "result": [ 18:35:59 DEBUG "metric": { 18:35:59 DEBUG { 18:35:59 DEBUG "instance": "10.142.0.11:9100" 18:35:59 DEBUG "metric": { 18:35:59 DEBUG }, 18:35:59 DEBUG "pod": "ds-cts-0" 18:35:59 DEBUG "value": [ 18:35:59 DEBUG }, 18:35:59 DEBUG 18:35:59 DEBUG "value": [ 18:35:59 DEBUG 18:35:59 DEBUG 18:35:59 DEBUG 18:35:59 DEBUG 18:35:59 DEBUG [...] 18:35:59 DEBUG 18:35:59 DEBUG 18:35:59 DEBUG [...] 18:35:59 DEBUG 18:35:59 DEBUG 18:35:59 DEBUG 18:35:59 DEBUG 18:35:59 DEBUG "instance": "10.142.0.51:9100" 18:35:59 DEBUG 18:35:59 DEBUG }, 18:35:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:35:59 DEBUG "value": [ 18:35:59 DEBUG }, 18:35:59 DEBUG 1734460559, 18:35:59 DEBUG "value": [ 18:35:59 DEBUG "0.0004666666666693686" 18:35:59 DEBUG 1734460559, 18:35:59 DEBUG ] 18:35:59 DEBUG "2181.024082262084" 18:35:59 DEBUG } 18:35:59 DEBUG ] 18:35:59 DEBUG ] 18:35:59 DEBUG } 18:35:59 DEBUG } 18:35:59 DEBUG ] 18:35:59 DEBUG } 18:35:59 DEBUG } 18:35:59 DEBUG } 18:36:01 INFO 18:36:01 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:36:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:36:01 INFO [loop_until]: OK (rc = 0) 18:36:01 DEBUG --- stdout --- 18:36:01 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 543Mi ds-idrepo-0 7m 713Mi rcs-5d696ff5b-7ttvr 3m 298Mi rcs-5d696ff5b-wwlbr 2m 339Mi 18:36:01 DEBUG --- stderr --- 18:36:01 DEBUG 18:36:01 INFO 18:36:01 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:36:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:36:01 INFO [loop_until]: OK (rc = 0) 18:36:01 DEBUG --- stdout --- 18:36:01 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 110m 0% 2289Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1133m 3% 3814Mi 13% 18:36:01 DEBUG --- stderr --- 18:36:01 DEBUG 18:36:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460618" 18:36:58 INFO [http_cmd]: http status code OK 18:36:58 DEBUG --- status code --- 18:36:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:36:58 DEBUG --- http response --- 18:36:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460619" 18:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734460619" 18:36:59 INFO [http_cmd]: http status code OK 18:36:59 DEBUG --- status code --- 18:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:36:59 DEBUG --- http response --- 18:36:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:36:59 INFO [http_cmd]: http status code OK 18:36:59 DEBUG --- status code --- 18:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:36:59 DEBUG --- http response --- 18:36:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460619" 18:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460619" 18:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460619" 18:36:59 INFO [http_cmd]: http status code OK 18:36:59 DEBUG --- status code --- 18:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:36:59 DEBUG --- http response --- 18:36:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:36:59 INFO [http_cmd]: http status code OK 18:36:59 DEBUG --- status code --- 18:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:36:59 DEBUG --- http response --- 18:36:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:36:59 INFO [http_cmd]: http status code OK 18:36:59 DEBUG --- status code --- 18:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:36:59 DEBUG --- http response --- 18:36:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460619" 18:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460619" 18:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460619" 18:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460619" 18:36:59 INFO [http_cmd]: http status code OK 18:36:59 DEBUG --- status code --- 18:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:36:59 DEBUG --- http response --- 18:36:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:36:59 INFO [http_cmd]: http status code OK 18:36:59 DEBUG --- status code --- 18:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:36:59 DEBUG --- http response --- 18:36:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:36:59 INFO [http_cmd]: http status code OK 18:36:59 DEBUG --- status code --- 18:36:59 INFO [http_cmd]: http status code OK 18:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:36:59 DEBUG --- status code --- 18:36:59 DEBUG --- http response --- 18:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:36:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:36:59 DEBUG --- http response --- 18:36:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460619" 18:36:59 INFO [http_cmd]: http status code OK 18:36:59 DEBUG --- status code --- 18:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:36:59 DEBUG --- http response --- 18:36:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460619" 18:36:59 INFO [http_cmd]: http status code OK 18:36:59 DEBUG --- status code --- 18:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:36:59 DEBUG --- http response --- 18:36:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460619" 18:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734460619" 18:36:59 INFO [http_cmd]: http status code OK 18:36:59 DEBUG --- status code --- 18:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:36:59 ERROR [http_cmd]: ERROR 18:36:59 DEBUG --- http response --- 18:36:59 ERROR --- status code --- 18:36:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:36:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:36:59 ERROR --- http response --- 18:36:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:36:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460619" 18:36:59 INFO [http_cmd]: http status code OK 18:36:59 DEBUG --- status code --- 18:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:36:59 DEBUG --- http response --- 18:36:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460619" 18:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460619" 18:36:59 INFO [http_cmd]: http status code OK 18:36:59 DEBUG --- status code --- 18:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:36:59 DEBUG --- http response --- 18:36:59 INFO [http_cmd]: http status code OK 18:36:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:36:59 DEBUG --- status code --- 18:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:36:59 DEBUG --- http response --- 18:36:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460619" 18:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460619" 18:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460619" 18:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460619" 18:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460619" 18:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734460619" 18:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460619" 18:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734460619" 18:36:59 INFO [http_cmd]: http status code OK 18:36:59 DEBUG --- status code --- 18:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:36:59 DEBUG --- http response --- 18:36:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734460619" 18:36:59 INFO [http_cmd]: http status code OK 18:36:59 DEBUG --- status code --- 18:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:36:59 DEBUG --- http response --- 18:36:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460619" 18:36:59 INFO [http_cmd]: http status code OK 18:36:59 DEBUG --- status code --- 18:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:36:59 DEBUG --- http response --- 18:36:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:36:59 INFO [http_cmd]: http status code OK 18:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460619" 18:36:59 INFO [http_cmd]: http status code OK 18:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460619" 18:36:59 INFO [http_cmd]: http status code OK 18:36:59 DEBUG --- status code --- 18:36:59 INFO [http_cmd]: http status code OK 18:36:59 DEBUG --- status code --- 18:36:59 INFO [http_cmd]: http status code OK 18:36:59 INFO [http_cmd]: http status code OK 18:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460619" 18:36:59 DEBUG --- status code --- 18:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460619" 18:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:36:59 DEBUG --- status code --- 18:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:36:59 DEBUG --- status code --- 18:36:59 DEBUG --- status code --- 18:36:59 INFO [http_cmd]: http status code OK 18:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734460619" 18:36:59 DEBUG --- http response --- 18:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:36:59 DEBUG --- http response --- 18:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:36:59 DEBUG --- status code --- 18:36:59 DEBUG --- http response --- 18:36:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:36:59 DEBUG --- http response --- 18:36:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:36:59 DEBUG --- http response --- 18:36:59 DEBUG --- http response --- 18:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:36:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:36:59 INFO [http_cmd]: http status code OK 18:36:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:36:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:36:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:36:59 DEBUG --- http response --- 18:36:59 DEBUG --- status code --- 18:36:59 INFO [http_cmd]: http status code OK 18:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460619" 18:36:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:36:59 INFO [http_cmd]: http status code OK 18:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:36:59 DEBUG --- status code --- 18:36:59 DEBUG --- status code --- 18:36:59 DEBUG --- http response --- 18:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:36:59 DEBUG --- http response --- 18:36:59 DEBUG --- http response --- 18:36:59 DEBUG ----- output ----- 18:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:36:59 INFO [http_cmd]: http status code OK 18:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:36:59 DEBUG { 18:36:59 INFO [http_cmd]: http status code OK 18:36:59 DEBUG ----- output ----- 18:36:59 DEBUG --- status code --- 18:36:59 DEBUG ----- output ----- 18:36:59 DEBUG "status": "success", 18:36:59 DEBUG --- status code --- 18:36:59 DEBUG { 18:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:36:59 DEBUG { 18:36:59 DEBUG "data": { 18:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:36:59 DEBUG "status": "success", 18:36:59 DEBUG --- http response --- 18:36:59 DEBUG "status": "success", 18:36:59 DEBUG "resultType": "vector", 18:36:59 DEBUG --- http response --- 18:36:59 DEBUG "data": { 18:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:36:59 DEBUG "data": { 18:36:59 DEBUG "result": [ 18:36:59 DEBUG "resultType": "vector", 18:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:36:59 DEBUG ----- output ----- 18:36:59 DEBUG "resultType": "vector", 18:36:59 DEBUG { 18:36:59 DEBUG "result": [ 18:36:59 DEBUG ----- output ----- 18:36:59 DEBUG { 18:36:59 DEBUG "result": [ 18:36:59 DEBUG { 18:36:59 DEBUG { 18:36:59 DEBUG { 18:36:59 DEBUG "status": "success", 18:36:59 DEBUG "metric": { 18:36:59 DEBUG "metric": { 18:36:59 DEBUG "metric": { 18:36:59 DEBUG "status": "success", 18:36:59 DEBUG "data": { 18:36:59 DEBUG "pod": "ds-cts-0" 18:36:59 DEBUG "pod": "ds-cts-0" 18:36:59 DEBUG "pod": "ds-cts-0" 18:36:59 DEBUG "data": { 18:36:59 DEBUG "resultType": "vector", 18:36:59 DEBUG }, 18:36:59 DEBUG }, 18:36:59 DEBUG }, 18:36:59 DEBUG "resultType": "vector", 18:36:59 DEBUG "result": [ 18:36:59 DEBUG "value": [ 18:36:59 DEBUG "value": [ 18:36:59 DEBUG "value": [ 18:36:59 DEBUG "result": [ 18:36:59 DEBUG { 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG { 18:36:59 DEBUG "metric": { 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG "metric": { 18:36:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG "instance": "10.142.0.11:9100" 18:36:59 DEBUG }, 18:36:59 DEBUG [...] 18:36:59 DEBUG [...] 18:36:59 DEBUG [...] 18:36:59 DEBUG }, 18:36:59 DEBUG "value": [ 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG "value": [ 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:36:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:36:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:36:59 DEBUG 18:36:59 DEBUG [...] 18:36:59 DEBUG }, 18:36:59 DEBUG }, 18:36:59 DEBUG }, 18:36:59 DEBUG [...] 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG "value": [ 18:36:59 DEBUG "value": [ 18:36:59 DEBUG 18:36:59 DEBUG "value": [ 18:36:59 DEBUG 18:36:59 DEBUG 1734460619, 18:36:59 DEBUG 1734460619, 18:36:59 DEBUG 18:36:59 DEBUG 1734460619, 18:36:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:36:59 DEBUG "0.0031988565556479998" 18:36:59 DEBUG "712220672" 18:36:59 DEBUG 18:36:59 DEBUG "0" 18:36:59 DEBUG }, 18:36:59 DEBUG ] 18:36:59 INFO [http_cmd]: http status code OK 18:36:59 DEBUG ] 18:36:59 DEBUG "instance": "10.142.0.51:9100" 18:36:59 DEBUG ] 18:36:59 DEBUG "value": [ 18:36:59 DEBUG } 18:36:59 DEBUG --- status code --- 18:36:59 DEBUG } 18:36:59 DEBUG }, 18:36:59 DEBUG } 18:36:59 DEBUG 1734460619, 18:36:59 DEBUG ] 18:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:36:59 DEBUG ] 18:36:59 DEBUG "value": [ 18:36:59 DEBUG ] 18:36:59 DEBUG "1.0552859128671033" 18:36:59 DEBUG } 18:36:59 DEBUG --- http response --- 18:36:59 DEBUG } 18:36:59 DEBUG 1734460619, 18:36:59 DEBUG } 18:36:59 DEBUG ] 18:36:59 DEBUG } 18:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:36:59 DEBUG } 18:36:59 DEBUG "129535.03333333335" 18:36:59 DEBUG } 18:36:59 DEBUG } 18:36:59 DEBUG ----- output ----- 18:36:59 DEBUG ] 18:36:59 DEBUG ] 18:36:59 DEBUG { 18:36:59 DEBUG } 18:36:59 DEBUG } 18:36:59 DEBUG "status": "success", 18:36:59 DEBUG ] 18:36:59 DEBUG } 18:36:59 DEBUG "data": { 18:36:59 DEBUG } 18:36:59 DEBUG "resultType": "vector", 18:36:59 DEBUG } 18:36:59 DEBUG "result": [ 18:36:59 DEBUG { 18:36:59 DEBUG "metric": { 18:36:59 DEBUG "instance": "10.142.0.11:9100" 18:36:59 DEBUG }, 18:36:59 DEBUG "value": [ 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG [...] 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG "instance": "10.142.0.51:9100" 18:36:59 DEBUG }, 18:36:59 DEBUG "value": [ 18:36:59 DEBUG 1734460619, 18:36:59 DEBUG "130306.20000000001" 18:36:59 DEBUG ] 18:36:59 DEBUG } 18:36:59 DEBUG ] 18:36:59 DEBUG } 18:36:59 DEBUG } 18:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460619" 18:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460619" 18:36:59 INFO [http_cmd]: http status code OK 18:36:59 DEBUG --- status code --- 18:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:36:59 DEBUG --- http response --- 18:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:36:59 DEBUG ----- output ----- 18:36:59 DEBUG { 18:36:59 DEBUG "status": "success", 18:36:59 DEBUG "data": { 18:36:59 DEBUG "resultType": "vector", 18:36:59 DEBUG "result": [ 18:36:59 DEBUG { 18:36:59 DEBUG "metric": { 18:36:59 DEBUG "instance": "10.142.0.11:9100" 18:36:59 DEBUG }, 18:36:59 DEBUG "value": [ 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG [...] 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG "instance": "10.142.0.51:9100" 18:36:59 DEBUG }, 18:36:59 DEBUG "value": [ 18:36:59 DEBUG 1734460619, 18:36:59 DEBUG "27170.13333333333" 18:36:59 DEBUG ] 18:36:59 DEBUG } 18:36:59 DEBUG ] 18:36:59 DEBUG } 18:36:59 DEBUG } 18:36:59 INFO [http_cmd]: http status code OK 18:36:59 DEBUG --- status code --- 18:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:36:59 DEBUG --- http response --- 18:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:36:59 DEBUG ----- output ----- 18:36:59 DEBUG { 18:36:59 DEBUG "status": "success", 18:36:59 DEBUG "data": { 18:36:59 DEBUG "resultType": "vector", 18:36:59 DEBUG "result": [ 18:36:59 DEBUG { 18:36:59 DEBUG "metric": { 18:36:59 DEBUG "instance": "10.142.0.11:9100" 18:36:59 DEBUG }, 18:36:59 DEBUG "value": [ 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG [...] 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG "instance": "10.142.0.51:9100" 18:36:59 DEBUG }, 18:36:59 DEBUG "value": [ 18:36:59 DEBUG 1734460619, 18:36:59 DEBUG "0.0003333333333333854" 18:36:59 DEBUG ] 18:36:59 DEBUG } 18:36:59 DEBUG ] 18:36:59 DEBUG } 18:36:59 DEBUG } 18:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734460619" 18:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460619" 18:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460619" 18:36:59 INFO [http_cmd]: http status code OK 18:36:59 DEBUG --- status code --- 18:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:36:59 DEBUG --- http response --- 18:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:36:59 DEBUG ----- output ----- 18:36:59 DEBUG { 18:36:59 DEBUG "status": "success", 18:36:59 DEBUG "data": { 18:36:59 DEBUG "resultType": "vector", 18:36:59 DEBUG "result": [ 18:36:59 DEBUG { 18:36:59 DEBUG "metric": { 18:36:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:36:59 DEBUG }, 18:36:59 DEBUG "value": [ 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG [...] 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:36:59 DEBUG }, 18:36:59 DEBUG "value": [ 18:36:59 DEBUG 1734460619, 18:36:59 DEBUG "3038269440" 18:36:59 DEBUG ] 18:36:59 DEBUG } 18:36:59 DEBUG ] 18:36:59 DEBUG } 18:36:59 DEBUG } 18:36:59 INFO [http_cmd]: http status code OK 18:36:59 DEBUG --- status code --- 18:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:36:59 DEBUG --- http response --- 18:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:36:59 DEBUG ----- output ----- 18:36:59 DEBUG { 18:36:59 DEBUG "status": "success", 18:36:59 DEBUG "data": { 18:36:59 DEBUG "resultType": "vector", 18:36:59 DEBUG "result": [ 18:36:59 DEBUG { 18:36:59 DEBUG "metric": { 18:36:59 DEBUG "pod": "ds-cts-0" 18:36:59 DEBUG }, 18:36:59 DEBUG "value": [ 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG [...] 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:36:59 DEBUG }, 18:36:59 DEBUG "value": [ 18:36:59 DEBUG 1734460619, 18:36:59 DEBUG "1003.0944002397996" 18:36:59 DEBUG ] 18:36:59 DEBUG } 18:36:59 DEBUG ] 18:36:59 DEBUG } 18:36:59 DEBUG } 18:36:59 INFO [http_cmd]: http status code OK 18:36:59 DEBUG --- status code --- 18:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:36:59 DEBUG --- http response --- 18:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:36:59 DEBUG ----- output ----- 18:36:59 DEBUG { 18:36:59 DEBUG "status": "success", 18:36:59 DEBUG "data": { 18:36:59 DEBUG "resultType": "vector", 18:36:59 DEBUG "result": [ 18:36:59 DEBUG { 18:36:59 DEBUG "metric": { 18:36:59 DEBUG "instance": "10.142.0.11:9100" 18:36:59 DEBUG }, 18:36:59 DEBUG "value": [ 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG [...] 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG 18:36:59 DEBUG "instance": "10.142.0.51:9100" 18:36:59 DEBUG }, 18:36:59 DEBUG "value": [ 18:36:59 DEBUG 1734460619, 18:36:59 DEBUG "0.000599999999999549" 18:36:59 DEBUG ] 18:36:59 DEBUG } 18:36:59 DEBUG ] 18:36:59 DEBUG } 18:36:59 DEBUG } 18:37:01 INFO 18:37:01 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:37:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:37:01 INFO [loop_until]: OK (rc = 0) 18:37:01 DEBUG --- stdout --- 18:37:01 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 5m 543Mi ds-idrepo-0 6m 715Mi rcs-5d696ff5b-7ttvr 2m 298Mi rcs-5d696ff5b-wwlbr 2m 339Mi 18:37:01 DEBUG --- stderr --- 18:37:01 DEBUG 18:37:01 INFO 18:37:01 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:37:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:37:01 INFO [loop_until]: OK (rc = 0) 18:37:01 DEBUG --- stdout --- 18:37:01 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 115m 0% 2283Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1125m 3% 3816Mi 13% 18:37:01 DEBUG --- stderr --- 18:37:01 DEBUG 18:37:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460678" 18:37:58 INFO [http_cmd]: http status code OK 18:37:58 DEBUG --- status code --- 18:37:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:37:58 DEBUG --- http response --- 18:37:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460679" 18:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734460679" 18:37:59 INFO [http_cmd]: http status code OK 18:37:59 DEBUG --- status code --- 18:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:37:59 DEBUG --- http response --- 18:37:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:37:59 INFO [http_cmd]: http status code OK 18:37:59 DEBUG --- status code --- 18:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:37:59 DEBUG --- http response --- 18:37:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460679" 18:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460679" 18:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460679" 18:37:59 INFO [http_cmd]: http status code OK 18:37:59 DEBUG --- status code --- 18:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:37:59 DEBUG --- http response --- 18:37:59 INFO [http_cmd]: http status code OK 18:37:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:37:59 DEBUG --- status code --- 18:37:59 INFO [http_cmd]: http status code OK 18:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:37:59 DEBUG --- http response --- 18:37:59 DEBUG --- status code --- 18:37:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:37:59 DEBUG --- http response --- 18:37:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460679" 18:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460679" 18:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460679" 18:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460679" 18:37:59 INFO [http_cmd]: http status code OK 18:37:59 INFO [http_cmd]: http status code OK 18:37:59 INFO [http_cmd]: http status code OK 18:37:59 DEBUG --- status code --- 18:37:59 DEBUG --- status code --- 18:37:59 DEBUG --- status code --- 18:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:37:59 DEBUG --- http response --- 18:37:59 DEBUG --- http response --- 18:37:59 DEBUG --- http response --- 18:37:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:37:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:37:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:37:59 INFO [http_cmd]: http status code OK 18:37:59 DEBUG --- status code --- 18:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:37:59 DEBUG --- http response --- 18:37:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460679" 18:37:59 INFO [http_cmd]: http status code OK 18:37:59 DEBUG --- status code --- 18:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:37:59 DEBUG --- http response --- 18:37:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460679" 18:37:59 INFO [http_cmd]: http status code OK 18:37:59 DEBUG --- status code --- 18:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:37:59 DEBUG --- http response --- 18:37:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734460679" 18:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460679" 18:37:59 ERROR [http_cmd]: ERROR 18:37:59 ERROR --- status code --- 18:37:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:37:59 ERROR --- http response --- 18:37:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:37:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:37:59 INFO [http_cmd]: http status code OK 18:37:59 DEBUG --- status code --- 18:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:37:59 DEBUG --- http response --- 18:37:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460679" 18:37:59 INFO [http_cmd]: http status code OK 18:37:59 DEBUG --- status code --- 18:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:37:59 DEBUG --- http response --- 18:37:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460679" 18:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460679" 18:37:59 INFO [http_cmd]: http status code OK 18:37:59 DEBUG --- status code --- 18:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:37:59 DEBUG --- http response --- 18:37:59 INFO [http_cmd]: http status code OK 18:37:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:37:59 DEBUG --- status code --- 18:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:37:59 DEBUG --- http response --- 18:37:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460679" 18:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460679" 18:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460679" 18:37:59 INFO [http_cmd]: http status code OK 18:37:59 DEBUG --- status code --- 18:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:37:59 DEBUG --- http response --- 18:37:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:37:59 INFO [http_cmd]: http status code OK 18:37:59 DEBUG --- status code --- 18:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:37:59 DEBUG --- http response --- 18:37:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734460679" 18:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460679" 18:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460679" 18:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460679" 18:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734460679" 18:37:59 INFO [http_cmd]: http status code OK 18:37:59 DEBUG --- status code --- 18:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:37:59 DEBUG --- http response --- 18:37:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734460679" 18:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460679" 18:37:59 INFO [http_cmd]: http status code OK 18:37:59 DEBUG --- status code --- 18:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:37:59 DEBUG --- http response --- 18:37:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:37:59 INFO [http_cmd]: http status code OK 18:37:59 DEBUG --- status code --- 18:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:37:59 DEBUG --- http response --- 18:37:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:37:59 INFO [http_cmd]: http status code OK 18:37:59 INFO [http_cmd]: http status code OK 18:37:59 DEBUG --- status code --- 18:37:59 INFO [http_cmd]: http status code OK 18:37:59 INFO [http_cmd]: http status code OK 18:37:59 DEBUG --- status code --- 18:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:37:59 DEBUG --- status code --- 18:37:59 DEBUG --- status code --- 18:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:37:59 DEBUG --- http response --- 18:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:37:59 DEBUG --- http response --- 18:37:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:37:59 DEBUG --- http response --- 18:37:59 DEBUG --- http response --- 18:37:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:37:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:37:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:37:59 INFO [http_cmd]: http status code OK 18:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460679" 18:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460679" 18:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460679" 18:37:59 DEBUG --- status code --- 18:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734460679" 18:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460679" 18:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460679" 18:37:59 DEBUG --- http response --- 18:37:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:37:59 INFO [http_cmd]: http status code OK 18:37:59 DEBUG --- status code --- 18:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:37:59 DEBUG --- http response --- 18:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:37:59 DEBUG ----- output ----- 18:37:59 DEBUG { 18:37:59 INFO [http_cmd]: http status code OK 18:37:59 DEBUG "status": "success", 18:37:59 INFO [http_cmd]: http status code OK 18:37:59 DEBUG --- status code --- 18:37:59 DEBUG "data": { 18:37:59 DEBUG --- status code --- 18:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:37:59 DEBUG "resultType": "vector", 18:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:37:59 DEBUG --- http response --- 18:37:59 DEBUG "result": [ 18:37:59 DEBUG --- http response --- 18:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:37:59 DEBUG { 18:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:37:59 DEBUG ----- output ----- 18:37:59 DEBUG "metric": { 18:37:59 DEBUG ----- output ----- 18:37:59 DEBUG { 18:37:59 DEBUG "pod": "ds-cts-0" 18:37:59 DEBUG { 18:37:59 DEBUG "status": "success", 18:37:59 DEBUG }, 18:37:59 DEBUG "status": "success", 18:37:59 DEBUG "data": { 18:37:59 DEBUG "value": [ 18:37:59 DEBUG "data": { 18:37:59 DEBUG "resultType": "vector", 18:37:59 DEBUG 18:37:59 DEBUG "resultType": "vector", 18:37:59 DEBUG "result": [ 18:37:59 DEBUG 18:37:59 DEBUG "result": [ 18:37:59 DEBUG { 18:37:59 DEBUG 18:37:59 DEBUG { 18:37:59 DEBUG "metric": { 18:37:59 DEBUG [...] 18:37:59 DEBUG "metric": { 18:37:59 DEBUG "pod": "ds-cts-0" 18:37:59 DEBUG 18:37:59 DEBUG "pod": "ds-cts-0" 18:37:59 DEBUG }, 18:37:59 DEBUG 18:37:59 DEBUG }, 18:37:59 DEBUG "value": [ 18:37:59 DEBUG 18:37:59 DEBUG "value": [ 18:37:59 DEBUG 18:37:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:37:59 DEBUG 18:37:59 DEBUG 18:37:59 DEBUG }, 18:37:59 DEBUG 18:37:59 DEBUG 18:37:59 DEBUG "value": [ 18:37:59 DEBUG 18:37:59 DEBUG [...] 18:37:59 INFO [http_cmd]: http status code OK 18:37:59 INFO [http_cmd]: http status code OK 18:37:59 DEBUG 1734460679, 18:37:59 DEBUG [...] 18:37:59 DEBUG 18:37:59 DEBUG 18:37:59 DEBUG --- status code --- 18:37:59 DEBUG "0" 18:37:59 DEBUG 18:37:59 DEBUG --- status code --- 18:37:59 DEBUG 18:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:37:59 DEBUG ] 18:37:59 DEBUG 18:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:37:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:37:59 DEBUG --- http response --- 18:37:59 DEBUG } 18:37:59 DEBUG 18:37:59 DEBUG --- http response --- 18:37:59 DEBUG }, 18:37:59 INFO [http_cmd]: http status code OK 18:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:37:59 DEBUG ] 18:37:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:37:59 DEBUG "value": [ 18:37:59 DEBUG --- status code --- 18:37:59 DEBUG ----- output ----- 18:37:59 DEBUG } 18:37:59 DEBUG }, 18:37:59 DEBUG ----- output ----- 18:37:59 DEBUG 1734460679, 18:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:37:59 DEBUG { 18:37:59 DEBUG } 18:37:59 DEBUG "value": [ 18:37:59 DEBUG { 18:37:59 DEBUG "712273920" 18:37:59 DEBUG --- http response --- 18:37:59 DEBUG "status": "success", 18:37:59 DEBUG 1734460679, 18:37:59 DEBUG "status": "success", 18:37:59 DEBUG ] 18:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:37:59 DEBUG "data": { 18:37:59 DEBUG "0.002927816239414612" 18:37:59 DEBUG "data": { 18:37:59 DEBUG } 18:37:59 DEBUG ----- output ----- 18:37:59 DEBUG "resultType": "vector", 18:37:59 DEBUG ] 18:37:59 DEBUG "resultType": "vector", 18:37:59 DEBUG ] 18:37:59 DEBUG { 18:37:59 DEBUG "result": [ 18:37:59 DEBUG } 18:37:59 DEBUG "result": [ 18:37:59 DEBUG } 18:37:59 DEBUG "status": "success", 18:37:59 DEBUG { 18:37:59 DEBUG ] 18:37:59 DEBUG { 18:37:59 DEBUG } 18:37:59 DEBUG "data": { 18:37:59 DEBUG "metric": { 18:37:59 DEBUG } 18:37:59 DEBUG "metric": { 18:37:59 DEBUG "resultType": "vector", 18:37:59 DEBUG "instance": "10.142.0.11:9100" 18:37:59 DEBUG } 18:37:59 DEBUG "instance": "10.142.0.11:9100" 18:37:59 DEBUG "result": [ 18:37:59 DEBUG }, 18:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460679" 18:37:59 DEBUG }, 18:37:59 DEBUG { 18:37:59 DEBUG "value": [ 18:37:59 DEBUG "value": [ 18:37:59 DEBUG "metric": { 18:37:59 DEBUG 18:37:59 DEBUG 18:37:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:37:59 DEBUG 18:37:59 DEBUG 18:37:59 DEBUG }, 18:37:59 DEBUG 18:37:59 DEBUG 18:37:59 DEBUG "value": [ 18:37:59 DEBUG [...] 18:37:59 DEBUG [...] 18:37:59 DEBUG 18:37:59 DEBUG 18:37:59 DEBUG 18:37:59 DEBUG 18:37:59 DEBUG 18:37:59 DEBUG 18:37:59 DEBUG 18:37:59 DEBUG 18:37:59 DEBUG 18:37:59 DEBUG [...] 18:37:59 DEBUG "instance": "10.142.0.51:9100" 18:37:59 DEBUG "instance": "10.142.0.51:9100" 18:37:59 DEBUG 18:37:59 DEBUG }, 18:37:59 DEBUG }, 18:37:59 DEBUG 18:37:59 DEBUG "value": [ 18:37:59 DEBUG "value": [ 18:37:59 DEBUG 18:37:59 DEBUG 1734460679, 18:37:59 DEBUG 1734460679, 18:37:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:37:59 DEBUG "126216.99999999999" 18:37:59 DEBUG "127859.16666666664" 18:37:59 DEBUG }, 18:37:59 DEBUG ] 18:37:59 DEBUG ] 18:37:59 DEBUG "value": [ 18:37:59 DEBUG } 18:37:59 DEBUG } 18:37:59 DEBUG 1734460679, 18:37:59 DEBUG ] 18:37:59 DEBUG ] 18:37:59 DEBUG "1.0616417716998248" 18:37:59 DEBUG } 18:37:59 DEBUG } 18:37:59 DEBUG ] 18:37:59 DEBUG } 18:37:59 DEBUG } 18:37:59 DEBUG } 18:37:59 DEBUG ] 18:37:59 DEBUG } 18:37:59 DEBUG } 18:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460679" 18:37:59 INFO [http_cmd]: http status code OK 18:37:59 DEBUG --- status code --- 18:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:37:59 DEBUG --- http response --- 18:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:37:59 DEBUG ----- output ----- 18:37:59 DEBUG { 18:37:59 DEBUG "status": "success", 18:37:59 DEBUG "data": { 18:37:59 DEBUG "resultType": "vector", 18:37:59 DEBUG "result": [ 18:37:59 DEBUG { 18:37:59 DEBUG "metric": { 18:37:59 DEBUG "instance": "10.142.0.11:9100" 18:37:59 DEBUG }, 18:37:59 DEBUG "value": [ 18:37:59 DEBUG 18:37:59 DEBUG 18:37:59 DEBUG 18:37:59 DEBUG [...] 18:37:59 DEBUG 18:37:59 DEBUG 18:37:59 DEBUG 18:37:59 DEBUG "instance": "10.142.0.51:9100" 18:37:59 DEBUG }, 18:37:59 DEBUG "value": [ 18:37:59 DEBUG 1734460679, 18:37:59 DEBUG "25122.13333333333" 18:37:59 DEBUG ] 18:37:59 DEBUG } 18:37:59 DEBUG ] 18:37:59 DEBUG } 18:37:59 DEBUG } 18:37:59 INFO [http_cmd]: http status code OK 18:37:59 DEBUG --- status code --- 18:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:37:59 DEBUG --- http response --- 18:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:37:59 DEBUG ----- output ----- 18:37:59 DEBUG { 18:37:59 DEBUG "status": "success", 18:37:59 DEBUG "data": { 18:37:59 DEBUG "resultType": "vector", 18:37:59 DEBUG "result": [ 18:37:59 DEBUG { 18:37:59 DEBUG "metric": { 18:37:59 DEBUG "instance": "10.142.0.11:9100" 18:37:59 DEBUG }, 18:37:59 DEBUG "value": [ 18:37:59 DEBUG 18:37:59 DEBUG 18:37:59 DEBUG 18:37:59 DEBUG [...] 18:37:59 DEBUG 18:37:59 DEBUG 18:37:59 DEBUG 18:37:59 DEBUG "instance": "10.142.0.51:9100" 18:37:59 DEBUG }, 18:37:59 DEBUG "value": [ 18:37:59 DEBUG 1734460679, 18:37:59 DEBUG "0" 18:37:59 DEBUG ] 18:37:59 DEBUG } 18:37:59 DEBUG ] 18:37:59 DEBUG } 18:37:59 DEBUG } 18:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734460679" 18:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460679" 18:37:59 INFO [http_cmd]: http status code OK 18:37:59 DEBUG --- status code --- 18:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:37:59 DEBUG --- http response --- 18:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460679" 18:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:37:59 DEBUG ----- output ----- 18:37:59 DEBUG { 18:37:59 DEBUG "status": "success", 18:37:59 DEBUG "data": { 18:37:59 DEBUG "resultType": "vector", 18:37:59 DEBUG "result": [ 18:37:59 DEBUG { 18:37:59 DEBUG "metric": { 18:37:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:37:59 DEBUG }, 18:37:59 DEBUG "value": [ 18:37:59 DEBUG 18:37:59 DEBUG 18:37:59 DEBUG 18:37:59 DEBUG [...] 18:37:59 DEBUG 18:37:59 DEBUG 18:37:59 DEBUG 18:37:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:37:59 DEBUG }, 18:37:59 DEBUG "value": [ 18:37:59 DEBUG 1734460679, 18:37:59 DEBUG "3046203392" 18:37:59 DEBUG ] 18:37:59 DEBUG } 18:37:59 DEBUG ] 18:37:59 DEBUG } 18:37:59 DEBUG } 18:37:59 INFO [http_cmd]: http status code OK 18:37:59 DEBUG --- status code --- 18:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:37:59 DEBUG --- http response --- 18:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:37:59 DEBUG ----- output ----- 18:37:59 DEBUG { 18:37:59 DEBUG "status": "success", 18:37:59 DEBUG "data": { 18:37:59 DEBUG "resultType": "vector", 18:37:59 DEBUG "result": [ 18:37:59 DEBUG { 18:37:59 DEBUG "metric": { 18:37:59 DEBUG "pod": "ds-cts-0" 18:37:59 DEBUG }, 18:37:59 DEBUG "value": [ 18:37:59 DEBUG 18:37:59 DEBUG 18:37:59 DEBUG 18:37:59 DEBUG [...] 18:37:59 DEBUG 18:37:59 DEBUG 18:37:59 DEBUG 18:37:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:37:59 DEBUG }, 18:37:59 DEBUG "value": [ 18:37:59 DEBUG 1734460679, 18:37:59 DEBUG "1594.6175702291894" 18:37:59 DEBUG ] 18:37:59 DEBUG } 18:37:59 DEBUG ] 18:37:59 DEBUG } 18:37:59 DEBUG } 18:37:59 INFO [http_cmd]: http status code OK 18:37:59 DEBUG --- status code --- 18:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:37:59 DEBUG --- http response --- 18:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:37:59 DEBUG ----- output ----- 18:37:59 DEBUG { 18:37:59 DEBUG "status": "success", 18:37:59 DEBUG "data": { 18:37:59 DEBUG "resultType": "vector", 18:37:59 DEBUG "result": [ 18:37:59 DEBUG { 18:37:59 DEBUG "metric": { 18:37:59 DEBUG "instance": "10.142.0.11:9100" 18:37:59 DEBUG }, 18:37:59 DEBUG "value": [ 18:37:59 DEBUG 18:37:59 DEBUG 18:37:59 DEBUG 18:37:59 DEBUG [...] 18:37:59 DEBUG 18:37:59 DEBUG 18:37:59 DEBUG 18:37:59 DEBUG "instance": "10.142.0.51:9100" 18:37:59 DEBUG }, 18:37:59 DEBUG "value": [ 18:37:59 DEBUG 1734460679, 18:37:59 DEBUG "0.0003666666666669963" 18:37:59 DEBUG ] 18:37:59 DEBUG } 18:37:59 DEBUG ] 18:37:59 DEBUG } 18:37:59 DEBUG } 18:38:01 INFO 18:38:01 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:38:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:38:01 INFO [loop_until]: OK (rc = 0) 18:38:01 DEBUG --- stdout --- 18:38:01 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 542Mi ds-idrepo-0 7m 718Mi rcs-5d696ff5b-7ttvr 4m 298Mi rcs-5d696ff5b-wwlbr 3m 339Mi 18:38:01 DEBUG --- stderr --- 18:38:01 DEBUG 18:38:01 INFO 18:38:01 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:38:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:38:01 INFO [loop_until]: OK (rc = 0) 18:38:01 DEBUG --- stdout --- 18:38:01 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 114m 0% 2287Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1135m 3% 3818Mi 13% 18:38:01 DEBUG --- stderr --- 18:38:01 DEBUG 18:38:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460738" 18:38:59 INFO [http_cmd]: http status code OK 18:38:59 DEBUG --- status code --- 18:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:38:59 DEBUG --- http response --- 18:38:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460739" 18:38:59 INFO [http_cmd]: http status code OK 18:38:59 DEBUG --- status code --- 18:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:38:59 DEBUG --- http response --- 18:38:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734460739" 18:38:59 INFO [http_cmd]: http status code OK 18:38:59 DEBUG --- status code --- 18:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:38:59 DEBUG --- http response --- 18:38:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460739" 18:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460739" 18:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460739" 18:38:59 INFO [http_cmd]: http status code OK 18:38:59 INFO [http_cmd]: http status code OK 18:38:59 DEBUG --- status code --- 18:38:59 DEBUG --- status code --- 18:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:38:59 DEBUG --- http response --- 18:38:59 DEBUG --- http response --- 18:38:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:38:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:38:59 INFO [http_cmd]: http status code OK 18:38:59 DEBUG --- status code --- 18:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:38:59 DEBUG --- http response --- 18:38:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460739" 18:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460739" 18:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460739" 18:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460739" 18:38:59 INFO [http_cmd]: http status code OK 18:38:59 DEBUG --- status code --- 18:38:59 INFO [http_cmd]: http status code OK 18:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:38:59 DEBUG --- status code --- 18:38:59 DEBUG --- http response --- 18:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:38:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:38:59 DEBUG --- http response --- 18:38:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:38:59 INFO [http_cmd]: http status code OK 18:38:59 DEBUG --- status code --- 18:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:38:59 DEBUG --- http response --- 18:38:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460739" 18:38:59 INFO [http_cmd]: http status code OK 18:38:59 DEBUG --- status code --- 18:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:38:59 DEBUG --- http response --- 18:38:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:38:59 INFO [http_cmd]: http status code OK 18:38:59 DEBUG --- status code --- 18:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:38:59 DEBUG --- http response --- 18:38:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460739" 18:38:59 INFO [http_cmd]: http status code OK 18:38:59 DEBUG --- status code --- 18:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:38:59 DEBUG --- http response --- 18:38:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734460739" 18:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460739" 18:38:59 ERROR [http_cmd]: ERROR 18:38:59 ERROR --- status code --- 18:38:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:38:59 ERROR --- http response --- 18:38:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:38:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:38:59 INFO [http_cmd]: http status code OK 18:38:59 DEBUG --- status code --- 18:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:38:59 DEBUG --- http response --- 18:38:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460739" 18:38:59 INFO [http_cmd]: http status code OK 18:38:59 DEBUG --- status code --- 18:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:38:59 DEBUG --- http response --- 18:38:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460739" 18:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460739" 18:38:59 INFO [http_cmd]: http status code OK 18:38:59 DEBUG --- status code --- 18:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:38:59 DEBUG --- http response --- 18:38:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:38:59 INFO [http_cmd]: http status code OK 18:38:59 DEBUG --- status code --- 18:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:38:59 DEBUG --- http response --- 18:38:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460739" 18:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460739" 18:38:59 INFO [http_cmd]: http status code OK 18:38:59 DEBUG --- status code --- 18:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:38:59 DEBUG --- http response --- 18:38:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:38:59 INFO [http_cmd]: http status code OK 18:38:59 DEBUG --- status code --- 18:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:38:59 DEBUG --- http response --- 18:38:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460739" 18:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460739" 18:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734460739" 18:38:59 INFO [http_cmd]: http status code OK 18:38:59 DEBUG --- status code --- 18:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:38:59 INFO [http_cmd]: http status code OK 18:38:59 DEBUG --- http response --- 18:38:59 DEBUG --- status code --- 18:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:38:59 DEBUG --- http response --- 18:38:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:38:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734460739" 18:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734460739" 18:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460739" 18:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460739" 18:38:59 INFO [http_cmd]: http status code OK 18:38:59 DEBUG --- status code --- 18:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:38:59 DEBUG --- http response --- 18:38:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460739" 18:38:59 INFO [http_cmd]: http status code OK 18:38:59 DEBUG --- status code --- 18:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:38:59 DEBUG --- http response --- 18:38:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:38:59 INFO [http_cmd]: http status code OK 18:38:59 DEBUG --- status code --- 18:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:38:59 DEBUG --- http response --- 18:38:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:38:59 INFO [http_cmd]: http status code OK 18:38:59 DEBUG --- status code --- 18:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:38:59 DEBUG --- http response --- 18:38:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:38:59 INFO [http_cmd]: http status code OK 18:38:59 DEBUG --- status code --- 18:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:38:59 DEBUG --- http response --- 18:38:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:38:59 INFO [http_cmd]: http status code OK 18:38:59 DEBUG --- status code --- 18:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:38:59 DEBUG --- http response --- 18:38:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460739" 18:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460739" 18:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460739" 18:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460739" 18:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734460739" 18:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460739" 18:38:59 INFO [http_cmd]: http status code OK 18:38:59 DEBUG --- status code --- 18:38:59 INFO [http_cmd]: http status code OK 18:38:59 INFO [http_cmd]: http status code OK 18:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:38:59 DEBUG --- status code --- 18:38:59 DEBUG --- status code --- 18:38:59 DEBUG --- http response --- 18:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:38:59 DEBUG --- http response --- 18:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:38:59 DEBUG --- http response --- 18:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460739" 18:38:59 DEBUG ----- output ----- 18:38:59 DEBUG ----- output ----- 18:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:38:59 DEBUG { 18:38:59 DEBUG { 18:38:59 DEBUG ----- output ----- 18:38:59 DEBUG "status": "success", 18:38:59 DEBUG "data": { 18:38:59 DEBUG { 18:38:59 DEBUG "status": "success", 18:38:59 DEBUG "resultType": "vector", 18:38:59 DEBUG "result": [ 18:38:59 DEBUG "status": "success", 18:38:59 DEBUG "data": { 18:38:59 DEBUG { 18:38:59 DEBUG "data": { 18:38:59 DEBUG "resultType": "vector", 18:38:59 DEBUG "metric": { 18:38:59 DEBUG "resultType": "vector", 18:38:59 DEBUG "result": [ 18:38:59 INFO [http_cmd]: http status code OK 18:38:59 DEBUG --- status code --- 18:38:59 DEBUG "pod": "ds-cts-0" 18:38:59 DEBUG "result": [ 18:38:59 DEBUG { 18:38:59 INFO [http_cmd]: http status code OK 18:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:38:59 DEBUG }, 18:38:59 DEBUG { 18:38:59 DEBUG "metric": { 18:38:59 DEBUG "pod": "ds-cts-0" 18:38:59 DEBUG }, 18:38:59 DEBUG "value": [ 18:38:59 DEBUG "value": [ 18:38:59 DEBUG --- status code --- 18:38:59 DEBUG --- http response --- 18:38:59 DEBUG "metric": { 18:38:59 DEBUG 18:38:59 DEBUG 18:38:59 DEBUG 18:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:38:59 DEBUG "pod": "ds-cts-0" 18:38:59 DEBUG 18:38:59 DEBUG [...] 18:38:59 DEBUG 18:38:59 DEBUG --- http response --- 18:38:59 DEBUG }, 18:38:59 DEBUG 18:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:38:59 DEBUG 18:38:59 DEBUG "value": [ 18:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:38:59 DEBUG 18:38:59 DEBUG ----- output ----- 18:38:59 DEBUG 18:38:59 DEBUG 18:38:59 DEBUG 18:38:59 DEBUG 18:38:59 DEBUG [...] 18:38:59 DEBUG 18:38:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:38:59 DEBUG }, 18:38:59 DEBUG { 18:38:59 DEBUG "status": "success", 18:38:59 DEBUG ----- output ----- 18:38:59 DEBUG [...] 18:38:59 DEBUG "value": [ 18:38:59 DEBUG 18:38:59 DEBUG "data": { 18:38:59 DEBUG { 18:38:59 DEBUG 18:38:59 DEBUG 1734460739, 18:38:59 DEBUG 18:38:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460739" 18:38:59 DEBUG "status": "success", 18:38:59 INFO [http_cmd]: http status code OK 18:38:59 DEBUG 18:38:59 DEBUG "0" 18:38:59 DEBUG "resultType": "vector", 18:38:59 DEBUG }, 18:38:59 DEBUG "data": { 18:38:59 DEBUG --- status code --- 18:38:59 DEBUG 18:38:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:38:59 DEBUG }, 18:38:59 DEBUG "value": [ 18:38:59 DEBUG 1734460739, 18:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:38:59 DEBUG ] 18:38:59 DEBUG "result": [ 18:38:59 DEBUG "value": [ 18:38:59 DEBUG "resultType": "vector", 18:38:59 DEBUG "712323072" 18:38:59 DEBUG --- http response --- 18:38:59 DEBUG } 18:38:59 DEBUG { 18:38:59 DEBUG 1734460739, 18:38:59 DEBUG "0.0030376939009435357" 18:38:59 DEBUG ] 18:38:59 DEBUG } 18:38:59 DEBUG ] 18:38:59 DEBUG } 18:38:59 DEBUG } 18:38:59 INFO [http_cmd]: http status code OK 18:38:59 DEBUG "result": [ 18:38:59 DEBUG ] 18:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:38:59 DEBUG ] 18:38:59 DEBUG "metric": { 18:38:59 DEBUG --- status code --- 18:38:59 DEBUG { 18:38:59 DEBUG } 18:38:59 DEBUG ----- output ----- 18:38:59 DEBUG } 18:38:59 DEBUG "instance": "10.142.0.11:9100" 18:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:38:59 DEBUG "metric": { 18:38:59 DEBUG ] 18:38:59 DEBUG { 18:38:59 DEBUG } 18:38:59 DEBUG }, 18:38:59 DEBUG --- http response --- 18:38:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:38:59 DEBUG } 18:38:59 DEBUG "status": "success", 18:38:59 DEBUG "value": [ 18:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:38:59 DEBUG }, 18:38:59 DEBUG } 18:38:59 DEBUG "data": { 18:38:59 DEBUG 18:38:59 DEBUG ----- output ----- 18:38:59 DEBUG "value": [ 18:38:59 DEBUG "resultType": "vector", 18:38:59 DEBUG 18:38:59 DEBUG { 18:38:59 DEBUG 18:38:59 DEBUG "result": [ 18:38:59 DEBUG { 18:38:59 DEBUG "metric": { 18:38:59 DEBUG "instance": "10.142.0.11:9100" 18:38:59 DEBUG 18:38:59 DEBUG "status": "success", 18:38:59 DEBUG }, 18:38:59 DEBUG "value": [ 18:38:59 DEBUG 18:38:59 DEBUG 18:38:59 DEBUG "data": { 18:38:59 DEBUG 18:38:59 DEBUG 18:38:59 DEBUG [...] 18:38:59 DEBUG [...] 18:38:59 DEBUG 18:38:59 DEBUG 18:38:59 DEBUG "resultType": "vector", 18:38:59 DEBUG 18:38:59 DEBUG [...] 18:38:59 DEBUG 18:38:59 DEBUG 18:38:59 DEBUG "instance": "10.142.0.51:9100" 18:38:59 DEBUG 18:38:59 DEBUG "result": [ 18:38:59 DEBUG 18:38:59 DEBUG }, 18:38:59 DEBUG "value": [ 18:38:59 DEBUG 1734460739, 18:38:59 DEBUG { 18:38:59 DEBUG 18:38:59 DEBUG "128893.46666666665" 18:38:59 DEBUG 18:38:59 DEBUG "metric": { 18:38:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:38:59 INFO [http_cmd]: http status code OK 18:38:59 DEBUG ] 18:38:59 DEBUG } 18:38:59 DEBUG ] 18:38:59 DEBUG "instance": "10.142.0.11:9100" 18:38:59 DEBUG --- status code --- 18:38:59 DEBUG 18:38:59 DEBUG } 18:38:59 DEBUG }, 18:38:59 DEBUG }, 18:38:59 DEBUG "value": [ 18:38:59 DEBUG 18:38:59 DEBUG } 18:38:59 DEBUG "value": [ 18:38:59 DEBUG 1734460739, 18:38:59 DEBUG "1.0532950566962151" 18:38:59 DEBUG ] 18:38:59 DEBUG } 18:38:59 DEBUG ] 18:38:59 DEBUG } 18:38:59 DEBUG } 18:38:59 DEBUG "instance": "10.142.0.51:9100" 18:38:59 DEBUG 18:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:38:59 DEBUG }, 18:38:59 DEBUG 18:38:59 DEBUG --- http response --- 18:38:59 DEBUG "value": [ 18:38:59 DEBUG [...] 18:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:38:59 DEBUG 1734460739, 18:38:59 DEBUG 18:38:59 DEBUG ----- output ----- 18:38:59 DEBUG "126705.96666666667" 18:38:59 DEBUG 18:38:59 DEBUG { 18:38:59 DEBUG ] 18:38:59 DEBUG 18:38:59 DEBUG "instance": "10.142.0.51:9100" 18:38:59 DEBUG } 18:38:59 DEBUG "status": "success", 18:38:59 DEBUG }, 18:38:59 DEBUG ] 18:38:59 DEBUG "data": { 18:38:59 DEBUG "value": [ 18:38:59 DEBUG } 18:38:59 DEBUG "resultType": "vector", 18:38:59 DEBUG 1734460739, 18:38:59 DEBUG } 18:38:59 DEBUG "result": [ 18:38:59 DEBUG "25941.333333333332" 18:38:59 DEBUG ] 18:38:59 DEBUG { 18:38:59 DEBUG } 18:38:59 DEBUG ] 18:38:59 DEBUG "metric": { 18:38:59 DEBUG } 18:38:59 DEBUG "instance": "10.142.0.11:9100" 18:38:59 DEBUG } 18:38:59 DEBUG }, 18:38:59 DEBUG "value": [ 18:38:59 DEBUG 18:38:59 DEBUG 18:38:59 DEBUG 18:38:59 DEBUG [...] 18:38:59 DEBUG 18:38:59 DEBUG 18:38:59 DEBUG 18:38:59 DEBUG "instance": "10.142.0.51:9100" 18:38:59 DEBUG }, 18:38:59 DEBUG "value": [ 18:38:59 DEBUG 1734460739, 18:38:59 DEBUG "0" 18:38:59 DEBUG ] 18:38:59 DEBUG } 18:38:59 DEBUG ] 18:38:59 DEBUG } 18:38:59 DEBUG } 18:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734460739" 18:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460739" 18:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460739" 18:38:59 INFO [http_cmd]: http status code OK 18:38:59 DEBUG --- status code --- 18:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:38:59 DEBUG --- http response --- 18:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:38:59 DEBUG ----- output ----- 18:38:59 DEBUG { 18:38:59 DEBUG "status": "success", 18:38:59 DEBUG "data": { 18:38:59 DEBUG "resultType": "vector", 18:38:59 DEBUG "result": [ 18:38:59 DEBUG { 18:38:59 DEBUG "metric": { 18:38:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:38:59 DEBUG }, 18:38:59 DEBUG "value": [ 18:38:59 DEBUG 18:38:59 DEBUG 18:38:59 DEBUG 18:38:59 DEBUG [...] 18:38:59 DEBUG 18:38:59 DEBUG 18:38:59 DEBUG 18:38:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:38:59 DEBUG }, 18:38:59 DEBUG "value": [ 18:38:59 DEBUG 1734460739, 18:38:59 DEBUG "3054264320" 18:38:59 DEBUG ] 18:38:59 DEBUG } 18:38:59 DEBUG ] 18:38:59 DEBUG } 18:38:59 DEBUG } 18:38:59 INFO [http_cmd]: http status code OK 18:38:59 DEBUG --- status code --- 18:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:38:59 DEBUG --- http response --- 18:38:59 INFO [http_cmd]: http status code OK 18:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:38:59 DEBUG --- status code --- 18:38:59 DEBUG ----- output ----- 18:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:38:59 DEBUG { 18:38:59 DEBUG --- http response --- 18:38:59 DEBUG "status": "success", 18:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:38:59 DEBUG "data": { 18:38:59 DEBUG ----- output ----- 18:38:59 DEBUG "resultType": "vector", 18:38:59 DEBUG { 18:38:59 DEBUG "result": [ 18:38:59 DEBUG "status": "success", 18:38:59 DEBUG { 18:38:59 DEBUG "data": { 18:38:59 DEBUG "metric": { 18:38:59 DEBUG "resultType": "vector", 18:38:59 DEBUG "instance": "10.142.0.11:9100" 18:38:59 DEBUG "result": [ 18:38:59 DEBUG }, 18:38:59 DEBUG { 18:38:59 DEBUG "value": [ 18:38:59 DEBUG "metric": { 18:38:59 DEBUG 18:38:59 DEBUG "pod": "ds-cts-0" 18:38:59 DEBUG 18:38:59 DEBUG }, 18:38:59 DEBUG 18:38:59 DEBUG "value": [ 18:38:59 DEBUG [...] 18:38:59 DEBUG 18:38:59 DEBUG 18:38:59 DEBUG 18:38:59 DEBUG 18:38:59 DEBUG 18:38:59 DEBUG 18:38:59 DEBUG [...] 18:38:59 DEBUG "instance": "10.142.0.51:9100" 18:38:59 DEBUG 18:38:59 DEBUG }, 18:38:59 DEBUG 18:38:59 DEBUG "value": [ 18:38:59 DEBUG 18:38:59 DEBUG 1734460739, 18:38:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:38:59 DEBUG "0.0003666666666669963" 18:38:59 DEBUG }, 18:38:59 DEBUG ] 18:38:59 DEBUG "value": [ 18:38:59 DEBUG } 18:38:59 DEBUG 1734460739, 18:38:59 DEBUG ] 18:38:59 DEBUG "1263.90631445301" 18:38:59 DEBUG } 18:38:59 DEBUG ] 18:38:59 DEBUG } 18:38:59 DEBUG } 18:38:59 DEBUG ] 18:38:59 DEBUG } 18:38:59 DEBUG } 18:39:01 INFO 18:39:01 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:39:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:39:01 INFO [loop_until]: OK (rc = 0) 18:39:01 DEBUG --- stdout --- 18:39:01 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 543Mi ds-idrepo-0 6m 718Mi rcs-5d696ff5b-7ttvr 3m 298Mi rcs-5d696ff5b-wwlbr 2m 339Mi 18:39:01 DEBUG --- stderr --- 18:39:01 DEBUG 18:39:01 INFO 18:39:01 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:39:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:39:01 INFO [loop_until]: OK (rc = 0) 18:39:01 DEBUG --- stdout --- 18:39:01 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 116m 0% 2291Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1136m 3% 3818Mi 13% 18:39:01 DEBUG --- stderr --- 18:39:01 DEBUG 18:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460799" 18:39:59 INFO [http_cmd]: http status code OK 18:39:59 DEBUG --- status code --- 18:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:39:59 DEBUG --- http response --- 18:39:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460799" 18:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734460799" 18:39:59 INFO [http_cmd]: http status code OK 18:39:59 DEBUG --- status code --- 18:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:39:59 DEBUG --- http response --- 18:39:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:39:59 INFO [http_cmd]: http status code OK 18:39:59 DEBUG --- status code --- 18:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:39:59 DEBUG --- http response --- 18:39:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460799" 18:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460799" 18:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460799" 18:39:59 INFO [http_cmd]: http status code OK 18:39:59 INFO [http_cmd]: http status code OK 18:39:59 DEBUG --- status code --- 18:39:59 INFO [http_cmd]: http status code OK 18:39:59 DEBUG --- status code --- 18:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:39:59 DEBUG --- status code --- 18:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:39:59 DEBUG --- http response --- 18:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:39:59 DEBUG --- http response --- 18:39:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:39:59 DEBUG --- http response --- 18:39:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:39:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460799" 18:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460799" 18:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460799" 18:39:59 INFO [http_cmd]: http status code OK 18:39:59 DEBUG --- status code --- 18:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:39:59 DEBUG --- http response --- 18:39:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:39:59 INFO [http_cmd]: http status code OK 18:39:59 DEBUG --- status code --- 18:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:39:59 DEBUG --- http response --- 18:39:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:39:59 INFO [http_cmd]: http status code OK 18:39:59 DEBUG --- status code --- 18:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:39:59 DEBUG --- http response --- 18:39:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460799" 18:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460799" 18:39:59 INFO [http_cmd]: http status code OK 18:39:59 INFO [http_cmd]: http status code OK 18:39:59 DEBUG --- status code --- 18:39:59 DEBUG --- status code --- 18:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:39:59 DEBUG --- http response --- 18:39:59 DEBUG --- http response --- 18:39:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:39:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460799" 18:39:59 INFO [http_cmd]: http status code OK 18:39:59 DEBUG --- status code --- 18:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:39:59 DEBUG --- http response --- 18:39:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734460799" 18:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460799" 18:39:59 ERROR [http_cmd]: ERROR 18:39:59 ERROR --- status code --- 18:39:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:39:59 ERROR --- http response --- 18:39:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:39:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:39:59 INFO [http_cmd]: http status code OK 18:39:59 DEBUG --- status code --- 18:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:39:59 DEBUG --- http response --- 18:39:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460799" 18:39:59 INFO [http_cmd]: http status code OK 18:39:59 DEBUG --- status code --- 18:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:39:59 DEBUG --- http response --- 18:39:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460799" 18:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460799" 18:39:59 INFO [http_cmd]: http status code OK 18:39:59 DEBUG --- status code --- 18:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:39:59 DEBUG --- http response --- 18:39:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:39:59 INFO [http_cmd]: http status code OK 18:39:59 DEBUG --- status code --- 18:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:39:59 DEBUG --- http response --- 18:39:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460799" 18:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460799" 18:39:59 INFO [http_cmd]: http status code OK 18:39:59 INFO [http_cmd]: http status code OK 18:39:59 DEBUG --- status code --- 18:39:59 DEBUG --- status code --- 18:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:39:59 DEBUG --- http response --- 18:39:59 DEBUG --- http response --- 18:39:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:39:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460799" 18:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460799" 18:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734460799" 18:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734460799" 18:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734460799" 18:39:59 INFO [http_cmd]: http status code OK 18:39:59 INFO [http_cmd]: http status code OK 18:39:59 DEBUG --- status code --- 18:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460799" 18:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:39:59 DEBUG --- status code --- 18:39:59 DEBUG --- http response --- 18:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:39:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:39:59 DEBUG --- http response --- 18:39:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:39:59 INFO [http_cmd]: http status code OK 18:39:59 DEBUG --- status code --- 18:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:39:59 INFO [http_cmd]: http status code OK 18:39:59 DEBUG --- http response --- 18:39:59 DEBUG --- status code --- 18:39:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:39:59 DEBUG --- http response --- 18:39:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:39:59 INFO [http_cmd]: http status code OK 18:39:59 DEBUG --- status code --- 18:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:39:59 DEBUG --- http response --- 18:39:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460799" 18:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460799" 18:39:59 INFO [http_cmd]: http status code OK 18:39:59 DEBUG --- status code --- 18:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:39:59 DEBUG --- http response --- 18:39:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:39:59 INFO [http_cmd]: http status code OK 18:39:59 DEBUG --- status code --- 18:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:39:59 DEBUG --- http response --- 18:39:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:39:59 INFO [http_cmd]: http status code OK 18:39:59 DEBUG --- status code --- 18:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:39:59 DEBUG --- http response --- 18:39:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460799" 18:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460799" 18:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460799" 18:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460799" 18:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734460799" 18:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460799" 18:39:59 INFO [http_cmd]: http status code OK 18:39:59 DEBUG --- status code --- 18:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:39:59 DEBUG --- http response --- 18:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460799" 18:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460799" 18:39:59 INFO [http_cmd]: http status code OK 18:39:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:39:59 INFO [http_cmd]: http status code OK 18:39:59 DEBUG --- status code --- 18:39:59 DEBUG ----- output ----- 18:39:59 DEBUG --- status code --- 18:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:39:59 DEBUG { 18:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:39:59 DEBUG --- http response --- 18:39:59 DEBUG "status": "success", 18:39:59 DEBUG --- http response --- 18:39:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:39:59 DEBUG "data": { 18:39:59 DEBUG ----- output ----- 18:39:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:39:59 DEBUG "resultType": "vector", 18:39:59 DEBUG { 18:39:59 DEBUG ----- output ----- 18:39:59 DEBUG "result": [ 18:39:59 DEBUG "status": "success", 18:39:59 DEBUG { 18:39:59 DEBUG { 18:39:59 DEBUG "data": { 18:39:59 DEBUG "status": "success", 18:39:59 DEBUG "metric": { 18:39:59 DEBUG "resultType": "vector", 18:39:59 DEBUG "data": { 18:39:59 DEBUG "pod": "ds-cts-0" 18:39:59 DEBUG "result": [ 18:39:59 DEBUG "resultType": "vector", 18:39:59 DEBUG }, 18:39:59 DEBUG { 18:39:59 DEBUG "result": [ 18:39:59 DEBUG "value": [ 18:39:59 INFO [http_cmd]: http status code OK 18:39:59 DEBUG "metric": { 18:39:59 DEBUG { 18:39:59 DEBUG 18:39:59 DEBUG --- status code --- 18:39:59 DEBUG "pod": "ds-cts-0" 18:39:59 DEBUG "metric": { 18:39:59 DEBUG 18:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:39:59 DEBUG }, 18:39:59 DEBUG "pod": "ds-cts-0" 18:39:59 DEBUG 18:39:59 DEBUG --- http response --- 18:39:59 DEBUG "value": [ 18:39:59 DEBUG }, 18:39:59 DEBUG [...] 18:39:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:39:59 DEBUG 18:39:59 DEBUG "value": [ 18:39:59 DEBUG 18:39:59 DEBUG ----- output ----- 18:39:59 DEBUG 18:39:59 DEBUG 18:39:59 DEBUG 18:39:59 DEBUG { 18:39:59 DEBUG 18:39:59 DEBUG 18:39:59 DEBUG 18:39:59 DEBUG "status": "success", 18:39:59 DEBUG [...] 18:39:59 DEBUG 18:39:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:39:59 DEBUG "data": { 18:39:59 DEBUG 18:39:59 DEBUG [...] 18:39:59 INFO [http_cmd]: http status code OK 18:39:59 DEBUG --- status code --- 18:39:59 DEBUG }, 18:39:59 INFO [http_cmd]: http status code OK 18:39:59 DEBUG "resultType": "vector", 18:39:59 DEBUG 18:39:59 DEBUG 18:39:59 DEBUG 18:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:39:59 DEBUG --- http response --- 18:39:59 DEBUG --- status code --- 18:39:59 DEBUG "result": [ 18:39:59 DEBUG 18:39:59 INFO [http_cmd]: http status code OK 18:39:59 DEBUG 18:39:59 DEBUG "value": [ 18:39:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:39:59 DEBUG { 18:39:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:39:59 DEBUG --- status code --- 18:39:59 INFO [http_cmd]: http status code OK 18:39:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:39:59 DEBUG 1734460799, 18:39:59 DEBUG ----- output ----- 18:39:59 DEBUG --- http response --- 18:39:59 DEBUG "metric": { 18:39:59 DEBUG }, 18:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734460799" 18:39:59 DEBUG --- status code --- 18:39:59 DEBUG }, 18:39:59 DEBUG "0" 18:39:59 DEBUG { 18:39:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:39:59 DEBUG "instance": "10.142.0.11:9100" 18:39:59 DEBUG "value": [ 18:39:59 DEBUG --- http response --- 18:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:39:59 DEBUG "value": [ 18:39:59 DEBUG ] 18:39:59 DEBUG "status": "success", 18:39:59 DEBUG ----- output ----- 18:39:59 DEBUG }, 18:39:59 DEBUG "value": [ 18:39:59 DEBUG --- http response --- 18:39:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:39:59 DEBUG 1734460799, 18:39:59 DEBUG } 18:39:59 DEBUG "data": { 18:39:59 DEBUG "resultType": "vector", 18:39:59 DEBUG 1734460799, 18:39:59 DEBUG 18:39:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:39:59 DEBUG ----- output ----- 18:39:59 DEBUG "712364032" 18:39:59 DEBUG ] 18:39:59 DEBUG { 18:39:59 DEBUG "result": [ 18:39:59 DEBUG "0.0031569867943823733" 18:39:59 DEBUG 18:39:59 DEBUG ----- output ----- 18:39:59 DEBUG { 18:39:59 DEBUG ] 18:39:59 DEBUG } 18:39:59 DEBUG "status": "success", 18:39:59 DEBUG { 18:39:59 DEBUG ] 18:39:59 DEBUG 18:39:59 DEBUG [...] 18:39:59 DEBUG "status": "success", 18:39:59 DEBUG } 18:39:59 DEBUG } 18:39:59 DEBUG "data": { 18:39:59 DEBUG "metric": { 18:39:59 DEBUG } 18:39:59 DEBUG ] 18:39:59 DEBUG 18:39:59 DEBUG "data": { 18:39:59 DEBUG ] 18:39:59 DEBUG "resultType": "vector", 18:39:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:39:59 DEBUG }, 18:39:59 DEBUG } 18:39:59 DEBUG 18:39:59 DEBUG "resultType": "vector", 18:39:59 DEBUG } 18:39:59 DEBUG "result": [ 18:39:59 DEBUG { 18:39:59 DEBUG "value": [ 18:39:59 DEBUG } 18:39:59 DEBUG 18:39:59 DEBUG "result": [ 18:39:59 DEBUG } 18:39:59 DEBUG { 18:39:59 DEBUG "metric": { 18:39:59 DEBUG 18:39:59 DEBUG "instance": "10.142.0.51:9100" 18:39:59 DEBUG { 18:39:59 DEBUG "metric": { 18:39:59 DEBUG "instance": "10.142.0.11:9100" 18:39:59 DEBUG 18:39:59 DEBUG }, 18:39:59 DEBUG "value": [ 18:39:59 DEBUG "instance": "10.142.0.11:9100" 18:39:59 DEBUG }, 18:39:59 DEBUG }, 18:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460799" 18:39:59 DEBUG 18:39:59 DEBUG "status": "success", 18:39:59 DEBUG 1734460799, 18:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460799" 18:39:59 DEBUG "value": [ 18:39:59 DEBUG "value": [ 18:39:59 DEBUG [...] 18:39:59 DEBUG "data": { 18:39:59 DEBUG "130241.29999999999" 18:39:59 DEBUG 18:39:59 DEBUG 18:39:59 DEBUG 18:39:59 DEBUG "resultType": "vector", 18:39:59 DEBUG ] 18:39:59 DEBUG 18:39:59 DEBUG 18:39:59 DEBUG 18:39:59 DEBUG "result": [ 18:39:59 DEBUG } 18:39:59 DEBUG 18:39:59 DEBUG 18:39:59 DEBUG 18:39:59 DEBUG { 18:39:59 DEBUG ] 18:39:59 DEBUG [...] 18:39:59 DEBUG [...] 18:39:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:39:59 DEBUG }, 18:39:59 DEBUG } 18:39:59 DEBUG 18:39:59 DEBUG 18:39:59 DEBUG "metric": { 18:39:59 DEBUG "value": [ 18:39:59 DEBUG } 18:39:59 DEBUG 18:39:59 DEBUG 18:39:59 DEBUG "instance": "10.142.0.11:9100" 18:39:59 DEBUG 1734460799, 18:39:59 DEBUG 18:39:59 DEBUG 18:39:59 DEBUG "instance": "10.142.0.51:9100" 18:39:59 DEBUG }, 18:39:59 DEBUG "instance": "10.142.0.51:9100" 18:39:59 DEBUG }, 18:39:59 DEBUG "1.0590128143074162" 18:39:59 DEBUG "value": [ 18:39:59 DEBUG }, 18:39:59 DEBUG "value": [ 18:39:59 DEBUG ] 18:39:59 DEBUG 1734460799, 18:39:59 DEBUG "value": [ 18:39:59 INFO [http_cmd]: http status code OK 18:39:59 DEBUG 18:39:59 DEBUG } 18:39:59 DEBUG "128066.5" 18:39:59 DEBUG 1734460799, 18:39:59 DEBUG --- status code --- 18:39:59 DEBUG 18:39:59 DEBUG ] 18:39:59 DEBUG ] 18:39:59 DEBUG "0.0003333333333333854" 18:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:39:59 DEBUG --- http response --- 18:39:59 DEBUG } 18:39:59 DEBUG } 18:39:59 DEBUG ] 18:39:59 DEBUG } 18:39:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:39:59 DEBUG } 18:39:59 DEBUG ] 18:39:59 DEBUG 18:39:59 DEBUG ] 18:39:59 DEBUG ----- output ----- 18:39:59 DEBUG } 18:39:59 DEBUG } 18:39:59 DEBUG } 18:39:59 DEBUG { 18:39:59 DEBUG [...] 18:39:59 DEBUG } 18:39:59 DEBUG "status": "success", 18:39:59 DEBUG 18:39:59 DEBUG "data": { 18:39:59 DEBUG 18:39:59 DEBUG "resultType": "vector", 18:39:59 DEBUG 18:39:59 DEBUG "result": [ 18:39:59 DEBUG "instance": "10.142.0.51:9100" 18:39:59 DEBUG { 18:39:59 DEBUG }, 18:39:59 DEBUG "metric": { 18:39:59 DEBUG "value": [ 18:39:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:39:59 DEBUG 1734460799, 18:39:59 DEBUG }, 18:39:59 DEBUG "30583.466666666667" 18:39:59 DEBUG "value": [ 18:39:59 DEBUG ] 18:39:59 DEBUG 18:39:59 DEBUG } 18:39:59 DEBUG 18:39:59 INFO [http_cmd]: http status code OK 18:39:59 DEBUG ] 18:39:59 DEBUG 18:39:59 DEBUG --- status code --- 18:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:39:59 DEBUG [...] 18:39:59 DEBUG 18:39:59 DEBUG --- http response --- 18:39:59 DEBUG } 18:39:59 DEBUG 18:39:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:39:59 DEBUG ----- output ----- 18:39:59 DEBUG 18:39:59 DEBUG } 18:39:59 DEBUG { 18:39:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:39:59 INFO [http_cmd]: http status code OK 18:39:59 DEBUG "status": "success", 18:39:59 DEBUG }, 18:39:59 DEBUG --- status code --- 18:39:59 DEBUG "data": { 18:39:59 DEBUG "value": [ 18:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:39:59 DEBUG "resultType": "vector", 18:39:59 DEBUG 1734460799, 18:39:59 DEBUG --- http response --- 18:39:59 DEBUG "result": [ 18:39:59 DEBUG "3053408256" 18:39:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:39:59 DEBUG { 18:39:59 DEBUG ] 18:39:59 DEBUG ----- output ----- 18:39:59 DEBUG "metric": { 18:39:59 DEBUG } 18:39:59 DEBUG { 18:39:59 DEBUG "pod": "ds-cts-0" 18:39:59 DEBUG ] 18:39:59 DEBUG "status": "success", 18:39:59 DEBUG }, 18:39:59 DEBUG } 18:39:59 DEBUG "data": { 18:39:59 DEBUG "value": [ 18:39:59 DEBUG } 18:39:59 DEBUG "resultType": "vector", 18:39:59 DEBUG 18:39:59 DEBUG "result": [ 18:39:59 DEBUG 18:39:59 DEBUG { 18:39:59 DEBUG 18:39:59 DEBUG "metric": { 18:39:59 DEBUG [...] 18:39:59 DEBUG "instance": "10.142.0.11:9100" 18:39:59 DEBUG 18:39:59 DEBUG }, 18:39:59 DEBUG 18:39:59 DEBUG "value": [ 18:39:59 DEBUG 18:39:59 DEBUG 18:39:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:39:59 DEBUG 18:39:59 DEBUG }, 18:39:59 DEBUG 18:39:59 DEBUG "value": [ 18:39:59 DEBUG [...] 18:39:59 DEBUG 1734460799, 18:39:59 DEBUG 18:39:59 DEBUG "1725.4507796508278" 18:39:59 DEBUG 18:39:59 DEBUG 18:39:59 DEBUG ] 18:39:59 DEBUG "instance": "10.142.0.51:9100" 18:39:59 DEBUG } 18:39:59 DEBUG }, 18:39:59 DEBUG ] 18:39:59 DEBUG "value": [ 18:39:59 DEBUG } 18:39:59 DEBUG 1734460799, 18:39:59 DEBUG } 18:39:59 DEBUG "0.0005333333333344588" 18:39:59 DEBUG ] 18:39:59 DEBUG } 18:39:59 DEBUG ] 18:39:59 DEBUG } 18:39:59 DEBUG } 18:40:01 INFO 18:40:01 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:40:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:40:01 INFO [loop_until]: OK (rc = 0) 18:40:01 DEBUG --- stdout --- 18:40:01 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 543Mi ds-idrepo-0 7m 721Mi rcs-5d696ff5b-7ttvr 4m 298Mi rcs-5d696ff5b-wwlbr 2m 339Mi 18:40:01 DEBUG --- stderr --- 18:40:01 DEBUG 18:40:01 INFO 18:40:01 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:40:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:40:01 INFO [loop_until]: OK (rc = 0) 18:40:01 DEBUG --- stdout --- 18:40:01 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 115m 0% 2279Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1127m 3% 3823Mi 13% 18:40:01 DEBUG --- stderr --- 18:40:01 DEBUG 18:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460859" 18:40:59 INFO [http_cmd]: http status code OK 18:40:59 DEBUG --- status code --- 18:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:40:59 DEBUG --- http response --- 18:40:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460859" 18:40:59 INFO [http_cmd]: http status code OK 18:40:59 DEBUG --- status code --- 18:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:40:59 DEBUG --- http response --- 18:40:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734460859" 18:40:59 INFO [http_cmd]: http status code OK 18:40:59 DEBUG --- status code --- 18:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:40:59 DEBUG --- http response --- 18:40:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460859" 18:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460859" 18:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460859" 18:40:59 INFO [http_cmd]: http status code OK 18:40:59 DEBUG --- status code --- 18:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:40:59 DEBUG --- http response --- 18:40:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:40:59 INFO [http_cmd]: http status code OK 18:40:59 INFO [http_cmd]: http status code OK 18:40:59 DEBUG --- status code --- 18:40:59 DEBUG --- status code --- 18:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:40:59 DEBUG --- http response --- 18:40:59 DEBUG --- http response --- 18:40:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:40:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460859" 18:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460859" 18:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460859" 18:40:59 INFO [http_cmd]: http status code OK 18:40:59 DEBUG --- status code --- 18:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:40:59 DEBUG --- http response --- 18:40:59 INFO [http_cmd]: http status code OK 18:40:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:40:59 DEBUG --- status code --- 18:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:40:59 DEBUG --- http response --- 18:40:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:40:59 INFO [http_cmd]: http status code OK 18:40:59 DEBUG --- status code --- 18:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:40:59 DEBUG --- http response --- 18:40:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460859" 18:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460859" 18:40:59 INFO [http_cmd]: http status code OK 18:40:59 DEBUG --- status code --- 18:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:40:59 DEBUG --- http response --- 18:40:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:40:59 INFO [http_cmd]: http status code OK 18:40:59 DEBUG --- status code --- 18:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:40:59 DEBUG --- http response --- 18:40:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460859" 18:40:59 INFO [http_cmd]: http status code OK 18:40:59 DEBUG --- status code --- 18:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:40:59 DEBUG --- http response --- 18:40:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734460859" 18:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460859" 18:40:59 INFO [http_cmd]: http status code OK 18:40:59 ERROR [http_cmd]: ERROR 18:40:59 DEBUG --- status code --- 18:40:59 ERROR --- status code --- 18:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:40:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:40:59 DEBUG --- http response --- 18:40:59 ERROR --- http response --- 18:40:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:40:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:40:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460859" 18:40:59 INFO [http_cmd]: http status code OK 18:40:59 DEBUG --- status code --- 18:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:40:59 DEBUG --- http response --- 18:40:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460859" 18:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460859" 18:40:59 INFO [http_cmd]: http status code OK 18:40:59 DEBUG --- status code --- 18:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:40:59 DEBUG --- http response --- 18:40:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:40:59 INFO [http_cmd]: http status code OK 18:40:59 DEBUG --- status code --- 18:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:40:59 DEBUG --- http response --- 18:40:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460859" 18:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460859" 18:40:59 INFO [http_cmd]: http status code OK 18:40:59 INFO [http_cmd]: http status code OK 18:40:59 DEBUG --- status code --- 18:40:59 DEBUG --- status code --- 18:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:40:59 DEBUG --- http response --- 18:40:59 DEBUG --- http response --- 18:40:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:40:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460859" 18:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460859" 18:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734460859" 18:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734460859" 18:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734460859" 18:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460859" 18:40:59 INFO [http_cmd]: http status code OK 18:40:59 INFO [http_cmd]: http status code OK 18:40:59 DEBUG --- status code --- 18:40:59 DEBUG --- status code --- 18:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:40:59 DEBUG --- http response --- 18:40:59 DEBUG --- http response --- 18:40:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:40:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:40:59 INFO [http_cmd]: http status code OK 18:40:59 DEBUG --- status code --- 18:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:40:59 DEBUG --- http response --- 18:40:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460859" 18:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460859" 18:40:59 INFO [http_cmd]: http status code OK 18:40:59 INFO [http_cmd]: http status code OK 18:40:59 DEBUG --- status code --- 18:40:59 DEBUG --- status code --- 18:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:40:59 DEBUG --- http response --- 18:40:59 DEBUG --- http response --- 18:40:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:40:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:40:59 INFO [http_cmd]: http status code OK 18:40:59 DEBUG --- status code --- 18:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:40:59 DEBUG --- http response --- 18:40:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:40:59 INFO [http_cmd]: http status code OK 18:40:59 INFO [http_cmd]: http status code OK 18:40:59 DEBUG --- status code --- 18:40:59 DEBUG --- status code --- 18:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:40:59 DEBUG --- http response --- 18:40:59 DEBUG --- http response --- 18:40:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:40:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460859" 18:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460859" 18:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460859" 18:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460859" 18:40:59 INFO [http_cmd]: http status code OK 18:40:59 INFO [http_cmd]: http status code OK 18:40:59 DEBUG --- status code --- 18:40:59 DEBUG --- status code --- 18:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734460859" 18:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:40:59 DEBUG --- http response --- 18:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460859" 18:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460859" 18:40:59 DEBUG --- http response --- 18:40:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:40:59 INFO [http_cmd]: http status code OK 18:40:59 DEBUG ----- output ----- 18:40:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460859" 18:40:59 DEBUG --- status code --- 18:40:59 DEBUG { 18:40:59 DEBUG ----- output ----- 18:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:40:59 DEBUG "status": "success", 18:40:59 DEBUG { 18:40:59 DEBUG --- http response --- 18:40:59 DEBUG "data": { 18:40:59 DEBUG "status": "success", 18:40:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:40:59 DEBUG "resultType": "vector", 18:40:59 DEBUG "data": { 18:40:59 DEBUG ----- output ----- 18:40:59 DEBUG "result": [ 18:40:59 DEBUG "resultType": "vector", 18:40:59 DEBUG { 18:40:59 DEBUG "status": "success", 18:40:59 DEBUG { 18:40:59 DEBUG "result": [ 18:40:59 DEBUG "data": { 18:40:59 DEBUG "metric": { 18:40:59 DEBUG { 18:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734460859" 18:40:59 DEBUG "resultType": "vector", 18:40:59 DEBUG "pod": "ds-cts-0" 18:40:59 DEBUG "metric": { 18:40:59 DEBUG "result": [ 18:40:59 DEBUG }, 18:40:59 DEBUG "pod": "ds-cts-0" 18:40:59 DEBUG { 18:40:59 DEBUG "value": [ 18:40:59 DEBUG }, 18:40:59 DEBUG "metric": { 18:40:59 DEBUG 18:40:59 DEBUG "value": [ 18:40:59 DEBUG "pod": "ds-cts-0" 18:40:59 DEBUG 18:40:59 DEBUG 18:40:59 DEBUG }, 18:40:59 DEBUG 18:40:59 DEBUG 18:40:59 DEBUG "value": [ 18:40:59 DEBUG [...] 18:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460859" 18:40:59 DEBUG 18:40:59 DEBUG [...] 18:40:59 DEBUG 18:40:59 DEBUG 18:40:59 DEBUG 18:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460859" 18:40:59 INFO [http_cmd]: http status code OK 18:40:59 DEBUG 18:40:59 DEBUG 18:40:59 DEBUG 18:40:59 DEBUG --- status code --- 18:40:59 INFO [http_cmd]: http status code OK 18:40:59 DEBUG 18:40:59 INFO [http_cmd]: http status code OK 18:40:59 INFO [http_cmd]: http status code OK 18:40:59 DEBUG 18:40:59 DEBUG [...] 18:40:59 DEBUG 18:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:40:59 DEBUG --- status code --- 18:40:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:40:59 DEBUG --- status code --- 18:40:59 DEBUG --- status code --- 18:40:59 INFO [http_cmd]: http status code OK 18:40:59 DEBUG 18:40:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:40:59 DEBUG --- http response --- 18:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:40:59 DEBUG }, 18:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:40:59 DEBUG --- http response --- 18:40:59 DEBUG 18:40:59 INFO [http_cmd]: http status code OK 18:40:59 DEBUG }, 18:40:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:40:59 DEBUG --- http response --- 18:40:59 DEBUG "value": [ 18:40:59 DEBUG --- http response --- 18:40:59 DEBUG --- status code --- 18:40:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:40:59 DEBUG 18:40:59 DEBUG --- status code --- 18:40:59 DEBUG "value": [ 18:40:59 DEBUG ----- output ----- 18:40:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:40:59 DEBUG 1734460859, 18:40:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:40:59 DEBUG ----- output ----- 18:40:59 DEBUG ----- output ----- 18:40:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:40:59 DEBUG 1734460859, 18:40:59 DEBUG { 18:40:59 DEBUG ----- output ----- 18:40:59 DEBUG "0.0034748029155156858" 18:40:59 DEBUG ] 18:40:59 DEBUG } 18:40:59 DEBUG { 18:40:59 DEBUG }, 18:40:59 DEBUG --- http response --- 18:40:59 DEBUG "712421376" 18:40:59 DEBUG "status": "success", 18:40:59 DEBUG { 18:40:59 DEBUG "status": "success", 18:40:59 DEBUG { 18:40:59 DEBUG ] 18:40:59 DEBUG "status": "success", 18:40:59 DEBUG "value": [ 18:40:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:40:59 DEBUG ] 18:40:59 DEBUG "data": { 18:40:59 DEBUG "resultType": "vector", 18:40:59 DEBUG "data": { 18:40:59 DEBUG "status": "success", 18:40:59 DEBUG } 18:40:59 DEBUG "data": { 18:40:59 DEBUG 1734460859, 18:40:59 DEBUG ----- output ----- 18:40:59 DEBUG } 18:40:59 DEBUG ] 18:40:59 DEBUG } 18:40:59 DEBUG "resultType": "vector", 18:40:59 DEBUG "data": { 18:40:59 DEBUG } 18:40:59 DEBUG "resultType": "vector", 18:40:59 DEBUG "0" 18:40:59 DEBUG { 18:40:59 DEBUG "status": "success", 18:40:59 DEBUG "result": [ 18:40:59 INFO [http_cmd]: http status code OK 18:40:59 DEBUG } 18:40:59 DEBUG "result": [ 18:40:59 DEBUG "resultType": "vector", 18:40:59 DEBUG "result": [ 18:40:59 DEBUG ] 18:40:59 DEBUG } 18:40:59 DEBUG "data": { 18:40:59 DEBUG { 18:40:59 DEBUG --- status code --- 18:40:59 DEBUG { 18:40:59 INFO [http_cmd]: http status code OK 18:40:59 DEBUG "result": [ 18:40:59 DEBUG { 18:40:59 DEBUG "metric": { 18:40:59 DEBUG "instance": "10.142.0.11:9100" 18:40:59 DEBUG }, 18:40:59 DEBUG "value": [ 18:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:40:59 DEBUG "metric": { 18:40:59 DEBUG --- status code --- 18:40:59 DEBUG { 18:40:59 DEBUG "metric": { 18:40:59 DEBUG ] 18:40:59 DEBUG "resultType": "vector", 18:40:59 DEBUG "metric": { 18:40:59 DEBUG 18:40:59 DEBUG --- http response --- 18:40:59 DEBUG "instance": "10.142.0.11:9100" 18:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:40:59 DEBUG --- http response --- 18:40:59 DEBUG "instance": "10.142.0.11:9100" 18:40:59 DEBUG } 18:40:59 DEBUG "result": [ 18:40:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:40:59 DEBUG 18:40:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:40:59 DEBUG }, 18:40:59 DEBUG "value": [ 18:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:40:59 DEBUG }, 18:40:59 DEBUG } 18:40:59 DEBUG { 18:40:59 DEBUG }, 18:40:59 DEBUG 18:40:59 DEBUG [...] 18:40:59 DEBUG 18:40:59 DEBUG 18:40:59 DEBUG 18:40:59 DEBUG "instance": "10.142.0.51:9100" 18:40:59 DEBUG }, 18:40:59 DEBUG "value": [ 18:40:59 DEBUG ----- output ----- 18:40:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:40:59 DEBUG 18:40:59 DEBUG 18:40:59 DEBUG "value": [ 18:40:59 DEBUG "metric": { 18:40:59 DEBUG "value": [ 18:40:59 DEBUG 18:40:59 DEBUG { 18:40:59 DEBUG ----- output ----- 18:40:59 DEBUG { 18:40:59 DEBUG "status": "success", 18:40:59 DEBUG 18:40:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:40:59 DEBUG 1734460859, 18:40:59 DEBUG 18:40:59 DEBUG "status": "success", 18:40:59 DEBUG "data": { 18:40:59 DEBUG "resultType": "vector", 18:40:59 DEBUG "result": [ 18:40:59 DEBUG { 18:40:59 DEBUG "metric": { 18:40:59 DEBUG "133815.46666666667" 18:40:59 DEBUG 18:40:59 DEBUG [...] 18:40:59 DEBUG 18:40:59 DEBUG "data": { 18:40:59 DEBUG 18:40:59 DEBUG }, 18:40:59 DEBUG "instance": "10.142.0.11:9100" 18:40:59 DEBUG ] 18:40:59 DEBUG } 18:40:59 DEBUG ] 18:40:59 DEBUG } 18:40:59 DEBUG } 18:40:59 DEBUG 18:40:59 DEBUG "value": [ 18:40:59 DEBUG }, 18:40:59 DEBUG "value": [ 18:40:59 DEBUG 18:40:59 DEBUG [...] 18:40:59 DEBUG "resultType": "vector", 18:40:59 DEBUG [...] 18:40:59 DEBUG 18:40:59 DEBUG 18:40:59 DEBUG 18:40:59 DEBUG 18:40:59 DEBUG 18:40:59 DEBUG "result": [ 18:40:59 DEBUG 18:40:59 DEBUG 18:40:59 DEBUG 18:40:59 DEBUG 18:40:59 DEBUG 18:40:59 DEBUG 18:40:59 DEBUG { 18:40:59 DEBUG --- http response --- 18:40:59 DEBUG 18:40:59 DEBUG [...] 18:40:59 DEBUG 18:40:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:40:59 DEBUG 18:40:59 DEBUG "metric": { 18:40:59 DEBUG "instance": "10.142.0.51:9100" 18:40:59 DEBUG 18:40:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:40:59 DEBUG [...] 18:40:59 DEBUG }, 18:40:59 DEBUG "instance": "10.142.0.51:9100" 18:40:59 DEBUG "pod": "ds-cts-0" 18:40:59 DEBUG }, 18:40:59 DEBUG 18:40:59 DEBUG 18:40:59 DEBUG 18:40:59 DEBUG "value": [ 18:40:59 DEBUG }, 18:40:59 DEBUG }, 18:40:59 DEBUG "value": [ 18:40:59 DEBUG ----- output ----- 18:40:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:40:59 DEBUG 18:40:59 DEBUG 1734460859, 18:40:59 DEBUG "value": [ 18:40:59 DEBUG "value": [ 18:40:59 DEBUG 1734460859, 18:40:59 DEBUG "25941.333333333332" 18:40:59 DEBUG }, 18:40:59 DEBUG 18:40:59 DEBUG "1.0564603149161262" 18:40:59 DEBUG 1734460859, 18:40:59 DEBUG 18:40:59 DEBUG 18:40:59 DEBUG 18:40:59 DEBUG [...] 18:40:59 DEBUG "instance": "10.142.0.51:9100" 18:40:59 DEBUG ] 18:40:59 DEBUG "131697.30000000002" 18:40:59 DEBUG { 18:40:59 DEBUG ] 18:40:59 DEBUG "value": [ 18:40:59 DEBUG 18:40:59 DEBUG }, 18:40:59 DEBUG } 18:40:59 DEBUG ] 18:40:59 DEBUG } 18:40:59 DEBUG } 18:40:59 DEBUG 1734460859, 18:40:59 DEBUG 18:40:59 DEBUG "value": [ 18:40:59 DEBUG ] 18:40:59 DEBUG "status": "success", 18:40:59 DEBUG ] 18:40:59 DEBUG ] 18:40:59 DEBUG "3051864064" 18:40:59 DEBUG 18:40:59 DEBUG 1734460859, 18:40:59 DEBUG } 18:40:59 DEBUG } 18:40:59 DEBUG } 18:40:59 DEBUG } 18:40:59 DEBUG ] 18:40:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:40:59 DEBUG "0.00043333333333303397" 18:40:59 DEBUG ] 18:40:59 DEBUG } 18:40:59 DEBUG } 18:40:59 DEBUG } 18:40:59 DEBUG }, 18:40:59 DEBUG "data": { 18:40:59 DEBUG } 18:40:59 DEBUG ] 18:40:59 DEBUG "value": [ 18:40:59 DEBUG 1734460859, 18:40:59 DEBUG ] 18:40:59 DEBUG } 18:40:59 DEBUG "resultType": "vector", 18:40:59 DEBUG "1043.4454978823528" 18:40:59 DEBUG } 18:40:59 DEBUG } 18:40:59 DEBUG "result": [ 18:40:59 DEBUG ] 18:40:59 DEBUG } 18:40:59 DEBUG { 18:40:59 DEBUG } 18:40:59 DEBUG ] 18:40:59 DEBUG "metric": { 18:40:59 DEBUG } 18:40:59 DEBUG "instance": "10.142.0.11:9100" 18:40:59 DEBUG } 18:40:59 DEBUG }, 18:40:59 DEBUG "value": [ 18:40:59 DEBUG 18:40:59 DEBUG 18:40:59 DEBUG 18:40:59 DEBUG [...] 18:40:59 DEBUG 18:40:59 DEBUG 18:40:59 DEBUG 18:40:59 DEBUG "instance": "10.142.0.51:9100" 18:40:59 DEBUG }, 18:40:59 DEBUG "value": [ 18:40:59 DEBUG 1734460859, 18:40:59 DEBUG "0" 18:40:59 DEBUG ] 18:40:59 DEBUG } 18:40:59 DEBUG ] 18:40:59 DEBUG } 18:40:59 DEBUG } 18:41:01 INFO 18:41:01 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:41:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:41:01 INFO [loop_until]: OK (rc = 0) 18:41:01 DEBUG --- stdout --- 18:41:01 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 543Mi ds-idrepo-0 6m 719Mi rcs-5d696ff5b-7ttvr 3m 298Mi rcs-5d696ff5b-wwlbr 2m 339Mi 18:41:01 DEBUG --- stderr --- 18:41:01 DEBUG 18:41:01 INFO 18:41:01 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:41:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:41:01 INFO [loop_until]: OK (rc = 0) 18:41:01 DEBUG --- stdout --- 18:41:01 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 115m 0% 2284Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1130m 3% 3818Mi 13% 18:41:01 DEBUG --- stderr --- 18:41:01 DEBUG 18:41:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460919" 18:41:59 INFO [http_cmd]: http status code OK 18:41:59 DEBUG --- status code --- 18:41:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:41:59 DEBUG --- http response --- 18:41:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:41:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460919" 18:41:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734460919" 18:41:59 INFO [http_cmd]: http status code OK 18:41:59 DEBUG --- status code --- 18:41:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:41:59 DEBUG --- http response --- 18:41:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:41:59 INFO [http_cmd]: http status code OK 18:41:59 DEBUG --- status code --- 18:41:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:41:59 DEBUG --- http response --- 18:41:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:41:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460919" 18:41:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460919" 18:41:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460919" 18:41:59 INFO [http_cmd]: http status code OK 18:41:59 DEBUG --- status code --- 18:41:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:41:59 DEBUG --- http response --- 18:41:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:41:59 INFO [http_cmd]: http status code OK 18:41:59 DEBUG --- status code --- 18:41:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:41:59 INFO [http_cmd]: http status code OK 18:41:59 DEBUG --- http response --- 18:41:59 DEBUG --- status code --- 18:41:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:41:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:41:59 DEBUG --- http response --- 18:41:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:41:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460919" 18:41:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460919" 18:41:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460919" 18:41:59 INFO [http_cmd]: http status code OK 18:41:59 INFO [http_cmd]: http status code OK 18:41:59 DEBUG --- status code --- 18:41:59 DEBUG --- status code --- 18:41:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:41:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:41:59 DEBUG --- http response --- 18:41:59 DEBUG --- http response --- 18:41:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:41:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:41:59 INFO [http_cmd]: http status code OK 18:41:59 DEBUG --- status code --- 18:41:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:41:59 DEBUG --- http response --- 18:41:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:41:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460919" 18:41:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460919" 18:41:59 INFO [http_cmd]: http status code OK 18:41:59 DEBUG --- status code --- 18:41:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:41:59 DEBUG --- http response --- 18:41:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:41:59 INFO [http_cmd]: http status code OK 18:41:59 DEBUG --- status code --- 18:41:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:41:59 DEBUG --- http response --- 18:41:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:41:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460919" 18:41:59 INFO [http_cmd]: http status code OK 18:41:59 DEBUG --- status code --- 18:41:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:41:59 DEBUG --- http response --- 18:41:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:41:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734460919" 18:41:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460919" 18:41:59 ERROR [http_cmd]: ERROR 18:41:59 ERROR --- status code --- 18:41:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:41:59 ERROR --- http response --- 18:41:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:41:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:41:59 INFO [http_cmd]: http status code OK 18:41:59 DEBUG --- status code --- 18:41:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:41:59 DEBUG --- http response --- 18:41:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:41:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460919" 18:41:59 INFO [http_cmd]: http status code OK 18:41:59 DEBUG --- status code --- 18:41:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:41:59 DEBUG --- http response --- 18:41:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:41:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460919" 18:41:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460919" 18:41:59 INFO [http_cmd]: http status code OK 18:41:59 DEBUG --- status code --- 18:41:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:41:59 INFO [http_cmd]: http status code OK 18:41:59 DEBUG --- http response --- 18:41:59 DEBUG --- status code --- 18:41:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:41:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:41:59 DEBUG --- http response --- 18:41:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:41:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460919" 18:41:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460919" 18:41:59 INFO [http_cmd]: http status code OK 18:41:59 DEBUG --- status code --- 18:41:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:41:59 DEBUG --- http response --- 18:41:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:41:59 INFO [http_cmd]: http status code OK 18:41:59 DEBUG --- status code --- 18:41:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:41:59 DEBUG --- http response --- 18:41:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:41:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460919" 18:41:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460919" 18:41:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734460919" 18:41:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734460919" 18:41:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734460919" 18:41:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460919" 18:41:59 INFO [http_cmd]: http status code OK 18:41:59 DEBUG --- status code --- 18:41:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:41:59 DEBUG --- http response --- 18:41:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:41:59 INFO [http_cmd]: http status code OK 18:41:59 DEBUG --- status code --- 18:41:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:41:59 DEBUG --- http response --- 18:41:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:41:59 INFO [http_cmd]: http status code OK 18:41:59 DEBUG --- status code --- 18:41:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:41:59 DEBUG --- http response --- 18:41:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:41:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460919" 18:41:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460919" 18:41:59 INFO [http_cmd]: http status code OK 18:41:59 INFO [http_cmd]: http status code OK 18:41:59 DEBUG --- status code --- 18:41:59 INFO [http_cmd]: http status code OK 18:41:59 DEBUG --- status code --- 18:41:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:41:59 DEBUG --- status code --- 18:41:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:41:59 DEBUG --- http response --- 18:41:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:41:59 DEBUG --- http response --- 18:41:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:41:59 DEBUG --- http response --- 18:41:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:41:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:41:59 INFO [http_cmd]: http status code OK 18:41:59 DEBUG --- status code --- 18:41:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:41:59 DEBUG --- http response --- 18:41:59 INFO [http_cmd]: http status code OK 18:41:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:41:59 DEBUG --- status code --- 18:41:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:41:59 DEBUG --- http response --- 18:41:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:41:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460919" 18:41:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460919" 18:41:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460919" 18:41:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460919" 18:41:59 INFO [http_cmd]: http status code OK 18:41:59 DEBUG --- status code --- 18:41:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:41:59 DEBUG --- http response --- 18:41:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:41:59 DEBUG ----- output ----- 18:41:59 DEBUG { 18:41:59 DEBUG "status": "success", 18:41:59 DEBUG "data": { 18:41:59 INFO [http_cmd]: http status code OK 18:41:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734460919" 18:41:59 DEBUG "resultType": "vector", 18:41:59 DEBUG --- status code --- 18:41:59 DEBUG "result": [ 18:41:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460919" 18:41:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460919" 18:41:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:41:59 DEBUG { 18:41:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734460919" 18:41:59 DEBUG --- http response --- 18:41:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460919" 18:41:59 DEBUG "metric": { 18:41:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460919" 18:41:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460919" 18:41:59 INFO [http_cmd]: http status code OK 18:41:59 INFO [http_cmd]: http status code OK 18:41:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:41:59 DEBUG "pod": "ds-cts-0" 18:41:59 DEBUG --- status code --- 18:41:59 DEBUG --- status code --- 18:41:59 DEBUG ----- output ----- 18:41:59 DEBUG }, 18:41:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:41:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:41:59 DEBUG { 18:41:59 DEBUG "value": [ 18:41:59 DEBUG --- http response --- 18:41:59 DEBUG --- http response --- 18:41:59 DEBUG "status": "success", 18:41:59 DEBUG 18:41:59 INFO [http_cmd]: http status code OK 18:41:59 INFO [http_cmd]: http status code OK 18:41:59 INFO [http_cmd]: http status code OK 18:41:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:41:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:41:59 DEBUG "data": { 18:41:59 DEBUG 18:41:59 DEBUG --- status code --- 18:41:59 DEBUG --- status code --- 18:41:59 DEBUG --- status code --- 18:41:59 DEBUG ----- output ----- 18:41:59 DEBUG ----- output ----- 18:41:59 DEBUG "resultType": "vector", 18:41:59 DEBUG 18:41:59 INFO [http_cmd]: http status code OK 18:41:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:41:59 DEBUG --- http response --- 18:41:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:41:59 DEBUG { 18:41:59 DEBUG { 18:41:59 DEBUG "result": [ 18:41:59 DEBUG [...] 18:41:59 DEBUG --- status code --- 18:41:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:41:59 DEBUG --- http response --- 18:41:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:41:59 DEBUG "status": "success", 18:41:59 DEBUG "status": "success", 18:41:59 DEBUG { 18:41:59 DEBUG 18:41:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:41:59 DEBUG --- http response --- 18:41:59 DEBUG ----- output ----- 18:41:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:41:59 INFO [http_cmd]: http status code OK 18:41:59 DEBUG "data": { 18:41:59 DEBUG "data": { 18:41:59 INFO [http_cmd]: http status code OK 18:41:59 DEBUG "metric": { 18:41:59 DEBUG "pod": "ds-cts-0" 18:41:59 DEBUG --- http response --- 18:41:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:41:59 DEBUG { 18:41:59 DEBUG ----- output ----- 18:41:59 DEBUG --- status code --- 18:41:59 DEBUG "resultType": "vector", 18:41:59 DEBUG "resultType": "vector", 18:41:59 DEBUG --- status code --- 18:41:59 DEBUG 18:41:59 DEBUG }, 18:41:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:41:59 DEBUG ----- output ----- 18:41:59 DEBUG "status": "success", 18:41:59 DEBUG { 18:41:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:41:59 DEBUG "result": [ 18:41:59 DEBUG "result": [ 18:41:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:41:59 DEBUG 18:41:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:41:59 DEBUG }, 18:41:59 DEBUG ----- output ----- 18:41:59 DEBUG "data": { 18:41:59 INFO [http_cmd]: http status code OK 18:41:59 DEBUG "status": "success", 18:41:59 DEBUG --- http response --- 18:41:59 DEBUG { 18:41:59 DEBUG { 18:41:59 DEBUG --- http response --- 18:41:59 DEBUG "value": [ 18:41:59 DEBUG "value": [ 18:41:59 DEBUG 1734460919, 18:41:59 DEBUG "0.0028661658750208656" 18:41:59 DEBUG ] 18:41:59 DEBUG } 18:41:59 DEBUG ] 18:41:59 DEBUG } 18:41:59 DEBUG "metric": { 18:41:59 DEBUG "metric": { 18:41:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:41:59 DEBUG 18:41:59 DEBUG { 18:41:59 DEBUG { 18:41:59 DEBUG "resultType": "vector", 18:41:59 DEBUG --- status code --- 18:41:59 DEBUG "data": { 18:41:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:41:59 DEBUG } 18:41:59 DEBUG "pod": "ds-cts-0" 18:41:59 DEBUG "instance": "10.142.0.11:9100" 18:41:59 DEBUG ----- output ----- 18:41:59 DEBUG 18:41:59 DEBUG "status": "success", 18:41:59 DEBUG "status": "success", 18:41:59 DEBUG "result": [ 18:41:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:41:59 DEBUG "resultType": "vector", 18:41:59 DEBUG ----- output ----- 18:41:59 DEBUG }, 18:41:59 DEBUG }, 18:41:59 DEBUG { 18:41:59 DEBUG 18:41:59 DEBUG "data": { 18:41:59 DEBUG "data": { 18:41:59 DEBUG { 18:41:59 DEBUG --- http response --- 18:41:59 DEBUG "result": [ 18:41:59 DEBUG { 18:41:59 DEBUG "value": [ 18:41:59 DEBUG "value": [ 18:41:59 DEBUG "status": "success", 18:41:59 DEBUG [...] 18:41:59 DEBUG "resultType": "vector", 18:41:59 DEBUG "resultType": "vector", 18:41:59 DEBUG "metric": { 18:41:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:41:59 DEBUG { 18:41:59 DEBUG "status": "success", 18:41:59 DEBUG 18:41:59 DEBUG 18:41:59 DEBUG "data": { 18:41:59 DEBUG 18:41:59 DEBUG "result": [ 18:41:59 DEBUG "result": [ 18:41:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:41:59 DEBUG ----- output ----- 18:41:59 DEBUG "metric": { 18:41:59 DEBUG "data": { 18:41:59 DEBUG 18:41:59 DEBUG 18:41:59 DEBUG "resultType": "vector", 18:41:59 DEBUG 18:41:59 DEBUG { 18:41:59 DEBUG { 18:41:59 DEBUG }, 18:41:59 DEBUG { 18:41:59 DEBUG "status": "success", 18:41:59 DEBUG "resultType": "vector", 18:41:59 DEBUG 18:41:59 DEBUG 18:41:59 DEBUG "result": [ 18:41:59 DEBUG 18:41:59 DEBUG "metric": { 18:41:59 DEBUG "metric": { 18:41:59 DEBUG "value": [ 18:41:59 DEBUG "instance": "10.142.0.11:9100" 18:41:59 DEBUG "data": { 18:41:59 DEBUG "result": [ 18:41:59 DEBUG [...] 18:41:59 DEBUG [...] 18:41:59 DEBUG { 18:41:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:41:59 DEBUG "instance": "10.142.0.11:9100" 18:41:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:41:59 DEBUG 18:41:59 DEBUG }, 18:41:59 DEBUG "resultType": "vector", 18:41:59 DEBUG { 18:41:59 DEBUG 18:41:59 DEBUG 18:41:59 DEBUG "metric": { 18:41:59 DEBUG }, 18:41:59 DEBUG }, 18:41:59 DEBUG }, 18:41:59 DEBUG 18:41:59 DEBUG 18:41:59 DEBUG [...] 18:41:59 DEBUG "metric": { 18:41:59 DEBUG 18:41:59 DEBUG 18:41:59 DEBUG "instance": "10.142.0.11:9100" 18:41:59 DEBUG }, 18:41:59 DEBUG "value": [ 18:41:59 DEBUG "value": [ 18:41:59 DEBUG "value": [ 18:41:59 DEBUG "result": [ 18:41:59 DEBUG 18:41:59 DEBUG "pod": "ds-cts-0" 18:41:59 DEBUG 18:41:59 DEBUG 18:41:59 DEBUG "value": [ 18:41:59 DEBUG "value": [ 18:41:59 DEBUG 18:41:59 DEBUG 18:41:59 DEBUG 18:41:59 DEBUG { 18:41:59 DEBUG 18:41:59 DEBUG }, 18:41:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:41:59 DEBUG "instance": "10.142.0.51:9100" 18:41:59 DEBUG 1734460919, 18:41:59 DEBUG 18:41:59 DEBUG 18:41:59 DEBUG 18:41:59 DEBUG 18:41:59 DEBUG "metric": { 18:41:59 DEBUG 18:41:59 DEBUG "value": [ 18:41:59 DEBUG }, 18:41:59 DEBUG }, 18:41:59 DEBUG "712445952" 18:41:59 DEBUG 18:41:59 DEBUG 18:41:59 DEBUG 18:41:59 DEBUG 18:41:59 DEBUG "instance": "10.142.0.11:9100" 18:41:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:41:59 DEBUG 18:41:59 DEBUG "value": [ 18:41:59 DEBUG "value": [ 18:41:59 DEBUG ] 18:41:59 DEBUG 18:41:59 DEBUG [...] 18:41:59 DEBUG [...] 18:41:59 DEBUG [...] 18:41:59 DEBUG }, 18:41:59 DEBUG }, 18:41:59 DEBUG 18:41:59 DEBUG 1734460919, 18:41:59 DEBUG 1734460919, 18:41:59 DEBUG } 18:41:59 DEBUG [...] 18:41:59 DEBUG 18:41:59 DEBUG 18:41:59 DEBUG 18:41:59 DEBUG "value": [ 18:41:59 DEBUG "value": [ 18:41:59 DEBUG 18:41:59 DEBUG "0" 18:41:59 DEBUG "131747.00000000003" 18:41:59 DEBUG ] 18:41:59 DEBUG 18:41:59 DEBUG 18:41:59 DEBUG 18:41:59 DEBUG 18:41:59 DEBUG 18:41:59 DEBUG 1734460919, 18:41:59 DEBUG [...] 18:41:59 DEBUG ] 18:41:59 DEBUG } 18:41:59 DEBUG ] 18:41:59 DEBUG } 18:41:59 DEBUG } 18:41:59 DEBUG 18:41:59 DEBUG 18:41:59 DEBUG 18:41:59 DEBUG "1.0519602581781504" 18:41:59 DEBUG 18:41:59 DEBUG ] 18:41:59 DEBUG } 18:41:59 DEBUG 18:41:59 DEBUG 18:41:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:41:59 DEBUG "instance": "10.142.0.51:9100" 18:41:59 DEBUG }, 18:41:59 DEBUG "value": [ 18:41:59 DEBUG 1734460919, 18:41:59 DEBUG "18568.533333333333" 18:41:59 DEBUG } 18:41:59 DEBUG 18:41:59 DEBUG "instance": "10.142.0.51:9100" 18:41:59 DEBUG }, 18:41:59 DEBUG 18:41:59 DEBUG ] 18:41:59 DEBUG 18:41:59 DEBUG ] 18:41:59 DEBUG } 18:41:59 DEBUG ] 18:41:59 DEBUG "instance": "10.142.0.51:9100" 18:41:59 DEBUG }, 18:41:59 DEBUG "value": [ 18:41:59 DEBUG [...] 18:41:59 DEBUG } 18:41:59 DEBUG 18:41:59 DEBUG } 18:41:59 DEBUG ] 18:41:59 DEBUG } 18:41:59 DEBUG } 18:41:59 DEBUG 1734460919, 18:41:59 DEBUG 18:41:59 DEBUG ] 18:41:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:41:59 DEBUG } 18:41:59 DEBUG }, 18:41:59 DEBUG "value": [ 18:41:59 DEBUG "3053305856" 18:41:59 DEBUG 18:41:59 DEBUG } 18:41:59 DEBUG }, 18:41:59 DEBUG } 18:41:59 DEBUG "value": [ 18:41:59 DEBUG 1734460919, 18:41:59 DEBUG ] 18:41:59 DEBUG 18:41:59 DEBUG } 18:41:59 DEBUG "value": [ 18:41:59 DEBUG 1734460919, 18:41:59 DEBUG "131971.63333333333" 18:41:59 DEBUG } 18:41:59 DEBUG "instance": "10.142.0.51:9100" 18:41:59 DEBUG 1734460919, 18:41:59 DEBUG "0.0003333333333333854" 18:41:59 DEBUG ] 18:41:59 DEBUG ] 18:41:59 DEBUG }, 18:41:59 DEBUG "1869.7733823872509" 18:41:59 DEBUG ] 18:41:59 DEBUG } 18:41:59 DEBUG } 18:41:59 DEBUG "value": [ 18:41:59 DEBUG ] 18:41:59 DEBUG } 18:41:59 DEBUG ] 18:41:59 DEBUG } 18:41:59 DEBUG 1734460919, 18:41:59 DEBUG } 18:41:59 DEBUG ] 18:41:59 DEBUG } 18:41:59 DEBUG "0.000366666666666049" 18:41:59 DEBUG ] 18:41:59 DEBUG } 18:41:59 DEBUG } 18:41:59 DEBUG ] 18:41:59 DEBUG } 18:41:59 DEBUG } 18:41:59 DEBUG } 18:41:59 DEBUG } 18:41:59 DEBUG ] 18:41:59 DEBUG } 18:41:59 DEBUG } 18:42:01 INFO 18:42:01 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:42:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:42:01 INFO [loop_until]: OK (rc = 0) 18:42:01 DEBUG --- stdout --- 18:42:01 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 543Mi ds-idrepo-0 6m 721Mi rcs-5d696ff5b-7ttvr 3m 298Mi rcs-5d696ff5b-wwlbr 3m 339Mi 18:42:01 DEBUG --- stderr --- 18:42:01 DEBUG 18:42:01 INFO 18:42:01 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:42:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:42:01 INFO [loop_until]: OK (rc = 0) 18:42:01 DEBUG --- stdout --- 18:42:01 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 103m 0% 2280Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1158m 3% 3610Mi 12% 18:42:01 DEBUG --- stderr --- 18:42:01 DEBUG 18:42:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460979" 18:42:59 INFO [http_cmd]: http status code OK 18:42:59 DEBUG --- status code --- 18:42:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:42:59 DEBUG --- http response --- 18:42:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:42:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460979" 18:42:59 INFO [http_cmd]: http status code OK 18:42:59 DEBUG --- status code --- 18:42:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:42:59 DEBUG --- http response --- 18:42:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:42:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734460979" 18:42:59 INFO [http_cmd]: http status code OK 18:42:59 DEBUG --- status code --- 18:42:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:42:59 DEBUG --- http response --- 18:42:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:42:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460979" 18:42:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460979" 18:42:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460979" 18:42:59 INFO [http_cmd]: http status code OK 18:42:59 DEBUG --- status code --- 18:42:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:42:59 DEBUG --- http response --- 18:42:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:42:59 INFO [http_cmd]: http status code OK 18:42:59 DEBUG --- status code --- 18:42:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:42:59 DEBUG --- http response --- 18:42:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:42:59 INFO [http_cmd]: http status code OK 18:42:59 DEBUG --- status code --- 18:42:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:42:59 DEBUG --- http response --- 18:42:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:42:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460979" 18:42:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460979" 18:42:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460979" 18:42:59 INFO [http_cmd]: http status code OK 18:42:59 INFO [http_cmd]: http status code OK 18:42:59 DEBUG --- status code --- 18:42:59 DEBUG --- status code --- 18:42:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:42:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:42:59 DEBUG --- http response --- 18:42:59 DEBUG --- http response --- 18:42:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:42:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:42:59 INFO [http_cmd]: http status code OK 18:42:59 DEBUG --- status code --- 18:42:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:42:59 DEBUG --- http response --- 18:42:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:42:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460979" 18:42:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460979" 18:42:59 INFO [http_cmd]: http status code OK 18:42:59 INFO [http_cmd]: http status code OK 18:42:59 DEBUG --- status code --- 18:42:59 DEBUG --- status code --- 18:42:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:42:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:42:59 DEBUG --- http response --- 18:42:59 DEBUG --- http response --- 18:42:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:42:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:42:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460979" 18:42:59 INFO [http_cmd]: http status code OK 18:42:59 DEBUG --- status code --- 18:42:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:42:59 DEBUG --- http response --- 18:42:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:42:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734460979" 18:42:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460979" 18:42:59 INFO [http_cmd]: http status code OK 18:42:59 ERROR [http_cmd]: ERROR 18:42:59 DEBUG --- status code --- 18:42:59 ERROR --- status code --- 18:42:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:42:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:42:59 DEBUG --- http response --- 18:42:59 ERROR --- http response --- 18:42:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:42:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:42:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:42:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460979" 18:42:59 INFO [http_cmd]: http status code OK 18:42:59 DEBUG --- status code --- 18:42:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:42:59 DEBUG --- http response --- 18:42:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:42:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460979" 18:42:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460979" 18:42:59 INFO [http_cmd]: http status code OK 18:42:59 DEBUG --- status code --- 18:42:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:42:59 DEBUG --- http response --- 18:42:59 INFO [http_cmd]: http status code OK 18:42:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:42:59 DEBUG --- status code --- 18:42:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:42:59 DEBUG --- http response --- 18:42:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:42:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460979" 18:42:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460979" 18:42:59 INFO [http_cmd]: http status code OK 18:42:59 DEBUG --- status code --- 18:42:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:42:59 DEBUG --- http response --- 18:42:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:42:59 INFO [http_cmd]: http status code OK 18:42:59 DEBUG --- status code --- 18:42:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:42:59 DEBUG --- http response --- 18:42:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:42:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460979" 18:42:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460979" 18:42:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734460979" 18:42:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460979" 18:42:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734460979" 18:42:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734460979" 18:42:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460979" 18:42:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460979" 18:42:59 INFO [http_cmd]: http status code OK 18:42:59 DEBUG --- status code --- 18:42:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:42:59 DEBUG --- http response --- 18:42:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:42:59 INFO [http_cmd]: http status code OK 18:42:59 DEBUG --- status code --- 18:42:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:42:59 DEBUG --- http response --- 18:42:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:42:59 INFO [http_cmd]: http status code OK 18:42:59 DEBUG --- status code --- 18:42:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:42:59 DEBUG --- http response --- 18:42:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:42:59 INFO [http_cmd]: http status code OK 18:42:59 DEBUG --- status code --- 18:42:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:42:59 INFO [http_cmd]: http status code OK 18:42:59 DEBUG --- http response --- 18:42:59 DEBUG --- status code --- 18:42:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:42:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:42:59 DEBUG --- http response --- 18:42:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:42:59 INFO [http_cmd]: http status code OK 18:42:59 INFO [http_cmd]: http status code OK 18:42:59 DEBUG --- status code --- 18:42:59 DEBUG --- status code --- 18:42:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:42:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:42:59 DEBUG --- http response --- 18:42:59 DEBUG --- http response --- 18:42:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:42:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:42:59 INFO [http_cmd]: http status code OK 18:42:59 DEBUG --- status code --- 18:42:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:42:59 DEBUG --- http response --- 18:42:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:42:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460979" 18:42:59 INFO [http_cmd]: http status code OK 18:42:59 DEBUG --- status code --- 18:42:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:42:59 DEBUG --- http response --- 18:42:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:42:59 DEBUG ----- output ----- 18:42:59 DEBUG { 18:42:59 DEBUG "status": "success", 18:42:59 DEBUG "data": { 18:42:59 DEBUG "resultType": "vector", 18:42:59 DEBUG "result": [ 18:42:59 DEBUG { 18:42:59 DEBUG "metric": { 18:42:59 DEBUG "pod": "ds-cts-0" 18:42:59 DEBUG }, 18:42:59 DEBUG "value": [ 18:42:59 DEBUG 18:42:59 DEBUG 18:42:59 DEBUG 18:42:59 DEBUG [...] 18:42:59 DEBUG 18:42:59 DEBUG 18:42:59 DEBUG 18:42:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:42:59 DEBUG }, 18:42:59 DEBUG "value": [ 18:42:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460979" 18:42:59 DEBUG 1734460979, 18:42:59 DEBUG "0.004460721457575055" 18:42:59 DEBUG ] 18:42:59 DEBUG } 18:42:59 DEBUG ] 18:42:59 DEBUG } 18:42:59 DEBUG } 18:42:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734460979" 18:42:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460979" 18:42:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460979" 18:42:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734460979" 18:42:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734460979" 18:42:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460979" 18:42:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734460979" 18:42:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460979" 18:42:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734460979" 18:42:59 INFO [http_cmd]: http status code OK 18:42:59 DEBUG --- status code --- 18:42:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:42:59 DEBUG --- http response --- 18:42:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:42:59 DEBUG ----- output ----- 18:42:59 DEBUG { 18:42:59 DEBUG "status": "success", 18:42:59 INFO [http_cmd]: http status code OK 18:42:59 INFO [http_cmd]: http status code OK 18:42:59 INFO [http_cmd]: http status code OK 18:42:59 INFO [http_cmd]: http status code OK 18:42:59 DEBUG "data": { 18:42:59 DEBUG --- status code --- 18:42:59 DEBUG --- status code --- 18:42:59 DEBUG --- status code --- 18:42:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:42:59 DEBUG "resultType": "vector", 18:42:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:42:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:42:59 DEBUG --- status code --- 18:42:59 DEBUG --- http response --- 18:42:59 DEBUG "result": [ 18:42:59 INFO [http_cmd]: http status code OK 18:42:59 INFO [http_cmd]: http status code OK 18:42:59 DEBUG --- http response --- 18:42:59 INFO [http_cmd]: http status code OK 18:42:59 DEBUG --- http response --- 18:42:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:42:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:42:59 DEBUG { 18:42:59 DEBUG "metric": { 18:42:59 DEBUG "pod": "ds-cts-0" 18:42:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:42:59 DEBUG --- status code --- 18:42:59 DEBUG --- http response --- 18:42:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:42:59 DEBUG ----- output ----- 18:42:59 DEBUG --- status code --- 18:42:59 DEBUG --- status code --- 18:42:59 DEBUG }, 18:42:59 DEBUG ----- output ----- 18:42:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:42:59 DEBUG ----- output ----- 18:42:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:42:59 DEBUG { 18:42:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:42:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:42:59 DEBUG "value": [ 18:42:59 DEBUG { 18:42:59 DEBUG --- http response --- 18:42:59 DEBUG { 18:42:59 DEBUG ----- output ----- 18:42:59 DEBUG "status": "success", 18:42:59 DEBUG --- http response --- 18:42:59 INFO [http_cmd]: http status code OK 18:42:59 INFO [http_cmd]: http status code OK 18:42:59 DEBUG --- http response --- 18:42:59 DEBUG 18:42:59 DEBUG "status": "success", 18:42:59 DEBUG "status": "success", 18:42:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:42:59 DEBUG { 18:42:59 DEBUG "data": { 18:42:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:42:59 DEBUG --- status code --- 18:42:59 DEBUG --- status code --- 18:42:59 DEBUG 18:42:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:42:59 DEBUG "data": { 18:42:59 DEBUG "data": { 18:42:59 DEBUG ----- output ----- 18:42:59 DEBUG "status": "success", 18:42:59 DEBUG "resultType": "vector", 18:42:59 DEBUG ----- output ----- 18:42:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:42:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:42:59 DEBUG 18:42:59 DEBUG ----- output ----- 18:42:59 DEBUG "resultType": "vector", 18:42:59 DEBUG "resultType": "vector", 18:42:59 DEBUG { 18:42:59 DEBUG "data": { 18:42:59 DEBUG "result": [ 18:42:59 DEBUG { 18:42:59 DEBUG --- http response --- 18:42:59 DEBUG --- http response --- 18:42:59 DEBUG [...] 18:42:59 DEBUG { 18:42:59 DEBUG "result": [ 18:42:59 DEBUG "result": [ 18:42:59 DEBUG "status": "success", 18:42:59 DEBUG "resultType": "vector", 18:42:59 DEBUG { 18:42:59 DEBUG "status": "success", 18:42:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:42:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:42:59 DEBUG 18:42:59 DEBUG "status": "success", 18:42:59 DEBUG { 18:42:59 DEBUG { 18:42:59 DEBUG "data": { 18:42:59 DEBUG "resultType": "vector", 18:42:59 DEBUG "result": [ 18:42:59 DEBUG "data": { 18:42:59 DEBUG ----- output ----- 18:42:59 DEBUG ----- output ----- 18:42:59 DEBUG 18:42:59 DEBUG "data": { 18:42:59 DEBUG "metric": { 18:42:59 DEBUG "metric": { 18:42:59 DEBUG "metric": { 18:42:59 DEBUG "result": [ 18:42:59 DEBUG { 18:42:59 DEBUG "resultType": "vector", 18:42:59 DEBUG { 18:42:59 DEBUG { 18:42:59 DEBUG 18:42:59 DEBUG "resultType": "vector", 18:42:59 DEBUG "pod": "ds-cts-0" 18:42:59 DEBUG "instance": "10.142.0.11:9100" 18:42:59 DEBUG "instance": "10.142.0.11:9100" 18:42:59 DEBUG { 18:42:59 DEBUG "metric": { 18:42:59 DEBUG "result": [ 18:42:59 DEBUG "status": "success", 18:42:59 DEBUG "status": "success", 18:42:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:42:59 DEBUG "result": [ 18:42:59 DEBUG }, 18:42:59 DEBUG }, 18:42:59 DEBUG }, 18:42:59 DEBUG "metric": { 18:42:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:42:59 DEBUG { 18:42:59 DEBUG "data": { 18:42:59 DEBUG "data": { 18:42:59 DEBUG }, 18:42:59 DEBUG { 18:42:59 DEBUG "value": [ 18:42:59 DEBUG "value": [ 18:42:59 DEBUG "value": [ 18:42:59 DEBUG "instance": "10.142.0.11:9100" 18:42:59 DEBUG }, 18:42:59 DEBUG "metric": { 18:42:59 DEBUG "resultType": "vector", 18:42:59 DEBUG "resultType": "vector", 18:42:59 DEBUG "value": [ 18:42:59 DEBUG "metric": { 18:42:59 DEBUG 18:42:59 DEBUG 18:42:59 DEBUG 18:42:59 DEBUG }, 18:42:59 DEBUG "value": [ 18:42:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:42:59 DEBUG "result": [ 18:42:59 DEBUG "result": [ 18:42:59 DEBUG 1734460979, 18:42:59 DEBUG "instance": "10.142.0.11:9100" 18:42:59 DEBUG 18:42:59 DEBUG 18:42:59 DEBUG 18:42:59 DEBUG "value": [ 18:42:59 DEBUG 18:42:59 DEBUG }, 18:42:59 DEBUG { 18:42:59 DEBUG { 18:42:59 DEBUG "metric": { 18:42:59 DEBUG "pod": "ds-cts-0" 18:42:59 DEBUG }, 18:42:59 DEBUG "value": [ 18:42:59 DEBUG 18:42:59 DEBUG 18:42:59 DEBUG 18:42:59 DEBUG "value": [ 18:42:59 DEBUG "metric": { 18:42:59 DEBUG "713281536" 18:42:59 DEBUG }, 18:42:59 DEBUG 18:42:59 DEBUG 18:42:59 DEBUG 18:42:59 DEBUG 18:42:59 DEBUG [...] 18:42:59 DEBUG 18:42:59 DEBUG 18:42:59 DEBUG "instance": "10.142.0.11:9100" 18:42:59 DEBUG ] 18:42:59 DEBUG "value": [ 18:42:59 DEBUG [...] 18:42:59 DEBUG [...] 18:42:59 DEBUG 18:42:59 DEBUG 18:42:59 DEBUG 18:42:59 DEBUG [...] 18:42:59 DEBUG 18:42:59 DEBUG }, 18:42:59 DEBUG } 18:42:59 DEBUG 18:42:59 DEBUG 18:42:59 DEBUG 18:42:59 DEBUG 18:42:59 DEBUG [...] 18:42:59 DEBUG 18:42:59 DEBUG 18:42:59 DEBUG 18:42:59 DEBUG "value": [ 18:42:59 DEBUG 18:42:59 DEBUG 18:42:59 DEBUG 18:42:59 DEBUG 18:42:59 DEBUG [...] 18:42:59 DEBUG 18:42:59 DEBUG 18:42:59 DEBUG 18:42:59 DEBUG 18:42:59 DEBUG ] 18:42:59 DEBUG 18:42:59 DEBUG 18:42:59 DEBUG 18:42:59 DEBUG 18:42:59 DEBUG 18:42:59 DEBUG 18:42:59 DEBUG "instance": "10.142.0.51:9100" 18:42:59 DEBUG [...] 18:42:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:42:59 DEBUG } 18:42:59 DEBUG 18:42:59 DEBUG [...] 18:42:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:42:59 DEBUG "instance": "10.142.0.51:9100" 18:42:59 DEBUG 18:42:59 DEBUG 18:42:59 DEBUG }, 18:42:59 DEBUG 18:42:59 DEBUG }, 18:42:59 DEBUG } 18:42:59 DEBUG [...] 18:42:59 DEBUG 18:42:59 DEBUG }, 18:42:59 DEBUG }, 18:42:59 DEBUG 18:42:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:42:59 DEBUG "value": [ 18:42:59 DEBUG 18:42:59 DEBUG "value": [ 18:42:59 DEBUG 18:42:59 DEBUG 18:42:59 DEBUG "value": [ 18:42:59 DEBUG "value": [ 18:42:59 DEBUG "instance": "10.142.0.51:9100" 18:42:59 DEBUG }, 18:42:59 DEBUG 1734460979, 18:42:59 DEBUG 18:42:59 DEBUG 1734460979, 18:42:59 DEBUG 18:42:59 DEBUG 18:42:59 DEBUG 1734460979, 18:42:59 DEBUG 1734460979, 18:42:59 DEBUG }, 18:42:59 DEBUG "value": [ 18:42:59 DEBUG "79014.4" 18:42:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:42:59 DEBUG "2712113152" 18:42:59 DEBUG 18:42:59 DEBUG "instance": "10.142.0.51:9100" 18:42:59 DEBUG "0" 18:42:59 DEBUG ] 18:42:59 DEBUG } 18:42:59 DEBUG 1734460979, 18:42:59 DEBUG ] 18:42:59 DEBUG }, 18:42:59 DEBUG ] 18:42:59 DEBUG "instance": "10.142.0.51:9100" 18:42:59 DEBUG }, 18:42:59 DEBUG "18978.13333333333" 18:42:59 DEBUG "value": [ 18:42:59 DEBUG ] 18:42:59 DEBUG "1078.6475181095059" 18:42:59 DEBUG } 18:42:59 DEBUG "value": [ 18:42:59 DEBUG } 18:42:59 DEBUG }, 18:42:59 DEBUG "value": [ 18:42:59 DEBUG ] 18:42:59 DEBUG 1734460979, 18:42:59 DEBUG } 18:42:59 DEBUG ] 18:42:59 DEBUG ] 18:42:59 DEBUG 1734460979, 18:42:59 DEBUG ] 18:42:59 DEBUG "value": [ 18:42:59 DEBUG 1734460979, 18:42:59 DEBUG } 18:42:59 DEBUG "0.0003333333333333854" 18:42:59 DEBUG } 18:42:59 DEBUG } 18:42:59 DEBUG } 18:42:59 DEBUG "1.7343266423842039" 18:42:59 DEBUG } 18:42:59 DEBUG 1734460979, 18:42:59 DEBUG "0.00043333333333208654" 18:42:59 DEBUG ] 18:42:59 DEBUG ] 18:42:59 DEBUG ] 18:42:59 DEBUG } 18:42:59 DEBUG ] 18:42:59 DEBUG } 18:42:59 DEBUG ] 18:42:59 DEBUG "77285.70000000001" 18:42:59 DEBUG } 18:42:59 DEBUG } 18:42:59 DEBUG } 18:42:59 DEBUG } 18:42:59 DEBUG } 18:42:59 DEBUG ] 18:42:59 DEBUG } 18:42:59 DEBUG ] 18:42:59 DEBUG } 18:42:59 DEBUG } 18:42:59 DEBUG } 18:42:59 DEBUG ] 18:42:59 DEBUG } 18:42:59 DEBUG ] 18:42:59 DEBUG } 18:42:59 DEBUG } 18:42:59 DEBUG } 18:42:59 DEBUG ] 18:42:59 DEBUG } 18:42:59 DEBUG } 18:42:59 DEBUG } 18:43:01 INFO 18:43:01 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:43:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:43:01 INFO [loop_until]: OK (rc = 0) 18:43:01 DEBUG --- stdout --- 18:43:01 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 543Mi ds-idrepo-0 7m 722Mi rcs-5d696ff5b-7ttvr 3m 299Mi rcs-5d696ff5b-wwlbr 3m 339Mi 18:43:01 DEBUG --- stderr --- 18:43:01 DEBUG 18:43:01 INFO 18:43:01 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:43:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:43:01 INFO [loop_until]: OK (rc = 0) 18:43:01 DEBUG --- stdout --- 18:43:01 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 111m 0% 2290Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1810m 5% 3613Mi 12% 18:43:01 DEBUG --- stderr --- 18:43:01 DEBUG 18:43:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461039" 18:43:59 INFO [http_cmd]: http status code OK 18:43:59 DEBUG --- status code --- 18:43:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:43:59 DEBUG --- http response --- 18:43:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:43:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461039" 18:43:59 INFO [http_cmd]: http status code OK 18:43:59 DEBUG --- status code --- 18:43:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:43:59 DEBUG --- http response --- 18:43:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:43:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734461039" 18:43:59 INFO [http_cmd]: http status code OK 18:43:59 DEBUG --- status code --- 18:43:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:43:59 DEBUG --- http response --- 18:43:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:43:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461039" 18:43:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461039" 18:43:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461039" 18:43:59 INFO [http_cmd]: http status code OK 18:43:59 DEBUG --- status code --- 18:43:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:43:59 DEBUG --- http response --- 18:43:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:43:59 INFO [http_cmd]: http status code OK 18:43:59 DEBUG --- status code --- 18:43:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:43:59 DEBUG --- http response --- 18:43:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:43:59 INFO [http_cmd]: http status code OK 18:43:59 DEBUG --- status code --- 18:43:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:43:59 DEBUG --- http response --- 18:43:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:43:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461039" 18:43:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461039" 18:43:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461039" 18:43:59 INFO [http_cmd]: http status code OK 18:43:59 DEBUG --- status code --- 18:43:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:43:59 INFO [http_cmd]: http status code OK 18:43:59 DEBUG --- http response --- 18:43:59 DEBUG --- status code --- 18:43:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:43:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:43:59 DEBUG --- http response --- 18:43:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:43:59 INFO [http_cmd]: http status code OK 18:43:59 DEBUG --- status code --- 18:43:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:43:59 DEBUG --- http response --- 18:43:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:43:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461039" 18:43:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461039" 18:43:59 INFO [http_cmd]: http status code OK 18:43:59 DEBUG --- status code --- 18:43:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:43:59 INFO [http_cmd]: http status code OK 18:43:59 DEBUG --- http response --- 18:43:59 DEBUG --- status code --- 18:43:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:43:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:43:59 DEBUG --- http response --- 18:43:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:43:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461039" 18:43:59 INFO [http_cmd]: http status code OK 18:43:59 DEBUG --- status code --- 18:43:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:43:59 DEBUG --- http response --- 18:43:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:43:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734461039" 18:43:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461039" 18:43:59 INFO [http_cmd]: http status code OK 18:43:59 ERROR [http_cmd]: ERROR 18:43:59 DEBUG --- status code --- 18:43:59 ERROR --- status code --- 18:43:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:43:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:43:59 DEBUG --- http response --- 18:43:59 ERROR --- http response --- 18:43:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:43:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:43:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:43:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461039" 18:43:59 INFO [http_cmd]: http status code OK 18:43:59 DEBUG --- status code --- 18:43:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:43:59 DEBUG --- http response --- 18:43:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:43:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461039" 18:43:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461039" 18:43:59 INFO [http_cmd]: http status code OK 18:43:59 DEBUG --- status code --- 18:43:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:43:59 INFO [http_cmd]: http status code OK 18:43:59 DEBUG --- http response --- 18:43:59 DEBUG --- status code --- 18:43:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:43:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:43:59 DEBUG --- http response --- 18:43:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:43:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461039" 18:43:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461039" 18:43:59 INFO [http_cmd]: http status code OK 18:43:59 DEBUG --- status code --- 18:43:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:43:59 DEBUG --- http response --- 18:43:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:43:59 INFO [http_cmd]: http status code OK 18:43:59 DEBUG --- status code --- 18:43:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:43:59 DEBUG --- http response --- 18:43:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:43:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734461039" 18:43:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461039" 18:43:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461039" 18:43:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734461039" 18:43:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461039" 18:43:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461039" 18:43:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734461039" 18:43:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461039" 18:43:59 INFO [http_cmd]: http status code OK 18:43:59 INFO [http_cmd]: http status code OK 18:43:59 DEBUG --- status code --- 18:43:59 INFO [http_cmd]: http status code OK 18:43:59 DEBUG --- status code --- 18:43:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:43:59 DEBUG --- status code --- 18:43:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:43:59 DEBUG --- http response --- 18:43:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:43:59 DEBUG --- http response --- 18:43:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:43:59 DEBUG --- http response --- 18:43:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:43:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:43:59 INFO [http_cmd]: http status code OK 18:43:59 DEBUG --- status code --- 18:43:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:43:59 INFO [http_cmd]: http status code OK 18:43:59 INFO [http_cmd]: http status code OK 18:43:59 DEBUG --- http response --- 18:43:59 INFO [http_cmd]: http status code OK 18:43:59 DEBUG --- status code --- 18:43:59 DEBUG --- status code --- 18:43:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:43:59 DEBUG --- status code --- 18:43:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:43:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:43:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:43:59 DEBUG --- http response --- 18:43:59 DEBUG --- http response --- 18:43:59 DEBUG --- http response --- 18:43:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:43:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:43:59 INFO [http_cmd]: http status code OK 18:43:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:43:59 DEBUG --- status code --- 18:43:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:43:59 DEBUG --- http response --- 18:43:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:43:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461039" 18:43:59 INFO [http_cmd]: http status code OK 18:43:59 DEBUG --- status code --- 18:43:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:43:59 DEBUG --- http response --- 18:43:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:43:59 DEBUG ----- output ----- 18:43:59 DEBUG { 18:43:59 DEBUG "status": "success", 18:43:59 DEBUG "data": { 18:43:59 DEBUG "resultType": "vector", 18:43:59 DEBUG "result": [ 18:43:59 DEBUG { 18:43:59 DEBUG "metric": { 18:43:59 DEBUG "pod": "ds-cts-0" 18:43:59 DEBUG }, 18:43:59 DEBUG "value": [ 18:43:59 DEBUG 18:43:59 DEBUG 18:43:59 DEBUG 18:43:59 DEBUG [...] 18:43:59 DEBUG 18:43:59 DEBUG 18:43:59 DEBUG 18:43:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:43:59 DEBUG }, 18:43:59 DEBUG "value": [ 18:43:59 DEBUG 1734461039, 18:43:59 DEBUG "0.003951504547041392" 18:43:59 DEBUG ] 18:43:59 DEBUG } 18:43:59 DEBUG ] 18:43:59 DEBUG } 18:43:59 DEBUG } 18:43:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461039" 18:43:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461039" 18:43:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461039" 18:43:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734461039" 18:43:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461039" 18:43:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461039" 18:43:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461039" 18:43:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461039" 18:43:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734461039" 18:43:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461039" 18:43:59 INFO [http_cmd]: http status code OK 18:43:59 DEBUG --- status code --- 18:43:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:43:59 DEBUG --- http response --- 18:43:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:43:59 DEBUG ----- output ----- 18:43:59 DEBUG { 18:43:59 DEBUG "status": "success", 18:43:59 INFO [http_cmd]: http status code OK 18:43:59 INFO [http_cmd]: http status code OK 18:43:59 INFO [http_cmd]: http status code OK 18:43:59 DEBUG "data": { 18:43:59 INFO [http_cmd]: http status code OK 18:43:59 INFO [http_cmd]: http status code OK 18:43:59 DEBUG --- status code --- 18:43:59 INFO [http_cmd]: http status code OK 18:43:59 DEBUG --- status code --- 18:43:59 DEBUG --- status code --- 18:43:59 DEBUG "resultType": "vector", 18:43:59 DEBUG --- status code --- 18:43:59 DEBUG --- status code --- 18:43:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:43:59 DEBUG --- status code --- 18:43:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:43:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:43:59 DEBUG "result": [ 18:43:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:43:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:43:59 DEBUG --- http response --- 18:43:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:43:59 DEBUG --- http response --- 18:43:59 DEBUG --- http response --- 18:43:59 INFO [http_cmd]: http status code OK 18:43:59 DEBUG { 18:43:59 DEBUG --- http response --- 18:43:59 DEBUG --- http response --- 18:43:59 INFO [http_cmd]: http status code OK 18:43:59 DEBUG --- http response --- 18:43:59 INFO [http_cmd]: http status code OK 18:43:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:43:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:43:59 DEBUG --- status code --- 18:43:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:43:59 DEBUG "metric": { 18:43:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:43:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:43:59 DEBUG --- status code --- 18:43:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:43:59 DEBUG --- status code --- 18:43:59 DEBUG ----- output ----- 18:43:59 DEBUG ----- output ----- 18:43:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:43:59 DEBUG ----- output ----- 18:43:59 DEBUG "pod": "ds-cts-0" 18:43:59 DEBUG ----- output ----- 18:43:59 DEBUG ----- output ----- 18:43:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:43:59 DEBUG ----- output ----- 18:43:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:43:59 DEBUG { 18:43:59 DEBUG { 18:43:59 DEBUG --- http response --- 18:43:59 DEBUG { 18:43:59 DEBUG }, 18:43:59 DEBUG { 18:43:59 DEBUG { 18:43:59 DEBUG --- http response --- 18:43:59 DEBUG { 18:43:59 DEBUG --- http response --- 18:43:59 DEBUG "status": "success", 18:43:59 DEBUG "status": "success", 18:43:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:43:59 DEBUG "status": "success", 18:43:59 DEBUG "value": [ 18:43:59 DEBUG "status": "success", 18:43:59 DEBUG "status": "success", 18:43:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:43:59 DEBUG "status": "success", 18:43:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:43:59 DEBUG "data": { 18:43:59 DEBUG "data": { 18:43:59 DEBUG ----- output ----- 18:43:59 DEBUG "data": { 18:43:59 DEBUG 18:43:59 DEBUG "data": { 18:43:59 DEBUG "data": { 18:43:59 DEBUG ----- output ----- 18:43:59 DEBUG "data": { 18:43:59 DEBUG ----- output ----- 18:43:59 DEBUG "resultType": "vector", 18:43:59 DEBUG "resultType": "vector", 18:43:59 DEBUG { 18:43:59 DEBUG "resultType": "vector", 18:43:59 DEBUG 18:43:59 DEBUG "resultType": "vector", 18:43:59 DEBUG "resultType": "vector", 18:43:59 DEBUG { 18:43:59 DEBUG "resultType": "vector", 18:43:59 DEBUG { 18:43:59 DEBUG "result": [ 18:43:59 DEBUG "result": [ 18:43:59 DEBUG "status": "success", 18:43:59 DEBUG "result": [ 18:43:59 DEBUG 18:43:59 DEBUG "result": [ 18:43:59 DEBUG "result": [ 18:43:59 DEBUG "status": "success", 18:43:59 DEBUG "result": [ 18:43:59 DEBUG "status": "success", 18:43:59 DEBUG { 18:43:59 DEBUG { 18:43:59 DEBUG "data": { 18:43:59 DEBUG { 18:43:59 DEBUG [...] 18:43:59 DEBUG { 18:43:59 DEBUG { 18:43:59 DEBUG "data": { 18:43:59 DEBUG { 18:43:59 DEBUG "data": { 18:43:59 DEBUG "metric": { 18:43:59 DEBUG "metric": { 18:43:59 DEBUG "resultType": "vector", 18:43:59 DEBUG "metric": { 18:43:59 DEBUG 18:43:59 DEBUG "metric": { 18:43:59 DEBUG "instance": "10.142.0.11:9100" 18:43:59 DEBUG "resultType": "vector", 18:43:59 DEBUG "metric": { 18:43:59 DEBUG "resultType": "vector", 18:43:59 DEBUG "instance": "10.142.0.11:9100" 18:43:59 DEBUG "instance": "10.142.0.11:9100" 18:43:59 DEBUG "result": [ 18:43:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:43:59 DEBUG 18:43:59 DEBUG "metric": { 18:43:59 DEBUG }, 18:43:59 DEBUG "result": [ 18:43:59 DEBUG "instance": "10.142.0.11:9100" 18:43:59 DEBUG "result": [ 18:43:59 DEBUG }, 18:43:59 DEBUG }, 18:43:59 DEBUG { 18:43:59 DEBUG }, 18:43:59 DEBUG 18:43:59 DEBUG "pod": "ds-cts-0" 18:43:59 DEBUG "value": [ 18:43:59 DEBUG { 18:43:59 DEBUG "metric": { 18:43:59 DEBUG "instance": "10.142.0.11:9100" 18:43:59 DEBUG }, 18:43:59 DEBUG "value": [ 18:43:59 DEBUG 18:43:59 DEBUG 18:43:59 DEBUG 18:43:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:43:59 DEBUG 18:43:59 DEBUG }, 18:43:59 DEBUG { 18:43:59 DEBUG "value": [ 18:43:59 DEBUG "value": [ 18:43:59 DEBUG 18:43:59 DEBUG "metric": { 18:43:59 DEBUG [...] 18:43:59 DEBUG }, 18:43:59 DEBUG }, 18:43:59 DEBUG 18:43:59 DEBUG 18:43:59 DEBUG [...] 18:43:59 DEBUG "metric": { 18:43:59 DEBUG "value": [ 18:43:59 DEBUG 18:43:59 DEBUG 18:43:59 DEBUG [...] 18:43:59 DEBUG 18:43:59 DEBUG "value": [ 18:43:59 DEBUG "value": [ 18:43:59 DEBUG 18:43:59 DEBUG 18:43:59 DEBUG "pod": "ds-cts-0" 18:43:59 DEBUG 18:43:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:43:59 DEBUG 18:43:59 DEBUG 18:43:59 DEBUG "value": [ 18:43:59 DEBUG 18:43:59 DEBUG 18:43:59 DEBUG 18:43:59 DEBUG 18:43:59 DEBUG }, 18:43:59 DEBUG 18:43:59 DEBUG }, 18:43:59 DEBUG 18:43:59 DEBUG 18:43:59 DEBUG 1734461039, 18:43:59 DEBUG 18:43:59 DEBUG 18:43:59 DEBUG 18:43:59 DEBUG 18:43:59 DEBUG "value": [ 18:43:59 DEBUG 18:43:59 DEBUG "value": [ 18:43:59 DEBUG 18:43:59 DEBUG "instance": "10.142.0.51:9100" 18:43:59 DEBUG "0" 18:43:59 DEBUG 18:43:59 DEBUG 18:43:59 DEBUG [...] 18:43:59 DEBUG "instance": "10.142.0.51:9100" 18:43:59 DEBUG 18:43:59 DEBUG [...] 18:43:59 DEBUG 18:43:59 DEBUG "instance": "10.142.0.51:9100" 18:43:59 DEBUG }, 18:43:59 DEBUG ] 18:43:59 DEBUG } 18:43:59 DEBUG ] 18:43:59 DEBUG } 18:43:59 DEBUG 18:43:59 DEBUG 18:43:59 DEBUG 18:43:59 DEBUG 18:43:59 DEBUG }, 18:43:59 DEBUG "value": [ 18:43:59 DEBUG 1734461039, 18:43:59 DEBUG "81508.16666666667" 18:43:59 DEBUG [...] 18:43:59 DEBUG } 18:43:59 DEBUG 18:43:59 DEBUG 18:43:59 DEBUG 18:43:59 DEBUG 18:43:59 DEBUG "value": [ 18:43:59 DEBUG [...] 18:43:59 DEBUG }, 18:43:59 DEBUG ] 18:43:59 DEBUG } 18:43:59 DEBUG 18:43:59 DEBUG "instance": "10.142.0.51:9100" 18:43:59 DEBUG 18:43:59 DEBUG [...] 18:43:59 DEBUG 1734461039, 18:43:59 DEBUG 18:43:59 DEBUG "value": [ 18:43:59 DEBUG 18:43:59 DEBUG ] 18:43:59 DEBUG } 18:43:59 DEBUG }, 18:43:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:43:59 DEBUG }, 18:43:59 DEBUG "value": [ 18:43:59 DEBUG 1734461039, 18:43:59 DEBUG 1734461039, 18:43:59 DEBUG 18:43:59 DEBUG [...] 18:43:59 DEBUG } 18:43:59 DEBUG "value": [ 18:43:59 DEBUG 18:43:59 DEBUG "0" 18:43:59 DEBUG 18:43:59 DEBUG "2611294208" 18:43:59 DEBUG "27852.8" 18:43:59 DEBUG 18:43:59 DEBUG 18:43:59 DEBUG 1734461039, 18:43:59 DEBUG 18:43:59 DEBUG ] 18:43:59 DEBUG 18:43:59 DEBUG ] 18:43:59 DEBUG ] 18:43:59 DEBUG "instance": "10.142.0.51:9100" 18:43:59 DEBUG 18:43:59 DEBUG "0.0005333333333306693" 18:43:59 DEBUG ] 18:43:59 DEBUG } 18:43:59 DEBUG ] 18:43:59 DEBUG } 18:43:59 DEBUG } 18:43:59 DEBUG }, 18:43:59 DEBUG 18:43:59 DEBUG 18:43:59 DEBUG } 18:43:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:43:59 DEBUG } 18:43:59 DEBUG } 18:43:59 DEBUG "value": [ 18:43:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:43:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:43:59 DEBUG ] 18:43:59 DEBUG }, 18:43:59 DEBUG ] 18:43:59 DEBUG ] 18:43:59 DEBUG 1734461039, 18:43:59 DEBUG }, 18:43:59 DEBUG }, 18:43:59 DEBUG } 18:43:59 DEBUG "value": [ 18:43:59 DEBUG } 18:43:59 DEBUG } 18:43:59 DEBUG "79916.33333333334" 18:43:59 DEBUG "value": [ 18:43:59 DEBUG "value": [ 18:43:59 DEBUG } 18:43:59 DEBUG 1734461039, 18:43:59 DEBUG } 18:43:59 DEBUG } 18:43:59 DEBUG ] 18:43:59 DEBUG 1734461039, 18:43:59 DEBUG 1734461039, 18:43:59 DEBUG "713330688" 18:43:59 DEBUG } 18:43:59 DEBUG "1691.336374879112" 18:43:59 DEBUG "1.7955347343225267" 18:43:59 DEBUG ] 18:43:59 DEBUG ] 18:43:59 DEBUG ] 18:43:59 DEBUG ] 18:43:59 DEBUG } 18:43:59 DEBUG } 18:43:59 DEBUG } 18:43:59 DEBUG } 18:43:59 DEBUG ] 18:43:59 DEBUG } 18:43:59 DEBUG ] 18:43:59 DEBUG ] 18:43:59 DEBUG } 18:43:59 DEBUG } 18:43:59 DEBUG } 18:43:59 DEBUG } 18:43:59 DEBUG } 18:43:59 DEBUG } 18:44:01 INFO 18:44:01 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:44:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:44:01 INFO [loop_until]: OK (rc = 0) 18:44:01 DEBUG --- stdout --- 18:44:01 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 543Mi ds-idrepo-0 12m 725Mi rcs-5d696ff5b-7ttvr 3m 299Mi rcs-5d696ff5b-wwlbr 3m 339Mi 18:44:01 DEBUG --- stderr --- 18:44:01 DEBUG 18:44:01 INFO 18:44:01 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:44:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:44:01 INFO [loop_until]: OK (rc = 0) 18:44:01 DEBUG --- stdout --- 18:44:01 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 111m 0% 2281Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1866m 5% 3617Mi 12% 18:44:01 DEBUG --- stderr --- 18:44:01 DEBUG 18:44:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461099" 18:44:59 INFO [http_cmd]: http status code OK 18:44:59 DEBUG --- status code --- 18:44:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:44:59 DEBUG --- http response --- 18:44:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:44:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461099" 18:44:59 INFO [http_cmd]: http status code OK 18:44:59 DEBUG --- status code --- 18:44:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:44:59 DEBUG --- http response --- 18:44:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:44:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734461099" 18:44:59 INFO [http_cmd]: http status code OK 18:44:59 DEBUG --- status code --- 18:44:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:44:59 DEBUG --- http response --- 18:44:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:44:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461099" 18:44:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461099" 18:44:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461099" 18:44:59 INFO [http_cmd]: http status code OK 18:44:59 DEBUG --- status code --- 18:44:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:44:59 DEBUG --- http response --- 18:44:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:44:59 INFO [http_cmd]: http status code OK 18:44:59 DEBUG --- status code --- 18:44:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:44:59 DEBUG --- http response --- 18:44:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:44:59 INFO [http_cmd]: http status code OK 18:44:59 DEBUG --- status code --- 18:44:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:44:59 DEBUG --- http response --- 18:44:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:44:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461099" 18:44:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461099" 18:44:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461099" 18:44:59 INFO [http_cmd]: http status code OK 18:44:59 DEBUG --- status code --- 18:44:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:44:59 DEBUG --- http response --- 18:44:59 INFO [http_cmd]: http status code OK 18:44:59 DEBUG --- status code --- 18:44:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:44:59 DEBUG --- http response --- 18:44:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:44:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:44:59 INFO [http_cmd]: http status code OK 18:44:59 DEBUG --- status code --- 18:44:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:44:59 DEBUG --- http response --- 18:44:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:44:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461099" 18:44:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461099" 18:44:59 INFO [http_cmd]: http status code OK 18:44:59 DEBUG --- status code --- 18:44:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:44:59 DEBUG --- http response --- 18:44:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:44:59 INFO [http_cmd]: http status code OK 18:44:59 DEBUG --- status code --- 18:44:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:44:59 DEBUG --- http response --- 18:44:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:44:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461099" 18:44:59 INFO [http_cmd]: http status code OK 18:44:59 DEBUG --- status code --- 18:44:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:44:59 DEBUG --- http response --- 18:44:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:44:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734461099" 18:44:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461099" 18:44:59 INFO [http_cmd]: http status code OK 18:44:59 DEBUG --- status code --- 18:44:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:44:59 ERROR [http_cmd]: ERROR 18:44:59 DEBUG --- http response --- 18:44:59 ERROR --- status code --- 18:44:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:44:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:44:59 ERROR --- http response --- 18:44:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:44:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:44:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461099" 18:44:59 INFO [http_cmd]: http status code OK 18:44:59 DEBUG --- status code --- 18:44:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:44:59 DEBUG --- http response --- 18:44:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:44:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461099" 18:44:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461099" 18:44:59 INFO [http_cmd]: http status code OK 18:44:59 DEBUG --- status code --- 18:44:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:44:59 DEBUG --- http response --- 18:44:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:44:59 INFO [http_cmd]: http status code OK 18:44:59 DEBUG --- status code --- 18:44:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:44:59 DEBUG --- http response --- 18:44:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:44:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461099" 18:44:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461099" 18:44:59 INFO [http_cmd]: http status code OK 18:44:59 DEBUG --- status code --- 18:44:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:44:59 DEBUG --- http response --- 18:44:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:44:59 INFO [http_cmd]: http status code OK 18:44:59 DEBUG --- status code --- 18:44:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:44:59 DEBUG --- http response --- 18:44:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:44:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461099" 18:44:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461099" 18:44:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734461099" 18:44:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734461099" 18:44:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734461099" 18:44:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461099" 18:44:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461099" 18:44:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461099" 18:44:59 INFO [http_cmd]: http status code OK 18:44:59 INFO [http_cmd]: http status code OK 18:44:59 DEBUG --- status code --- 18:44:59 DEBUG --- status code --- 18:44:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:44:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:44:59 DEBUG --- http response --- 18:44:59 DEBUG --- http response --- 18:44:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:44:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:44:59 INFO [http_cmd]: http status code OK 18:44:59 DEBUG --- status code --- 18:44:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:44:59 DEBUG --- http response --- 18:44:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:44:59 INFO [http_cmd]: http status code OK 18:44:59 DEBUG --- status code --- 18:44:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:44:59 DEBUG --- http response --- 18:44:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:44:59 INFO [http_cmd]: http status code OK 18:44:59 DEBUG --- status code --- 18:44:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:44:59 INFO [http_cmd]: http status code OK 18:44:59 DEBUG --- http response --- 18:44:59 DEBUG --- status code --- 18:44:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:44:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:44:59 INFO [http_cmd]: http status code OK 18:44:59 DEBUG --- http response --- 18:44:59 DEBUG --- status code --- 18:44:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:44:59 DEBUG --- http response --- 18:44:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:44:59 INFO [http_cmd]: http status code OK 18:44:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:44:59 DEBUG --- status code --- 18:44:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:44:59 DEBUG --- http response --- 18:44:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:44:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461099" 18:44:59 INFO [http_cmd]: http status code OK 18:44:59 DEBUG --- status code --- 18:44:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:44:59 DEBUG --- http response --- 18:44:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:44:59 DEBUG ----- output ----- 18:44:59 DEBUG { 18:44:59 DEBUG "status": "success", 18:44:59 DEBUG "data": { 18:44:59 DEBUG "resultType": "vector", 18:44:59 DEBUG "result": [ 18:44:59 DEBUG { 18:44:59 DEBUG "metric": { 18:44:59 DEBUG "pod": "ds-cts-0" 18:44:59 DEBUG }, 18:44:59 DEBUG "value": [ 18:44:59 DEBUG 18:44:59 DEBUG 18:44:59 DEBUG 18:44:59 DEBUG [...] 18:44:59 DEBUG 18:44:59 DEBUG 18:44:59 DEBUG 18:44:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:44:59 DEBUG }, 18:44:59 DEBUG "value": [ 18:44:59 DEBUG 1734461099, 18:44:59 DEBUG "0.00376747890246261" 18:44:59 DEBUG ] 18:44:59 DEBUG } 18:44:59 DEBUG ] 18:44:59 DEBUG } 18:44:59 DEBUG } 18:44:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461099" 18:44:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461099" 18:44:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461099" 18:44:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461099" 18:44:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734461099" 18:44:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461099" 18:44:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461099" 18:44:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461099" 18:44:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461099" 18:44:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734461099" 18:44:59 INFO [http_cmd]: http status code OK 18:44:59 DEBUG --- status code --- 18:44:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:44:59 DEBUG --- http response --- 18:44:59 INFO [http_cmd]: http status code OK 18:44:59 DEBUG --- status code --- 18:44:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:44:59 DEBUG --- http response --- 18:44:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:44:59 DEBUG ----- output ----- 18:44:59 DEBUG { 18:44:59 DEBUG "status": "success", 18:44:59 DEBUG "data": { 18:44:59 DEBUG "resultType": "vector", 18:44:59 DEBUG "result": [ 18:44:59 DEBUG { 18:44:59 DEBUG "metric": { 18:44:59 DEBUG "instance": "10.142.0.11:9100" 18:44:59 DEBUG }, 18:44:59 DEBUG "value": [ 18:44:59 DEBUG 18:44:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:44:59 DEBUG 18:44:59 INFO [http_cmd]: http status code OK 18:44:59 DEBUG ----- output ----- 18:44:59 DEBUG 18:44:59 DEBUG --- status code --- 18:44:59 DEBUG { 18:44:59 DEBUG [...] 18:44:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:44:59 INFO [http_cmd]: http status code OK 18:44:59 DEBUG "status": "success", 18:44:59 DEBUG 18:44:59 DEBUG --- http response --- 18:44:59 DEBUG --- status code --- 18:44:59 INFO [http_cmd]: http status code OK 18:44:59 DEBUG "data": { 18:44:59 INFO [http_cmd]: http status code OK 18:44:59 DEBUG 18:44:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:44:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:44:59 DEBUG --- status code --- 18:44:59 DEBUG "resultType": "vector", 18:44:59 DEBUG --- status code --- 18:44:59 DEBUG 18:44:59 DEBUG --- http response --- 18:44:59 INFO [http_cmd]: http status code OK 18:44:59 INFO [http_cmd]: http status code OK 18:44:59 DEBUG ----- output ----- 18:44:59 INFO [http_cmd]: http status code OK 18:44:59 INFO [http_cmd]: http status code OK 18:44:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:44:59 DEBUG "result": [ 18:44:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:44:59 DEBUG "instance": "10.142.0.51:9100" 18:44:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:44:59 DEBUG --- status code --- 18:44:59 DEBUG --- status code --- 18:44:59 DEBUG { 18:44:59 DEBUG --- status code --- 18:44:59 DEBUG --- status code --- 18:44:59 DEBUG --- http response --- 18:44:59 DEBUG { 18:44:59 DEBUG --- http response --- 18:44:59 DEBUG }, 18:44:59 DEBUG ----- output ----- 18:44:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:44:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:44:59 DEBUG "status": "success", 18:44:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:44:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:44:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:44:59 DEBUG "metric": { 18:44:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:44:59 DEBUG "value": [ 18:44:59 DEBUG { 18:44:59 DEBUG --- http response --- 18:44:59 DEBUG --- http response --- 18:44:59 DEBUG "data": { 18:44:59 DEBUG --- http response --- 18:44:59 DEBUG --- http response --- 18:44:59 DEBUG ----- output ----- 18:44:59 DEBUG "instance": "10.142.0.11:9100" 18:44:59 DEBUG ----- output ----- 18:44:59 DEBUG 1734461099, 18:44:59 DEBUG "status": "success", 18:44:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:44:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:44:59 DEBUG "resultType": "vector", 18:44:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:44:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:44:59 DEBUG { 18:44:59 DEBUG }, 18:44:59 DEBUG { 18:44:59 DEBUG "82686.43333333333" 18:44:59 DEBUG "data": { 18:44:59 DEBUG ----- output ----- 18:44:59 DEBUG ----- output ----- 18:44:59 DEBUG "result": [ 18:44:59 DEBUG ----- output ----- 18:44:59 DEBUG ----- output ----- 18:44:59 DEBUG "status": "success", 18:44:59 DEBUG "value": [ 18:44:59 DEBUG "status": "success", 18:44:59 DEBUG ] 18:44:59 DEBUG "resultType": "vector", 18:44:59 DEBUG { 18:44:59 DEBUG { 18:44:59 DEBUG { 18:44:59 DEBUG { 18:44:59 DEBUG { 18:44:59 DEBUG "data": { 18:44:59 DEBUG 18:44:59 DEBUG "data": { 18:44:59 DEBUG } 18:44:59 DEBUG "result": [ 18:44:59 DEBUG "status": "success", 18:44:59 DEBUG "status": "success", 18:44:59 DEBUG "metric": { 18:44:59 DEBUG "status": "success", 18:44:59 DEBUG "status": "success", 18:44:59 DEBUG "resultType": "vector", 18:44:59 DEBUG 18:44:59 DEBUG "resultType": "vector", 18:44:59 DEBUG ] 18:44:59 DEBUG { 18:44:59 DEBUG "data": { 18:44:59 DEBUG "data": { 18:44:59 DEBUG "pod": "ds-cts-0" 18:44:59 DEBUG "data": { 18:44:59 DEBUG "data": { 18:44:59 DEBUG "result": [ 18:44:59 DEBUG 18:44:59 DEBUG "result": [ 18:44:59 DEBUG } 18:44:59 DEBUG "metric": { 18:44:59 DEBUG "resultType": "vector", 18:44:59 DEBUG "resultType": "vector", 18:44:59 DEBUG }, 18:44:59 DEBUG "resultType": "vector", 18:44:59 DEBUG "resultType": "vector", 18:44:59 DEBUG { 18:44:59 DEBUG [...] 18:44:59 DEBUG { 18:44:59 DEBUG } 18:44:59 DEBUG "instance": "10.142.0.11:9100" 18:44:59 DEBUG "result": [ 18:44:59 DEBUG "result": [ 18:44:59 DEBUG "value": [ 18:44:59 DEBUG "result": [ 18:44:59 DEBUG "result": [ 18:44:59 DEBUG "metric": { 18:44:59 DEBUG 18:44:59 DEBUG "metric": { 18:44:59 DEBUG }, 18:44:59 DEBUG { 18:44:59 DEBUG { 18:44:59 DEBUG 18:44:59 DEBUG { 18:44:59 DEBUG { 18:44:59 DEBUG "instance": "10.142.0.11:9100" 18:44:59 DEBUG 18:44:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:44:59 DEBUG "value": [ 18:44:59 DEBUG "metric": { 18:44:59 DEBUG "metric": { 18:44:59 DEBUG 18:44:59 DEBUG "metric": { 18:44:59 DEBUG "metric": { 18:44:59 DEBUG }, 18:44:59 DEBUG 18:44:59 DEBUG }, 18:44:59 DEBUG 18:44:59 DEBUG "instance": "10.142.0.11:9100" 18:44:59 DEBUG "pod": "ds-cts-0" 18:44:59 DEBUG 18:44:59 DEBUG "pod": "ds-cts-0" 18:44:59 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:44:59 DEBUG "value": [ 18:44:59 DEBUG "instance": "10.142.0.51:9100" 18:44:59 DEBUG "value": [ 18:44:59 DEBUG 18:44:59 DEBUG }, 18:44:59 DEBUG }, 18:44:59 DEBUG [...] 18:44:59 DEBUG }, 18:44:59 DEBUG }, 18:44:59 DEBUG 18:44:59 DEBUG }, 18:44:59 DEBUG 18:44:59 DEBUG 18:44:59 DEBUG "value": [ 18:44:59 DEBUG "value": [ 18:44:59 DEBUG 18:44:59 DEBUG "value": [ 18:44:59 DEBUG "value": [ 18:44:59 DEBUG 18:44:59 DEBUG "value": [ 18:44:59 DEBUG 18:44:59 DEBUG [...] 18:44:59 DEBUG 18:44:59 DEBUG 18:44:59 DEBUG 18:44:59 DEBUG 18:44:59 DEBUG 18:44:59 DEBUG 18:44:59 DEBUG 1734461099, 18:44:59 DEBUG 18:44:59 DEBUG 18:44:59 DEBUG 18:44:59 DEBUG 18:44:59 DEBUG 18:44:59 DEBUG 18:44:59 DEBUG 18:44:59 DEBUG [...] 18:44:59 DEBUG "0.00043333333333303397" 18:44:59 DEBUG [...] 18:44:59 DEBUG 18:44:59 DEBUG 18:44:59 DEBUG 18:44:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:44:59 DEBUG 18:44:59 DEBUG 18:44:59 DEBUG 18:44:59 DEBUG ] 18:44:59 DEBUG 18:44:59 DEBUG 18:44:59 DEBUG [...] 18:44:59 DEBUG [...] 18:44:59 DEBUG }, 18:44:59 DEBUG [...] 18:44:59 DEBUG [...] 18:44:59 DEBUG 18:44:59 DEBUG } 18:44:59 DEBUG 18:44:59 DEBUG "instance": "10.142.0.51:9100" 18:44:59 DEBUG 18:44:59 DEBUG 18:44:59 DEBUG 18:44:59 DEBUG 18:44:59 DEBUG 18:44:59 DEBUG 18:44:59 DEBUG ] 18:44:59 DEBUG 18:44:59 DEBUG }, 18:44:59 DEBUG 18:44:59 DEBUG "value": [ 18:44:59 DEBUG 18:44:59 DEBUG 18:44:59 DEBUG 18:44:59 DEBUG "instance": "10.142.0.51:9100" 18:44:59 DEBUG } 18:44:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:44:59 DEBUG "value": [ 18:44:59 DEBUG 1734461099, 18:44:59 DEBUG "18568.533333333333" 18:44:59 DEBUG 1734461099, 18:44:59 DEBUG 18:44:59 DEBUG 18:44:59 DEBUG }, 18:44:59 DEBUG } 18:44:59 DEBUG }, 18:44:59 DEBUG 18:44:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:44:59 DEBUG ] 18:44:59 DEBUG "0" 18:44:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:44:59 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:44:59 DEBUG "value": [ 18:44:59 DEBUG "value": [ 18:44:59 DEBUG "instance": "10.142.0.51:9100" 18:44:59 DEBUG }, 18:44:59 DEBUG } 18:44:59 DEBUG ] 18:44:59 DEBUG }, 18:44:59 DEBUG }, 18:44:59 DEBUG 1734461099, 18:44:59 DEBUG 1734461099, 18:44:59 DEBUG }, 18:44:59 DEBUG "value": [ 18:44:59 DEBUG ] 18:44:59 DEBUG } 18:44:59 DEBUG "value": [ 18:44:59 DEBUG "value": [ 18:44:59 DEBUG "0.0003333333333333854" 18:44:59 DEBUG "2590609408" 18:44:59 DEBUG "value": [ 18:44:59 DEBUG 1734461099, 18:44:59 DEBUG } 18:44:59 DEBUG ] 18:44:59 DEBUG 1734461099, 18:44:59 DEBUG 1734461099, 18:44:59 DEBUG ] 18:44:59 DEBUG } 18:44:59 DEBUG ] 18:44:59 DEBUG "713371648" 18:44:59 DEBUG } 18:44:59 DEBUG } 18:44:59 DEBUG "1306.5905278624678" 18:44:59 DEBUG "1.7800249135331365" 18:44:59 DEBUG ] 18:44:59 DEBUG 1734461099, 18:44:59 DEBUG } 18:44:59 DEBUG ] 18:44:59 DEBUG } 18:44:59 DEBUG ] 18:44:59 DEBUG ] 18:44:59 DEBUG } 18:44:59 DEBUG "81092.03333333333" 18:44:59 DEBUG } 18:44:59 DEBUG } 18:44:59 DEBUG } 18:44:59 DEBUG } 18:44:59 DEBUG ] 18:44:59 DEBUG ] 18:44:59 DEBUG ] 18:44:59 DEBUG ] 18:44:59 DEBUG ] 18:44:59 DEBUG } 18:44:59 DEBUG } 18:44:59 DEBUG } 18:44:59 DEBUG } 18:44:59 DEBUG } 18:44:59 DEBUG } 18:44:59 DEBUG ] 18:44:59 DEBUG } 18:44:59 DEBUG } 18:44:59 DEBUG } 18:44:59 DEBUG } 18:44:59 DEBUG } 18:45:01 INFO 18:45:01 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:45:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:45:01 INFO [loop_until]: OK (rc = 0) 18:45:01 DEBUG --- stdout --- 18:45:01 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 5m 543Mi ds-idrepo-0 7m 725Mi rcs-5d696ff5b-7ttvr 4m 299Mi rcs-5d696ff5b-wwlbr 2m 339Mi 18:45:01 DEBUG --- stderr --- 18:45:01 DEBUG 18:45:01 INFO 18:45:01 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:45:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:45:01 INFO [loop_until]: OK (rc = 0) 18:45:01 DEBUG --- stdout --- 18:45:01 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 110m 0% 2291Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1849m 5% 3615Mi 12% 18:45:01 DEBUG --- stderr --- 18:45:01 DEBUG 18:45:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461159" 18:45:59 INFO [http_cmd]: http status code OK 18:45:59 DEBUG --- status code --- 18:45:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:45:59 DEBUG --- http response --- 18:45:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:45:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461159" 18:45:59 INFO [http_cmd]: http status code OK 18:45:59 DEBUG --- status code --- 18:45:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:45:59 DEBUG --- http response --- 18:45:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:45:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734461159" 18:45:59 INFO [http_cmd]: http status code OK 18:45:59 DEBUG --- status code --- 18:45:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:45:59 DEBUG --- http response --- 18:45:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:45:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461159" 18:45:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461159" 18:45:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461159" 18:45:59 INFO [http_cmd]: http status code OK 18:45:59 DEBUG --- status code --- 18:45:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:45:59 DEBUG --- http response --- 18:45:59 INFO [http_cmd]: http status code OK 18:45:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:45:59 DEBUG --- status code --- 18:45:59 INFO [http_cmd]: http status code OK 18:45:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:45:59 DEBUG --- status code --- 18:45:59 DEBUG --- http response --- 18:45:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:45:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:45:59 DEBUG --- http response --- 18:45:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:45:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461159" 18:45:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461159" 18:45:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461159" 18:45:59 INFO [http_cmd]: http status code OK 18:45:59 DEBUG --- status code --- 18:45:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:45:59 DEBUG --- http response --- 18:45:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:45:59 INFO [http_cmd]: http status code OK 18:45:59 DEBUG --- status code --- 18:45:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:45:59 INFO [http_cmd]: http status code OK 18:45:59 DEBUG --- http response --- 18:45:59 DEBUG --- status code --- 18:45:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:45:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:45:59 DEBUG --- http response --- 18:45:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:45:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461159" 18:45:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461159" 18:45:59 INFO [http_cmd]: http status code OK 18:45:59 DEBUG --- status code --- 18:45:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:45:59 DEBUG --- http response --- 18:45:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:45:59 INFO [http_cmd]: http status code OK 18:45:59 DEBUG --- status code --- 18:45:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:45:59 DEBUG --- http response --- 18:45:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:45:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461159" 18:45:59 INFO [http_cmd]: http status code OK 18:45:59 DEBUG --- status code --- 18:45:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:45:59 DEBUG --- http response --- 18:45:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:45:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734461159" 18:45:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461159" 18:45:59 INFO [http_cmd]: http status code OK 18:45:59 ERROR [http_cmd]: ERROR 18:45:59 DEBUG --- status code --- 18:45:59 ERROR --- status code --- 18:45:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:45:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:45:59 DEBUG --- http response --- 18:45:59 ERROR --- http response --- 18:45:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:45:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:45:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:45:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461159" 18:45:59 INFO [http_cmd]: http status code OK 18:45:59 DEBUG --- status code --- 18:45:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:45:59 DEBUG --- http response --- 18:45:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:45:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461159" 18:45:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461159" 18:45:59 INFO [http_cmd]: http status code OK 18:45:59 DEBUG --- status code --- 18:45:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:45:59 DEBUG --- http response --- 18:45:59 INFO [http_cmd]: http status code OK 18:45:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:45:59 DEBUG --- status code --- 18:45:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:45:59 DEBUG --- http response --- 18:45:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:45:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461159" 18:45:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461159" 18:45:59 INFO [http_cmd]: http status code OK 18:45:59 DEBUG --- status code --- 18:45:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:45:59 INFO [http_cmd]: http status code OK 18:45:59 DEBUG --- http response --- 18:45:59 DEBUG --- status code --- 18:45:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:45:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:45:59 DEBUG --- http response --- 18:45:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:45:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734461159" 18:45:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461159" 18:45:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734461159" 18:45:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461159" 18:45:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461159" 18:45:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734461159" 18:45:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461159" 18:45:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461159" 18:45:59 INFO [http_cmd]: http status code OK 18:45:59 DEBUG --- status code --- 18:45:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:45:59 DEBUG --- http response --- 18:45:59 INFO [http_cmd]: http status code OK 18:45:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:45:59 DEBUG --- status code --- 18:45:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:45:59 DEBUG --- http response --- 18:45:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:45:59 INFO [http_cmd]: http status code OK 18:45:59 DEBUG --- status code --- 18:45:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:45:59 DEBUG --- http response --- 18:45:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:45:59 INFO [http_cmd]: http status code OK 18:45:59 INFO [http_cmd]: http status code OK 18:45:59 DEBUG --- status code --- 18:45:59 DEBUG --- status code --- 18:45:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:45:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:45:59 DEBUG --- http response --- 18:45:59 DEBUG --- http response --- 18:45:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:45:59 INFO [http_cmd]: http status code OK 18:45:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:45:59 DEBUG --- status code --- 18:45:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:45:59 DEBUG --- http response --- 18:45:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:45:59 INFO [http_cmd]: http status code OK 18:45:59 DEBUG --- status code --- 18:45:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:45:59 DEBUG --- http response --- 18:45:59 INFO [http_cmd]: http status code OK 18:45:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:45:59 DEBUG --- status code --- 18:45:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:45:59 DEBUG --- http response --- 18:45:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:45:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461159" 18:45:59 INFO [http_cmd]: http status code OK 18:45:59 DEBUG --- status code --- 18:45:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:45:59 DEBUG --- http response --- 18:45:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:45:59 DEBUG ----- output ----- 18:45:59 DEBUG { 18:45:59 DEBUG "status": "success", 18:45:59 DEBUG "data": { 18:45:59 DEBUG "resultType": "vector", 18:45:59 DEBUG "result": [ 18:45:59 DEBUG { 18:45:59 DEBUG "metric": { 18:45:59 DEBUG "pod": "ds-cts-0" 18:45:59 DEBUG }, 18:45:59 DEBUG "value": [ 18:45:59 DEBUG 18:45:59 DEBUG 18:45:59 DEBUG 18:45:59 DEBUG [...] 18:45:59 DEBUG 18:45:59 DEBUG 18:45:59 DEBUG 18:45:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:45:59 DEBUG }, 18:45:59 DEBUG "value": [ 18:45:59 DEBUG 1734461159, 18:45:59 DEBUG "0.004134262162625469" 18:45:59 DEBUG ] 18:45:59 DEBUG } 18:45:59 DEBUG ] 18:45:59 DEBUG } 18:45:59 DEBUG } 18:45:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461159" 18:45:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461159" 18:45:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461159" 18:45:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461159" 18:45:59 INFO [http_cmd]: http status code OK 18:45:59 DEBUG --- status code --- 18:45:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:45:59 DEBUG --- http response --- 18:45:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461159" 18:45:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:45:59 DEBUG ----- output ----- 18:45:59 DEBUG { 18:45:59 DEBUG "status": "success", 18:45:59 DEBUG "data": { 18:45:59 DEBUG "resultType": "vector", 18:45:59 DEBUG "result": [ 18:45:59 DEBUG { 18:45:59 DEBUG "metric": { 18:45:59 DEBUG "instance": "10.142.0.11:9100" 18:45:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734461159" 18:45:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461159" 18:45:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461159" 18:45:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734461159" 18:45:59 DEBUG }, 18:46:00 DEBUG "value": [ 18:45:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461159" 18:46:00 DEBUG 18:46:00 DEBUG 18:46:00 DEBUG 18:46:00 DEBUG [...] 18:46:00 DEBUG 18:46:00 DEBUG 18:46:00 DEBUG 18:46:00 DEBUG "instance": "10.142.0.51:9100" 18:46:00 DEBUG }, 18:46:00 DEBUG "value": [ 18:46:00 DEBUG 1734461159, 18:46:00 DEBUG "82978.86666666667" 18:46:00 DEBUG ] 18:46:00 DEBUG } 18:46:00 DEBUG ] 18:46:00 DEBUG } 18:46:00 DEBUG } 18:46:00 INFO [http_cmd]: http status code OK 18:46:00 DEBUG --- status code --- 18:46:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:46:00 DEBUG --- http response --- 18:46:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:46:00 DEBUG ----- output ----- 18:46:00 DEBUG { 18:46:00 DEBUG "status": "success", 18:46:00 DEBUG "data": { 18:46:00 DEBUG "resultType": "vector", 18:46:00 DEBUG "result": [ 18:46:00 INFO [http_cmd]: http status code OK 18:46:00 DEBUG { 18:46:00 INFO [http_cmd]: http status code OK 18:46:00 DEBUG --- status code --- 18:46:00 DEBUG "metric": { 18:46:00 DEBUG --- status code --- 18:46:00 INFO [http_cmd]: http status code OK 18:46:00 INFO [http_cmd]: http status code OK 18:46:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:46:00 INFO [http_cmd]: http status code OK 18:46:00 DEBUG "instance": "10.142.0.11:9100" 18:46:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:46:00 DEBUG --- status code --- 18:46:00 DEBUG --- status code --- 18:46:00 DEBUG --- http response --- 18:46:00 DEBUG --- status code --- 18:46:00 DEBUG }, 18:46:00 DEBUG --- http response --- 18:46:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:46:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:46:00 INFO [http_cmd]: http status code OK 18:46:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:46:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:46:00 INFO [http_cmd]: http status code OK 18:46:00 DEBUG "value": [ 18:46:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:46:00 DEBUG --- http response --- 18:46:00 DEBUG --- http response --- 18:46:00 DEBUG --- status code --- 18:46:00 DEBUG ----- output ----- 18:46:00 DEBUG --- http response --- 18:46:00 DEBUG --- status code --- 18:46:00 DEBUG 18:46:00 DEBUG ----- output ----- 18:46:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:46:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:46:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:46:00 DEBUG { 18:46:00 INFO [http_cmd]: http status code OK 18:46:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:46:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:46:00 DEBUG 18:46:00 DEBUG { 18:46:00 DEBUG ----- output ----- 18:46:00 DEBUG ----- output ----- 18:46:00 DEBUG --- http response --- 18:46:00 DEBUG "status": "success", 18:46:00 DEBUG --- status code --- 18:46:00 DEBUG --- http response --- 18:46:00 DEBUG ----- output ----- 18:46:00 DEBUG 18:46:00 DEBUG "status": "success", 18:46:00 DEBUG { 18:46:00 DEBUG { 18:46:00 DEBUG "data": { 18:46:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:46:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:46:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:46:00 DEBUG { 18:46:00 DEBUG [...] 18:46:00 DEBUG "data": { 18:46:00 DEBUG "status": "success", 18:46:00 DEBUG "status": "success", 18:46:00 DEBUG "resultType": "vector", 18:46:00 DEBUG ----- output ----- 18:46:00 DEBUG --- http response --- 18:46:00 DEBUG ----- output ----- 18:46:00 DEBUG "status": "success", 18:46:00 DEBUG 18:46:00 DEBUG "resultType": "vector", 18:46:00 DEBUG "data": { 18:46:00 DEBUG "data": { 18:46:00 DEBUG "result": [ 18:46:00 DEBUG { 18:46:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:46:00 DEBUG { 18:46:00 DEBUG "data": { 18:46:00 DEBUG 18:46:00 DEBUG "result": [ 18:46:00 DEBUG "resultType": "vector", 18:46:00 DEBUG "resultType": "vector", 18:46:00 DEBUG { 18:46:00 DEBUG "status": "success", 18:46:00 DEBUG ----- output ----- 18:46:00 DEBUG "status": "success", 18:46:00 DEBUG "resultType": "vector", 18:46:00 DEBUG 18:46:00 DEBUG { 18:46:00 DEBUG "result": [ 18:46:00 DEBUG "result": [ 18:46:00 DEBUG "metric": { 18:46:00 DEBUG "data": { 18:46:00 DEBUG { 18:46:00 DEBUG "data": { 18:46:00 DEBUG "result": [ 18:46:00 DEBUG "instance": "10.142.0.51:9100" 18:46:00 DEBUG "metric": { 18:46:00 DEBUG { 18:46:00 DEBUG { 18:46:00 DEBUG "instance": "10.142.0.11:9100" 18:46:00 DEBUG "resultType": "vector", 18:46:00 DEBUG "status": "success", 18:46:00 DEBUG "resultType": "vector", 18:46:00 DEBUG { 18:46:00 DEBUG }, 18:46:00 DEBUG "pod": "ds-cts-0" 18:46:00 DEBUG "metric": { 18:46:00 DEBUG "metric": { 18:46:00 DEBUG }, 18:46:00 DEBUG "result": [ 18:46:00 DEBUG "data": { 18:46:00 DEBUG "result": [ 18:46:00 DEBUG "metric": { 18:46:00 DEBUG "value": [ 18:46:00 DEBUG }, 18:46:00 DEBUG "pod": "ds-cts-0" 18:46:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:46:00 DEBUG }, 18:46:00 DEBUG { 18:46:00 DEBUG "resultType": "vector", 18:46:00 DEBUG { 18:46:00 DEBUG "instance": "10.142.0.11:9100" 18:46:00 DEBUG 1734461159, 18:46:00 DEBUG "value": [ 18:46:00 DEBUG }, 18:46:00 DEBUG "value": [ 18:46:00 DEBUG "value": [ 18:46:00 DEBUG "metric": { 18:46:00 DEBUG "result": [ 18:46:00 DEBUG "metric": { 18:46:00 DEBUG }, 18:46:00 DEBUG "0.00043333333333303397" 18:46:00 DEBUG 18:46:00 DEBUG "value": [ 18:46:00 DEBUG 18:46:00 DEBUG 18:46:00 DEBUG "pod": "ds-cts-0" 18:46:00 DEBUG { 18:46:00 DEBUG "instance": "10.142.0.11:9100" 18:46:00 DEBUG "value": [ 18:46:00 DEBUG ] 18:46:00 DEBUG 18:46:00 DEBUG 18:46:00 DEBUG 18:46:00 DEBUG 18:46:00 DEBUG [...] 18:46:00 DEBUG "metric": { 18:46:00 DEBUG }, 18:46:00 DEBUG 18:46:00 DEBUG } 18:46:00 DEBUG 18:46:00 DEBUG 18:46:00 DEBUG 18:46:00 DEBUG }, 18:46:00 DEBUG 18:46:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:46:00 DEBUG "value": [ 18:46:00 DEBUG 18:46:00 DEBUG ] 18:46:00 DEBUG [...] 18:46:00 DEBUG 18:46:00 DEBUG 18:46:00 DEBUG "value": [ 18:46:00 DEBUG 18:46:00 DEBUG }, 18:46:00 DEBUG 18:46:00 DEBUG 18:46:00 DEBUG [...] 18:46:00 DEBUG 18:46:00 DEBUG [...] 18:46:00 DEBUG [...] 18:46:00 DEBUG 18:46:00 DEBUG 18:46:00 DEBUG "value": [ 18:46:00 DEBUG 18:46:00 DEBUG } 18:46:00 DEBUG 18:46:00 DEBUG 18:46:00 DEBUG 18:46:00 DEBUG 18:46:00 DEBUG 18:46:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:46:00 DEBUG 18:46:00 DEBUG 18:46:00 DEBUG } 18:46:00 DEBUG 18:46:00 DEBUG 18:46:00 DEBUG 18:46:00 DEBUG 18:46:00 DEBUG 18:46:00 DEBUG }, 18:46:00 DEBUG 18:46:00 DEBUG [...] 18:46:00 DEBUG 18:46:00 DEBUG "instance": "10.142.0.51:9100" 18:46:00 DEBUG 18:46:00 DEBUG 18:46:00 DEBUG [...] 18:46:00 DEBUG "value": [ 18:46:00 DEBUG 18:46:00 DEBUG 18:46:00 DEBUG 18:46:00 DEBUG }, 18:46:00 DEBUG "instance": "10.142.0.51:9100" 18:46:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:46:00 DEBUG 18:46:00 DEBUG 1734461159, 18:46:00 DEBUG [...] 18:46:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:46:00 DEBUG 18:46:00 DEBUG "value": [ 18:46:00 DEBUG }, 18:46:00 DEBUG }, 18:46:00 DEBUG 18:46:00 DEBUG "713576448" 18:46:00 DEBUG 18:46:00 DEBUG }, 18:46:00 DEBUG "instance": "10.142.0.51:9100" 18:46:00 DEBUG 1734461159, 18:46:00 DEBUG "value": [ 18:46:00 DEBUG "value": [ 18:46:00 DEBUG 18:46:00 DEBUG ] 18:46:00 DEBUG 18:46:00 DEBUG "value": [ 18:46:00 DEBUG }, 18:46:00 DEBUG "0.000333333333333267" 18:46:00 DEBUG 1734461159, 18:46:00 DEBUG 1734461159, 18:46:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:46:00 DEBUG } 18:46:00 DEBUG 18:46:00 DEBUG 1734461159, 18:46:00 DEBUG "value": [ 18:46:00 DEBUG ] 18:46:00 DEBUG "24576" 18:46:00 DEBUG "2597408768" 18:46:00 DEBUG }, 18:46:00 DEBUG ] 18:46:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:46:00 DEBUG "0" 18:46:00 DEBUG 1734461159, 18:46:00 DEBUG } 18:46:00 DEBUG ] 18:46:00 DEBUG ] 18:46:00 DEBUG "value": [ 18:46:00 DEBUG } 18:46:00 DEBUG }, 18:46:00 DEBUG ] 18:46:00 DEBUG "79167.73333333332" 18:46:00 DEBUG ] 18:46:00 DEBUG } 18:46:00 DEBUG } 18:46:00 DEBUG 1734461159, 18:46:00 DEBUG } 18:46:00 DEBUG "value": [ 18:46:00 DEBUG } 18:46:00 DEBUG ] 18:46:00 DEBUG } 18:46:00 DEBUG } 18:46:00 DEBUG ] 18:46:00 DEBUG "1734.9489819639248" 18:46:00 DEBUG 1734461159, 18:46:00 DEBUG "1.785592991427936" 18:46:00 DEBUG } 18:46:00 DEBUG ] 18:46:00 DEBUG } 18:46:00 DEBUG ] 18:46:00 DEBUG ] 18:46:00 DEBUG ] 18:46:00 DEBUG ] 18:46:00 DEBUG } 18:46:00 DEBUG } 18:46:00 DEBUG } 18:46:00 DEBUG } 18:46:00 DEBUG } 18:46:00 DEBUG } 18:46:00 DEBUG } 18:46:00 DEBUG ] 18:46:00 DEBUG } 18:46:00 DEBUG ] 18:46:00 DEBUG } 18:46:00 DEBUG } 18:46:00 DEBUG } 18:46:00 DEBUG } 18:46:00 DEBUG } 18:46:01 INFO 18:46:01 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:46:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:46:01 INFO [loop_until]: OK (rc = 0) 18:46:01 DEBUG --- stdout --- 18:46:01 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 543Mi ds-idrepo-0 7m 726Mi rcs-5d696ff5b-7ttvr 4m 299Mi rcs-5d696ff5b-wwlbr 3m 339Mi 18:46:01 DEBUG --- stderr --- 18:46:01 DEBUG 18:46:01 INFO 18:46:01 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:46:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:46:02 INFO [loop_until]: OK (rc = 0) 18:46:02 DEBUG --- stdout --- 18:46:02 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 92m 0% 2289Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1856m 5% 3620Mi 12% 18:46:02 DEBUG --- stderr --- 18:46:02 DEBUG 18:46:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461219" 18:46:59 INFO [http_cmd]: http status code OK 18:46:59 DEBUG --- status code --- 18:46:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:46:59 DEBUG --- http response --- 18:46:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:46:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461219" 18:46:59 INFO [http_cmd]: http status code OK 18:46:59 DEBUG --- status code --- 18:46:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:46:59 DEBUG --- http response --- 18:46:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:46:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734461219" 18:46:59 INFO [http_cmd]: http status code OK 18:46:59 DEBUG --- status code --- 18:46:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:46:59 DEBUG --- http response --- 18:46:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:46:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461219" 18:46:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461219" 18:46:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461219" 18:46:59 INFO [http_cmd]: http status code OK 18:46:59 INFO [http_cmd]: http status code OK 18:46:59 DEBUG --- status code --- 18:46:59 DEBUG --- status code --- 18:46:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:46:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:46:59 DEBUG --- http response --- 18:46:59 DEBUG --- http response --- 18:46:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:46:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:46:59 INFO [http_cmd]: http status code OK 18:46:59 DEBUG --- status code --- 18:46:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:46:59 DEBUG --- http response --- 18:46:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:46:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461219" 18:46:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461219" 18:46:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461219" 18:46:59 INFO [http_cmd]: http status code OK 18:46:59 INFO [http_cmd]: http status code OK 18:46:59 DEBUG --- status code --- 18:46:59 INFO [http_cmd]: http status code OK 18:46:59 DEBUG --- status code --- 18:46:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:46:59 DEBUG --- status code --- 18:46:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:46:59 DEBUG --- http response --- 18:46:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:46:59 DEBUG --- http response --- 18:46:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:46:59 DEBUG --- http response --- 18:46:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:46:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:46:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461219" 18:46:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461219" 18:46:59 INFO [http_cmd]: http status code OK 18:46:59 DEBUG --- status code --- 18:46:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:46:59 DEBUG --- http response --- 18:46:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:46:59 INFO [http_cmd]: http status code OK 18:46:59 DEBUG --- status code --- 18:46:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:46:59 DEBUG --- http response --- 18:46:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:46:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461219" 18:46:59 INFO [http_cmd]: http status code OK 18:46:59 DEBUG --- status code --- 18:46:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:46:59 DEBUG --- http response --- 18:46:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:46:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734461219" 18:46:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461219" 18:46:59 INFO [http_cmd]: http status code OK 18:46:59 ERROR [http_cmd]: ERROR 18:46:59 DEBUG --- status code --- 18:46:59 ERROR --- status code --- 18:46:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:46:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:46:59 DEBUG --- http response --- 18:46:59 ERROR --- http response --- 18:46:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:46:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:46:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:46:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461219" 18:46:59 INFO [http_cmd]: http status code OK 18:46:59 DEBUG --- status code --- 18:46:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:46:59 DEBUG --- http response --- 18:46:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:46:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461219" 18:46:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461219" 18:46:59 INFO [http_cmd]: http status code OK 18:46:59 DEBUG --- status code --- 18:46:59 INFO [http_cmd]: http status code OK 18:46:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:46:59 DEBUG --- status code --- 18:46:59 DEBUG --- http response --- 18:46:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:46:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:46:59 DEBUG --- http response --- 18:46:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:46:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461219" 18:46:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461219" 18:46:59 INFO [http_cmd]: http status code OK 18:46:59 INFO [http_cmd]: http status code OK 18:46:59 DEBUG --- status code --- 18:46:59 DEBUG --- status code --- 18:46:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:46:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:46:59 DEBUG --- http response --- 18:46:59 DEBUG --- http response --- 18:46:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:46:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:46:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734461219" 18:46:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734461219" 18:46:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461219" 18:46:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461219" 18:46:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461219" 18:46:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461219" 18:46:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734461219" 18:46:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461219" 18:46:59 INFO [http_cmd]: http status code OK 18:46:59 DEBUG --- status code --- 18:46:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:46:59 INFO [http_cmd]: http status code OK 18:46:59 INFO [http_cmd]: http status code OK 18:46:59 DEBUG --- http response --- 18:46:59 DEBUG --- status code --- 18:46:59 DEBUG --- status code --- 18:46:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:46:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:46:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:46:59 INFO [http_cmd]: http status code OK 18:46:59 INFO [http_cmd]: http status code OK 18:46:59 DEBUG --- http response --- 18:46:59 DEBUG --- http response --- 18:46:59 DEBUG --- status code --- 18:46:59 DEBUG --- status code --- 18:46:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:46:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:46:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:46:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:46:59 DEBUG --- http response --- 18:46:59 DEBUG --- http response --- 18:46:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:46:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:46:59 INFO [http_cmd]: http status code OK 18:46:59 DEBUG --- status code --- 18:46:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:46:59 DEBUG --- http response --- 18:46:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:46:59 INFO [http_cmd]: http status code OK 18:46:59 DEBUG --- status code --- 18:46:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:46:59 DEBUG --- http response --- 18:46:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:46:59 INFO [http_cmd]: http status code OK 18:46:59 DEBUG --- status code --- 18:46:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:46:59 DEBUG --- http response --- 18:46:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:46:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461219" 18:46:59 INFO [http_cmd]: http status code OK 18:46:59 DEBUG --- status code --- 18:46:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:46:59 DEBUG --- http response --- 18:46:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:46:59 DEBUG ----- output ----- 18:46:59 DEBUG { 18:46:59 DEBUG "status": "success", 18:46:59 DEBUG "data": { 18:46:59 DEBUG "resultType": "vector", 18:46:59 DEBUG "result": [ 18:46:59 DEBUG { 18:46:59 DEBUG "metric": { 18:46:59 DEBUG "pod": "ds-cts-0" 18:46:59 DEBUG }, 18:46:59 DEBUG "value": [ 18:46:59 DEBUG 18:46:59 DEBUG 18:46:59 DEBUG 18:46:59 DEBUG [...] 18:46:59 DEBUG 18:46:59 DEBUG 18:46:59 DEBUG 18:46:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:46:59 DEBUG }, 18:46:59 DEBUG "value": [ 18:46:59 DEBUG 1734461219, 18:46:59 DEBUG "0.018193901573118253" 18:46:59 DEBUG ] 18:46:59 DEBUG } 18:46:59 DEBUG ] 18:46:59 DEBUG } 18:46:59 DEBUG } 18:47:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461220" 18:47:00 INFO [http_cmd]: http status code OK 18:47:00 DEBUG --- status code --- 18:47:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:47:00 DEBUG --- http response --- 18:47:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:47:00 DEBUG ----- output ----- 18:47:00 DEBUG { 18:47:00 DEBUG "status": "success", 18:47:00 DEBUG "data": { 18:47:00 DEBUG "resultType": "vector", 18:47:00 DEBUG "result": [ 18:47:00 DEBUG { 18:47:00 DEBUG "metric": { 18:47:00 DEBUG "instance": "10.142.0.11:9100" 18:47:00 DEBUG }, 18:47:00 DEBUG "value": [ 18:47:00 DEBUG 18:47:00 DEBUG 18:47:00 DEBUG 18:47:00 DEBUG [...] 18:47:00 DEBUG 18:47:00 DEBUG 18:47:00 DEBUG 18:47:00 DEBUG "instance": "10.142.0.51:9100" 18:47:00 DEBUG }, 18:47:00 DEBUG "value": [ 18:47:00 DEBUG 1734461220, 18:47:00 DEBUG "88373.73333333335" 18:47:00 DEBUG ] 18:47:00 DEBUG } 18:47:00 DEBUG ] 18:47:00 DEBUG } 18:47:00 DEBUG } 18:47:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461220" 18:47:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461220" 18:47:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461220" 18:47:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734461220" 18:47:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461220" 18:47:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461220" 18:47:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461220" 18:47:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461220" 18:47:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734461220" 18:47:00 INFO [http_cmd]: http status code OK 18:47:00 DEBUG --- status code --- 18:47:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:47:00 DEBUG --- http response --- 18:47:00 INFO [http_cmd]: http status code OK 18:47:00 DEBUG --- status code --- 18:47:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:47:00 DEBUG --- http response --- 18:47:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:47:00 DEBUG ----- output ----- 18:47:00 DEBUG { 18:47:00 DEBUG "status": "success", 18:47:00 DEBUG "data": { 18:47:00 DEBUG "resultType": "vector", 18:47:00 DEBUG "result": [ 18:47:00 DEBUG { 18:47:00 DEBUG "metric": { 18:47:00 DEBUG "instance": "10.142.0.11:9100" 18:47:00 DEBUG }, 18:47:00 DEBUG "value": [ 18:47:00 DEBUG 18:47:00 DEBUG 18:47:00 DEBUG 18:47:00 DEBUG [...] 18:47:00 DEBUG 18:47:00 DEBUG 18:47:00 DEBUG 18:47:00 DEBUG "instance": "10.142.0.51:9100" 18:47:00 DEBUG }, 18:47:00 DEBUG "value": [ 18:47:00 DEBUG 1734461220, 18:47:00 DEBUG "0.0003333333333333855" 18:47:00 DEBUG ] 18:47:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:47:00 DEBUG } 18:47:00 INFO [http_cmd]: http status code OK 18:47:00 INFO [http_cmd]: http status code OK 18:47:00 INFO [http_cmd]: http status code OK 18:47:00 DEBUG ----- output ----- 18:47:00 DEBUG ] 18:47:00 DEBUG --- status code --- 18:47:00 DEBUG --- status code --- 18:47:00 DEBUG --- status code --- 18:47:00 DEBUG { 18:47:00 DEBUG } 18:47:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:47:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:47:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:47:00 DEBUG "status": "success", 18:47:00 DEBUG } 18:47:00 DEBUG --- http response --- 18:47:00 DEBUG --- http response --- 18:47:00 DEBUG --- http response --- 18:47:00 DEBUG "data": { 18:47:00 INFO [http_cmd]: http status code OK 18:47:00 INFO [http_cmd]: http status code OK 18:47:00 INFO [http_cmd]: http status code OK 18:47:00 INFO [http_cmd]: http status code OK 18:47:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:47:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:47:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:47:00 DEBUG "resultType": "vector", 18:47:00 DEBUG --- status code --- 18:47:00 DEBUG --- status code --- 18:47:00 DEBUG --- status code --- 18:47:00 DEBUG --- status code --- 18:47:00 DEBUG ----- output ----- 18:47:00 DEBUG ----- output ----- 18:47:00 DEBUG ----- output ----- 18:47:00 DEBUG "result": [ 18:47:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:47:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:47:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:47:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:47:00 DEBUG { 18:47:00 DEBUG { 18:47:00 DEBUG { 18:47:00 DEBUG { 18:47:00 DEBUG --- http response --- 18:47:00 DEBUG --- http response --- 18:47:00 DEBUG --- http response --- 18:47:00 DEBUG --- http response --- 18:47:00 DEBUG "status": "success", 18:47:00 DEBUG "status": "success", 18:47:00 DEBUG "status": "success", 18:47:00 DEBUG "metric": { 18:47:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:47:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:47:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:47:00 DEBUG "data": { 18:47:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:47:00 DEBUG "data": { 18:47:00 DEBUG "data": { 18:47:00 DEBUG "pod": "ds-cts-0" 18:47:00 DEBUG ----- output ----- 18:47:00 DEBUG ----- output ----- 18:47:00 DEBUG ----- output ----- 18:47:00 DEBUG "resultType": "vector", 18:47:00 DEBUG ----- output ----- 18:47:00 DEBUG "resultType": "vector", 18:47:00 DEBUG "resultType": "vector", 18:47:00 DEBUG }, 18:47:00 DEBUG { 18:47:00 DEBUG { 18:47:00 DEBUG { 18:47:00 DEBUG "result": [ 18:47:00 DEBUG { 18:47:00 DEBUG "result": [ 18:47:00 DEBUG "result": [ 18:47:00 DEBUG "value": [ 18:47:00 DEBUG "status": "success", 18:47:00 DEBUG "status": "success", 18:47:00 DEBUG "status": "success", 18:47:00 DEBUG { 18:47:00 DEBUG "status": "success", 18:47:00 DEBUG { 18:47:00 DEBUG { 18:47:00 DEBUG 18:47:00 DEBUG "data": { 18:47:00 DEBUG "data": { 18:47:00 DEBUG "data": { 18:47:00 DEBUG "metric": { 18:47:00 DEBUG "data": { 18:47:00 DEBUG "metric": { 18:47:00 DEBUG "metric": { 18:47:00 DEBUG 18:47:00 DEBUG "resultType": "vector", 18:47:00 DEBUG "resultType": "vector", 18:47:00 DEBUG "resultType": "vector", 18:47:00 DEBUG "instance": "10.142.0.11:9100" 18:47:00 DEBUG "resultType": "vector", 18:47:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:47:00 DEBUG "instance": "10.142.0.11:9100" 18:47:00 DEBUG 18:47:00 DEBUG "result": [ 18:47:00 DEBUG "result": [ 18:47:00 DEBUG "result": [ 18:47:00 DEBUG }, 18:47:00 DEBUG "result": [ 18:47:00 DEBUG }, 18:47:00 DEBUG }, 18:47:00 DEBUG [...] 18:47:00 DEBUG { 18:47:00 DEBUG { 18:47:00 DEBUG { 18:47:00 DEBUG "value": [ 18:47:00 DEBUG { 18:47:00 DEBUG "value": [ 18:47:00 DEBUG "value": [ 18:47:00 DEBUG 18:47:00 DEBUG "metric": { 18:47:00 DEBUG "metric": { 18:47:00 DEBUG "metric": { 18:47:00 DEBUG 18:47:00 DEBUG "metric": { 18:47:00 DEBUG 18:47:00 DEBUG 18:47:00 DEBUG 18:47:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:47:00 DEBUG "instance": "10.142.0.11:9100" 18:47:00 DEBUG "pod": "ds-cts-0" 18:47:00 DEBUG 18:47:00 DEBUG "pod": "ds-cts-0" 18:47:00 DEBUG 18:47:00 DEBUG 18:47:00 DEBUG 18:47:00 DEBUG }, 18:47:00 DEBUG }, 18:47:00 DEBUG }, 18:47:00 DEBUG 18:47:00 DEBUG }, 18:47:00 DEBUG 18:47:00 DEBUG 18:47:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:47:00 DEBUG "value": [ 18:47:00 DEBUG "value": [ 18:47:00 DEBUG "value": [ 18:47:00 DEBUG [...] 18:47:00 DEBUG "value": [ 18:47:00 DEBUG [...] 18:47:00 DEBUG [...] 18:47:00 DEBUG }, 18:47:00 DEBUG 18:47:00 DEBUG 18:47:00 DEBUG 18:47:00 DEBUG 18:47:00 DEBUG 18:47:00 DEBUG 18:47:00 DEBUG 18:47:00 DEBUG "value": [ 18:47:00 DEBUG 18:47:00 DEBUG 18:47:00 DEBUG 18:47:00 DEBUG 18:47:00 DEBUG 18:47:00 DEBUG 18:47:00 DEBUG 18:47:00 DEBUG 1734461220, 18:47:00 DEBUG 18:47:00 DEBUG 18:47:00 DEBUG 18:47:00 DEBUG 18:47:00 DEBUG 18:47:00 DEBUG 18:47:00 DEBUG 18:47:00 DEBUG "715816960" 18:47:00 DEBUG [...] 18:47:00 DEBUG [...] 18:47:00 DEBUG [...] 18:47:00 DEBUG "instance": "10.142.0.51:9100" 18:47:00 DEBUG [...] 18:47:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:47:00 DEBUG "instance": "10.142.0.51:9100" 18:47:00 DEBUG ] 18:47:00 DEBUG 18:47:00 DEBUG 18:47:00 DEBUG 18:47:00 DEBUG }, 18:47:00 DEBUG 18:47:00 DEBUG }, 18:47:00 DEBUG }, 18:47:00 DEBUG } 18:47:00 DEBUG 18:47:00 DEBUG 18:47:00 DEBUG 18:47:00 DEBUG "value": [ 18:47:00 DEBUG 18:47:00 DEBUG "value": [ 18:47:00 DEBUG "value": [ 18:47:00 DEBUG 1734461220, 18:47:00 DEBUG 18:47:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:47:00 DEBUG }, 18:47:00 DEBUG 18:47:00 DEBUG 18:47:00 DEBUG 1734461220, 18:47:00 DEBUG ] 18:47:00 DEBUG "0.00039999999999954156" 18:47:00 DEBUG 18:47:00 DEBUG 1734461220, 18:47:00 DEBUG "value": [ 18:47:00 DEBUG 1734461220, 18:47:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:47:00 DEBUG "2620133376" 18:47:00 DEBUG } 18:47:00 DEBUG ] 18:47:00 DEBUG "instance": "10.142.0.51:9100" 18:47:00 DEBUG "75136.40000000001" 18:47:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:47:00 DEBUG "1.785650640633136" 18:47:00 DEBUG }, 18:47:00 DEBUG ] 18:47:00 DEBUG } 18:47:00 DEBUG } 18:47:00 DEBUG }, 18:47:00 DEBUG ] 18:47:00 DEBUG }, 18:47:00 DEBUG ] 18:47:00 DEBUG "value": [ 18:47:00 DEBUG } 18:47:00 DEBUG ] 18:47:00 DEBUG } 18:47:00 DEBUG } 18:47:00 DEBUG "value": [ 18:47:00 DEBUG } 18:47:00 DEBUG 1734461220, 18:47:00 DEBUG ] 18:47:00 DEBUG "value": [ 18:47:00 DEBUG } 18:47:00 DEBUG ] 18:47:00 DEBUG 1734461220, 18:47:00 DEBUG ] 18:47:00 DEBUG "1208.0685538033986" 18:47:00 DEBUG } 18:47:00 DEBUG 1734461220, 18:47:00 DEBUG } 18:47:00 DEBUG "0" 18:47:00 DEBUG } 18:47:00 DEBUG ] 18:47:00 DEBUG } 18:47:00 DEBUG "17749.333333333336" 18:47:00 DEBUG } 18:47:00 DEBUG ] 18:47:00 DEBUG } 18:47:00 DEBUG } 18:47:00 DEBUG ] 18:47:00 DEBUG } 18:47:00 DEBUG ] 18:47:00 DEBUG } 18:47:00 DEBUG ] 18:47:00 DEBUG } 18:47:00 DEBUG ] 18:47:00 DEBUG } 18:47:00 DEBUG } 18:47:00 DEBUG } 18:47:00 DEBUG } 18:47:00 DEBUG } 18:47:01 INFO 18:47:01 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:47:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:47:01 INFO [loop_until]: OK (rc = 0) 18:47:01 DEBUG --- stdout --- 18:47:01 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 543Mi ds-idrepo-0 7m 727Mi rcs-5d696ff5b-7ttvr 3m 296Mi rcs-5d696ff5b-wwlbr 3m 340Mi 18:47:01 DEBUG --- stderr --- 18:47:01 DEBUG 18:47:02 INFO 18:47:02 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:47:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:47:02 INFO [loop_until]: OK (rc = 0) 18:47:02 DEBUG --- stdout --- 18:47:02 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 94m 0% 2292Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1856m 5% 3615Mi 12% 18:47:02 DEBUG --- stderr --- 18:47:02 DEBUG 18:47:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461279" 18:47:59 INFO [http_cmd]: http status code OK 18:47:59 DEBUG --- status code --- 18:47:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:47:59 DEBUG --- http response --- 18:47:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:47:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461279" 18:47:59 INFO [http_cmd]: http status code OK 18:47:59 DEBUG --- status code --- 18:47:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:47:59 DEBUG --- http response --- 18:47:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:47:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734461279" 18:47:59 INFO [http_cmd]: http status code OK 18:47:59 DEBUG --- status code --- 18:47:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:47:59 DEBUG --- http response --- 18:47:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:47:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461279" 18:47:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461279" 18:47:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461279" 18:47:59 INFO [http_cmd]: http status code OK 18:47:59 INFO [http_cmd]: http status code OK 18:47:59 DEBUG --- status code --- 18:47:59 DEBUG --- status code --- 18:47:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:47:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:47:59 DEBUG --- http response --- 18:47:59 DEBUG --- http response --- 18:47:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:47:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:47:59 INFO [http_cmd]: http status code OK 18:47:59 DEBUG --- status code --- 18:47:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:47:59 DEBUG --- http response --- 18:47:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:47:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461279" 18:47:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461279" 18:47:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461279" 18:47:59 INFO [http_cmd]: http status code OK 18:47:59 DEBUG --- status code --- 18:47:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:47:59 DEBUG --- http response --- 18:47:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:47:59 INFO [http_cmd]: http status code OK 18:47:59 DEBUG --- status code --- 18:47:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:47:59 DEBUG --- http response --- 18:47:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:47:59 INFO [http_cmd]: http status code OK 18:47:59 DEBUG --- status code --- 18:47:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:47:59 DEBUG --- http response --- 18:47:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:47:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461279" 18:47:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461279" 18:47:59 INFO [http_cmd]: http status code OK 18:47:59 DEBUG --- status code --- 18:47:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:47:59 DEBUG --- http response --- 18:47:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:47:59 INFO [http_cmd]: http status code OK 18:47:59 DEBUG --- status code --- 18:47:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:47:59 DEBUG --- http response --- 18:47:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:47:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461279" 18:47:59 INFO [http_cmd]: http status code OK 18:47:59 DEBUG --- status code --- 18:47:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:47:59 DEBUG --- http response --- 18:47:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:47:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734461279" 18:47:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461279" 18:47:59 ERROR [http_cmd]: ERROR 18:47:59 ERROR --- status code --- 18:47:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:47:59 ERROR --- http response --- 18:47:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:47:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:47:59 INFO [http_cmd]: http status code OK 18:47:59 DEBUG --- status code --- 18:47:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:47:59 DEBUG --- http response --- 18:47:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:47:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461279" 18:47:59 INFO [http_cmd]: http status code OK 18:47:59 DEBUG --- status code --- 18:47:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:47:59 DEBUG --- http response --- 18:47:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:47:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461279" 18:47:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461279" 18:47:59 INFO [http_cmd]: http status code OK 18:47:59 INFO [http_cmd]: http status code OK 18:47:59 DEBUG --- status code --- 18:47:59 DEBUG --- status code --- 18:47:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:47:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:47:59 DEBUG --- http response --- 18:47:59 DEBUG --- http response --- 18:47:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:47:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:47:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461279" 18:47:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461279" 18:47:59 INFO [http_cmd]: http status code OK 18:47:59 DEBUG --- status code --- 18:47:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:47:59 DEBUG --- http response --- 18:47:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:47:59 INFO [http_cmd]: http status code OK 18:47:59 DEBUG --- status code --- 18:47:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:47:59 DEBUG --- http response --- 18:47:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:47:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734461279" 18:47:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461279" 18:47:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734461279" 18:47:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461279" 18:47:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734461279" 18:47:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461279" 18:47:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461279" 18:47:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461279" 18:47:59 INFO [http_cmd]: http status code OK 18:47:59 INFO [http_cmd]: http status code OK 18:47:59 DEBUG --- status code --- 18:47:59 DEBUG --- status code --- 18:47:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:47:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:47:59 DEBUG --- http response --- 18:47:59 DEBUG --- http response --- 18:47:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:47:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:47:59 INFO [http_cmd]: http status code OK 18:47:59 DEBUG --- status code --- 18:47:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:47:59 DEBUG --- http response --- 18:47:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:47:59 INFO [http_cmd]: http status code OK 18:47:59 DEBUG --- status code --- 18:47:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:47:59 DEBUG --- http response --- 18:47:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:47:59 INFO [http_cmd]: http status code OK 18:47:59 DEBUG --- status code --- 18:47:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:47:59 DEBUG --- http response --- 18:47:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:47:59 INFO [http_cmd]: http status code OK 18:47:59 DEBUG --- status code --- 18:47:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:47:59 INFO [http_cmd]: http status code OK 18:47:59 DEBUG --- http response --- 18:47:59 DEBUG --- status code --- 18:47:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:47:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:47:59 DEBUG --- http response --- 18:47:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:47:59 INFO [http_cmd]: http status code OK 18:47:59 DEBUG --- status code --- 18:47:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:47:59 DEBUG --- http response --- 18:47:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:47:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461279" 18:47:59 INFO [http_cmd]: http status code OK 18:47:59 DEBUG --- status code --- 18:47:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:47:59 DEBUG --- http response --- 18:47:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:47:59 DEBUG ----- output ----- 18:47:59 DEBUG { 18:47:59 DEBUG "status": "success", 18:47:59 DEBUG "data": { 18:47:59 DEBUG "resultType": "vector", 18:47:59 DEBUG "result": [ 18:47:59 DEBUG { 18:47:59 DEBUG "metric": { 18:47:59 DEBUG "pod": "ds-cts-0" 18:47:59 DEBUG }, 18:47:59 DEBUG "value": [ 18:47:59 DEBUG 18:47:59 DEBUG 18:47:59 DEBUG 18:47:59 DEBUG [...] 18:47:59 DEBUG 18:47:59 DEBUG 18:47:59 DEBUG 18:47:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:47:59 DEBUG }, 18:47:59 DEBUG "value": [ 18:47:59 DEBUG 1734461279, 18:47:59 DEBUG "0.0027918147561504323" 18:47:59 DEBUG ] 18:47:59 DEBUG } 18:47:59 DEBUG ] 18:47:59 DEBUG } 18:47:59 DEBUG } 18:48:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461280" 18:48:00 INFO [http_cmd]: http status code OK 18:48:00 DEBUG --- status code --- 18:48:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:48:00 DEBUG --- http response --- 18:48:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:48:00 DEBUG ----- output ----- 18:48:00 DEBUG { 18:48:00 DEBUG "status": "success", 18:48:00 DEBUG "data": { 18:48:00 DEBUG "resultType": "vector", 18:48:00 DEBUG "result": [ 18:48:00 DEBUG { 18:48:00 DEBUG "metric": { 18:48:00 DEBUG "instance": "10.142.0.11:9100" 18:48:00 DEBUG }, 18:48:00 DEBUG "value": [ 18:48:00 DEBUG 18:48:00 DEBUG 18:48:00 DEBUG 18:48:00 DEBUG [...] 18:48:00 DEBUG 18:48:00 DEBUG 18:48:00 DEBUG 18:48:00 DEBUG "instance": "10.142.0.51:9100" 18:48:00 DEBUG }, 18:48:00 DEBUG "value": [ 18:48:00 DEBUG 1734461280, 18:48:00 DEBUG "87137.60000000002" 18:48:00 DEBUG ] 18:48:00 DEBUG } 18:48:00 DEBUG ] 18:48:00 DEBUG } 18:48:00 DEBUG } 18:48:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461280" 18:48:00 INFO [http_cmd]: http status code OK 18:48:00 DEBUG --- status code --- 18:48:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:48:00 DEBUG --- http response --- 18:48:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:48:00 DEBUG ----- output ----- 18:48:00 DEBUG { 18:48:00 DEBUG "status": "success", 18:48:00 DEBUG "data": { 18:48:00 DEBUG "resultType": "vector", 18:48:00 DEBUG "result": [ 18:48:00 DEBUG { 18:48:00 DEBUG "metric": { 18:48:00 DEBUG "instance": "10.142.0.11:9100" 18:48:00 DEBUG }, 18:48:00 DEBUG "value": [ 18:48:00 DEBUG 18:48:00 DEBUG 18:48:00 DEBUG 18:48:00 DEBUG [...] 18:48:00 DEBUG 18:48:00 DEBUG 18:48:00 DEBUG 18:48:00 DEBUG "instance": "10.142.0.51:9100" 18:48:00 DEBUG }, 18:48:00 DEBUG "value": [ 18:48:00 DEBUG 1734461280, 18:48:00 DEBUG "0.0003333333333332671" 18:48:00 DEBUG ] 18:48:00 DEBUG } 18:48:00 DEBUG ] 18:48:00 DEBUG } 18:48:00 DEBUG } 18:48:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461280" 18:48:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461280" 18:48:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734461280" 18:48:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461280" 18:48:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461280" 18:48:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461280" 18:48:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461280" 18:48:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734461280" 18:48:00 INFO [http_cmd]: http status code OK 18:48:00 INFO [http_cmd]: http status code OK 18:48:00 DEBUG --- status code --- 18:48:00 INFO [http_cmd]: http status code OK 18:48:00 DEBUG --- status code --- 18:48:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:48:00 DEBUG --- status code --- 18:48:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:48:00 DEBUG --- http response --- 18:48:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:48:00 DEBUG --- http response --- 18:48:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:48:00 DEBUG --- http response --- 18:48:00 DEBUG ----- output ----- 18:48:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:48:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:48:00 DEBUG { 18:48:00 INFO [http_cmd]: http status code OK 18:48:00 DEBUG ----- output ----- 18:48:00 DEBUG ----- output ----- 18:48:00 DEBUG "status": "success", 18:48:00 DEBUG --- status code --- 18:48:00 DEBUG { 18:48:00 DEBUG { 18:48:00 DEBUG "data": { 18:48:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:48:00 DEBUG "status": "success", 18:48:00 DEBUG "status": "success", 18:48:00 DEBUG "resultType": "vector", 18:48:00 INFO [http_cmd]: http status code OK 18:48:00 DEBUG --- http response --- 18:48:00 INFO [http_cmd]: http status code OK 18:48:00 DEBUG "data": { 18:48:00 DEBUG "data": { 18:48:00 DEBUG "result": [ 18:48:00 DEBUG --- status code --- 18:48:00 DEBUG --- status code --- 18:48:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:48:00 DEBUG "resultType": "vector", 18:48:00 DEBUG "resultType": "vector", 18:48:00 DEBUG { 18:48:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:48:00 INFO [http_cmd]: http status code OK 18:48:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:48:00 DEBUG --- http response --- 18:48:00 DEBUG ----- output ----- 18:48:00 DEBUG "result": [ 18:48:00 DEBUG "result": [ 18:48:00 DEBUG "metric": { 18:48:00 DEBUG --- http response --- 18:48:00 DEBUG --- status code --- 18:48:00 INFO [http_cmd]: http status code OK 18:48:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:48:00 DEBUG { 18:48:00 DEBUG { 18:48:00 DEBUG { 18:48:00 DEBUG "instance": "10.142.0.11:9100" 18:48:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:48:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:48:00 DEBUG --- status code --- 18:48:00 DEBUG ----- output ----- 18:48:00 DEBUG "status": "success", 18:48:00 DEBUG "metric": { 18:48:00 DEBUG "metric": { 18:48:00 DEBUG }, 18:48:00 DEBUG --- http response --- 18:48:00 DEBUG ----- output ----- 18:48:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:48:00 DEBUG { 18:48:00 DEBUG "data": { 18:48:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:48:00 DEBUG "pod": "ds-cts-0" 18:48:00 DEBUG "value": [ 18:48:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:48:00 DEBUG { 18:48:00 DEBUG --- http response --- 18:48:00 DEBUG "status": "success", 18:48:00 DEBUG "resultType": "vector", 18:48:00 DEBUG }, 18:48:00 DEBUG }, 18:48:00 DEBUG 18:48:00 DEBUG ----- output ----- 18:48:00 DEBUG "status": "success", 18:48:00 DEBUG "data": { 18:48:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:48:00 DEBUG "result": [ 18:48:00 DEBUG "value": [ 18:48:00 DEBUG "value": [ 18:48:00 DEBUG 18:48:00 DEBUG { 18:48:00 DEBUG "data": { 18:48:00 DEBUG "resultType": "vector", 18:48:00 DEBUG ----- output ----- 18:48:00 DEBUG { 18:48:00 DEBUG 18:48:00 DEBUG 18:48:00 DEBUG 18:48:00 DEBUG "status": "success", 18:48:00 DEBUG "resultType": "vector", 18:48:00 DEBUG "result": [ 18:48:00 DEBUG { 18:48:00 DEBUG "metric": { 18:48:00 DEBUG 18:48:00 DEBUG 18:48:00 DEBUG [...] 18:48:00 DEBUG "data": { 18:48:00 DEBUG "result": [ 18:48:00 DEBUG { 18:48:00 DEBUG "status": "success", 18:48:00 DEBUG "instance": "10.142.0.11:9100" 18:48:00 DEBUG 18:48:00 DEBUG 18:48:00 DEBUG 18:48:00 DEBUG "resultType": "vector", 18:48:00 DEBUG { 18:48:00 DEBUG "metric": { 18:48:00 DEBUG "data": { 18:48:00 DEBUG }, 18:48:00 DEBUG [...] 18:48:00 DEBUG [...] 18:48:00 DEBUG 18:48:00 DEBUG "result": [ 18:48:00 DEBUG "metric": { 18:48:00 DEBUG "pod": "ds-cts-0" 18:48:00 DEBUG "resultType": "vector", 18:48:00 DEBUG "value": [ 18:48:00 DEBUG 18:48:00 DEBUG 18:48:00 DEBUG 18:48:00 DEBUG { 18:48:00 DEBUG "instance": "10.142.0.11:9100" 18:48:00 DEBUG }, 18:48:00 DEBUG "result": [ 18:48:00 DEBUG 18:48:00 DEBUG 18:48:00 DEBUG 18:48:00 DEBUG "instance": "10.142.0.51:9100" 18:48:00 DEBUG "metric": { 18:48:00 DEBUG }, 18:48:00 DEBUG "value": [ 18:48:00 DEBUG { 18:48:00 DEBUG 18:48:00 DEBUG 18:48:00 DEBUG 18:48:00 DEBUG }, 18:48:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:48:00 DEBUG "value": [ 18:48:00 DEBUG 18:48:00 DEBUG "metric": { 18:48:00 DEBUG 18:48:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:48:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:48:00 DEBUG "value": [ 18:48:00 DEBUG }, 18:48:00 DEBUG 18:48:00 DEBUG 18:48:00 DEBUG 18:48:00 DEBUG [...] 18:48:00 DEBUG 18:48:00 DEBUG 18:48:00 DEBUG 18:48:00 DEBUG "value": [ 18:48:00 DEBUG 18:48:00 DEBUG "pod": "ds-cts-0" 18:48:00 DEBUG [...] 18:48:00 DEBUG }, 18:48:00 DEBUG }, 18:48:00 DEBUG 1734461280, 18:48:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:48:00 DEBUG 18:48:00 DEBUG 18:48:00 DEBUG }, 18:48:00 DEBUG 18:48:00 DEBUG "value": [ 18:48:00 DEBUG "value": [ 18:48:00 DEBUG "0.0004333333333330341" 18:48:00 DEBUG }, 18:48:00 DEBUG 18:48:00 DEBUG [...] 18:48:00 DEBUG "value": [ 18:48:00 DEBUG 18:48:00 DEBUG 1734461280, 18:48:00 DEBUG 1734461280, 18:48:00 DEBUG ] 18:48:00 DEBUG "value": [ 18:48:00 DEBUG 18:48:00 DEBUG 18:48:00 DEBUG 18:48:00 DEBUG 18:48:00 DEBUG "2558865408" 18:48:00 DEBUG "714117120" 18:48:00 DEBUG } 18:48:00 DEBUG 1734461280, 18:48:00 DEBUG [...] 18:48:00 DEBUG 18:48:00 DEBUG 18:48:00 DEBUG "instance": "10.142.0.51:9100" 18:48:00 DEBUG ] 18:48:00 DEBUG ] 18:48:00 DEBUG ] 18:48:00 DEBUG "0" 18:48:00 DEBUG 18:48:00 DEBUG 18:48:00 DEBUG 18:48:00 DEBUG }, 18:48:00 DEBUG } 18:48:00 DEBUG } 18:48:00 DEBUG } 18:48:00 DEBUG ] 18:48:00 DEBUG 18:48:00 DEBUG "instance": "10.142.0.51:9100" 18:48:00 DEBUG [...] 18:48:00 DEBUG "value": [ 18:48:00 DEBUG ] 18:48:00 DEBUG ] 18:48:00 DEBUG } 18:48:00 DEBUG } 18:48:00 DEBUG 18:48:00 DEBUG }, 18:48:00 DEBUG 18:48:00 DEBUG 1734461280, 18:48:00 DEBUG } 18:48:00 DEBUG } 18:48:00 DEBUG ] 18:48:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:48:00 DEBUG "value": [ 18:48:00 DEBUG 1734461280, 18:48:00 DEBUG "25941.33333333334" 18:48:00 DEBUG ] 18:48:00 DEBUG } 18:48:00 DEBUG ] 18:48:00 DEBUG } 18:48:00 DEBUG } 18:48:00 DEBUG 18:48:00 DEBUG } 18:48:00 DEBUG } 18:48:00 DEBUG } 18:48:00 DEBUG }, 18:48:00 DEBUG "74040.5666666667" 18:48:00 DEBUG 18:48:00 DEBUG } 18:48:00 DEBUG "value": [ 18:48:00 DEBUG ] 18:48:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:48:00 DEBUG 1734461280, 18:48:00 DEBUG } 18:48:00 DEBUG ] 18:48:00 DEBUG "1.780485326382468" 18:48:00 DEBUG ] 18:48:00 DEBUG } 18:48:00 DEBUG ] 18:48:00 DEBUG }, 18:48:00 DEBUG } 18:48:00 DEBUG } 18:48:00 DEBUG "value": [ 18:48:00 DEBUG } 18:48:00 DEBUG } 18:48:00 DEBUG 1734461280, 18:48:00 DEBUG "1221.7757256061705" 18:48:00 DEBUG ] 18:48:00 DEBUG } 18:48:00 DEBUG ] 18:48:00 DEBUG } 18:48:00 DEBUG } 18:48:01 INFO 18:48:01 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:48:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:48:02 INFO [loop_until]: OK (rc = 0) 18:48:02 DEBUG --- stdout --- 18:48:02 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 543Mi ds-idrepo-0 7m 729Mi rcs-5d696ff5b-7ttvr 3m 296Mi rcs-5d696ff5b-wwlbr 3m 340Mi 18:48:02 DEBUG --- stderr --- 18:48:02 DEBUG 18:48:02 INFO 18:48:02 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:48:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:48:02 INFO [loop_until]: OK (rc = 0) 18:48:02 DEBUG --- stdout --- 18:48:02 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 92m 0% 2291Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1859m 5% 3618Mi 12% 18:48:02 DEBUG --- stderr --- 18:48:02 DEBUG 18:48:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461339" 18:48:59 INFO [http_cmd]: http status code OK 18:48:59 DEBUG --- status code --- 18:48:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:48:59 DEBUG --- http response --- 18:48:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:48:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461339" 18:48:59 INFO [http_cmd]: http status code OK 18:48:59 DEBUG --- status code --- 18:48:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:48:59 DEBUG --- http response --- 18:48:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:48:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734461339" 18:48:59 INFO [http_cmd]: http status code OK 18:48:59 DEBUG --- status code --- 18:48:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:48:59 DEBUG --- http response --- 18:48:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:48:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461339" 18:48:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461339" 18:48:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461339" 18:48:59 INFO [http_cmd]: http status code OK 18:48:59 DEBUG --- status code --- 18:48:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:48:59 INFO [http_cmd]: http status code OK 18:48:59 DEBUG --- http response --- 18:48:59 DEBUG --- status code --- 18:48:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:48:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:48:59 DEBUG --- http response --- 18:48:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:48:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461339" 18:48:59 INFO [http_cmd]: http status code OK 18:48:59 DEBUG --- status code --- 18:48:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:48:59 DEBUG --- http response --- 18:48:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:48:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461339" 18:48:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461339" 18:48:59 INFO [http_cmd]: http status code OK 18:48:59 DEBUG --- status code --- 18:48:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:48:59 DEBUG --- http response --- 18:48:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:48:59 INFO [http_cmd]: http status code OK 18:48:59 INFO [http_cmd]: http status code OK 18:48:59 DEBUG --- status code --- 18:48:59 DEBUG --- status code --- 18:48:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:48:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:48:59 DEBUG --- http response --- 18:48:59 DEBUG --- http response --- 18:48:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:48:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:48:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461339" 18:48:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461339" 18:48:59 INFO [http_cmd]: http status code OK 18:48:59 DEBUG --- status code --- 18:48:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:48:59 DEBUG --- http response --- 18:48:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:48:59 INFO [http_cmd]: http status code OK 18:48:59 DEBUG --- status code --- 18:48:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:48:59 DEBUG --- http response --- 18:48:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:48:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461339" 18:48:59 INFO [http_cmd]: http status code OK 18:48:59 DEBUG --- status code --- 18:48:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:48:59 DEBUG --- http response --- 18:48:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:48:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734461339" 18:48:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461339" 18:48:59 INFO [http_cmd]: http status code OK 18:48:59 ERROR [http_cmd]: ERROR 18:48:59 DEBUG --- status code --- 18:48:59 ERROR --- status code --- 18:48:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:48:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:48:59 DEBUG --- http response --- 18:48:59 ERROR --- http response --- 18:48:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:48:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:48:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:48:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461339" 18:48:59 INFO [http_cmd]: http status code OK 18:48:59 DEBUG --- status code --- 18:48:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:48:59 DEBUG --- http response --- 18:48:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:48:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461339" 18:48:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461339" 18:48:59 INFO [http_cmd]: http status code OK 18:48:59 DEBUG --- status code --- 18:48:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:48:59 DEBUG --- http response --- 18:48:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:48:59 INFO [http_cmd]: http status code OK 18:48:59 DEBUG --- status code --- 18:48:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:48:59 DEBUG --- http response --- 18:48:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:48:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461339" 18:48:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461339" 18:48:59 INFO [http_cmd]: http status code OK 18:48:59 INFO [http_cmd]: http status code OK 18:48:59 DEBUG --- status code --- 18:48:59 DEBUG --- status code --- 18:48:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:48:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:48:59 DEBUG --- http response --- 18:48:59 DEBUG --- http response --- 18:48:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:48:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:48:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734461339" 18:48:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734461339" 18:48:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461339" 18:48:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734461339" 18:48:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461339" 18:48:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461339" 18:48:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461339" 18:48:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461339" 18:48:59 INFO [http_cmd]: http status code OK 18:48:59 DEBUG --- status code --- 18:48:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:48:59 DEBUG --- http response --- 18:48:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:48:59 INFO [http_cmd]: http status code OK 18:48:59 DEBUG --- status code --- 18:48:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:48:59 DEBUG --- http response --- 18:48:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:48:59 INFO [http_cmd]: http status code OK 18:48:59 DEBUG --- status code --- 18:48:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:48:59 DEBUG --- http response --- 18:48:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:48:59 INFO [http_cmd]: http status code OK 18:48:59 DEBUG --- status code --- 18:48:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:48:59 DEBUG --- http response --- 18:48:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:48:59 INFO [http_cmd]: http status code OK 18:48:59 DEBUG --- status code --- 18:48:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:48:59 DEBUG --- http response --- 18:48:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:48:59 INFO [http_cmd]: http status code OK 18:48:59 DEBUG --- status code --- 18:48:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:48:59 DEBUG --- http response --- 18:48:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:48:59 INFO [http_cmd]: http status code OK 18:48:59 DEBUG --- status code --- 18:48:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:48:59 DEBUG --- http response --- 18:48:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:48:59 INFO [http_cmd]: http status code OK 18:48:59 DEBUG --- status code --- 18:48:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:48:59 DEBUG --- http response --- 18:48:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:48:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461339" 18:48:59 INFO [http_cmd]: http status code OK 18:48:59 DEBUG --- status code --- 18:48:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:48:59 DEBUG --- http response --- 18:48:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:48:59 DEBUG ----- output ----- 18:48:59 DEBUG { 18:48:59 DEBUG "status": "success", 18:48:59 DEBUG "data": { 18:48:59 DEBUG "resultType": "vector", 18:48:59 DEBUG "result": [ 18:48:59 DEBUG { 18:48:59 DEBUG "metric": { 18:48:59 DEBUG "pod": "ds-cts-0" 18:48:59 DEBUG }, 18:48:59 DEBUG "value": [ 18:48:59 DEBUG 18:48:59 DEBUG 18:48:59 DEBUG 18:48:59 DEBUG [...] 18:48:59 DEBUG 18:48:59 DEBUG 18:48:59 DEBUG 18:48:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:48:59 DEBUG }, 18:48:59 DEBUG "value": [ 18:48:59 DEBUG 1734461339, 18:48:59 DEBUG "0.0052414473330023754" 18:48:59 DEBUG ] 18:48:59 DEBUG } 18:48:59 DEBUG ] 18:48:59 DEBUG } 18:48:59 DEBUG } 18:49:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461340" 18:49:00 INFO [http_cmd]: http status code OK 18:49:00 DEBUG --- status code --- 18:49:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:49:00 DEBUG --- http response --- 18:49:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:49:00 DEBUG ----- output ----- 18:49:00 DEBUG { 18:49:00 DEBUG "status": "success", 18:49:00 DEBUG "data": { 18:49:00 DEBUG "resultType": "vector", 18:49:00 DEBUG "result": [ 18:49:00 DEBUG { 18:49:00 DEBUG "metric": { 18:49:00 DEBUG "instance": "10.142.0.11:9100" 18:49:00 DEBUG }, 18:49:00 DEBUG "value": [ 18:49:00 DEBUG 18:49:00 DEBUG 18:49:00 DEBUG 18:49:00 DEBUG [...] 18:49:00 DEBUG 18:49:00 DEBUG 18:49:00 DEBUG 18:49:00 DEBUG "instance": "10.142.0.51:9100" 18:49:00 DEBUG }, 18:49:00 DEBUG "value": [ 18:49:00 DEBUG 1734461340, 18:49:00 DEBUG "90365.60000000002" 18:49:00 DEBUG ] 18:49:00 DEBUG } 18:49:00 DEBUG ] 18:49:00 DEBUG } 18:49:00 DEBUG } 18:49:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461340" 18:49:00 INFO [http_cmd]: http status code OK 18:49:00 DEBUG --- status code --- 18:49:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:49:00 DEBUG --- http response --- 18:49:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:49:00 DEBUG ----- output ----- 18:49:00 DEBUG { 18:49:00 DEBUG "status": "success", 18:49:00 DEBUG "data": { 18:49:00 DEBUG "resultType": "vector", 18:49:00 DEBUG "result": [ 18:49:00 DEBUG { 18:49:00 DEBUG "metric": { 18:49:00 DEBUG "instance": "10.142.0.11:9100" 18:49:00 DEBUG }, 18:49:00 DEBUG "value": [ 18:49:00 DEBUG 18:49:00 DEBUG 18:49:00 DEBUG 18:49:00 DEBUG [...] 18:49:00 DEBUG 18:49:00 DEBUG 18:49:00 DEBUG 18:49:00 DEBUG "instance": "10.142.0.51:9100" 18:49:00 DEBUG }, 18:49:00 DEBUG "value": [ 18:49:00 DEBUG 1734461340, 18:49:00 DEBUG "0" 18:49:00 DEBUG ] 18:49:00 DEBUG } 18:49:00 DEBUG ] 18:49:00 DEBUG } 18:49:00 DEBUG } 18:49:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461340" 18:49:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461340" 18:49:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734461340" 18:49:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461340" 18:49:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461340" 18:49:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734461340" 18:49:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461340" 18:49:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461340" 18:49:00 INFO [http_cmd]: http status code OK 18:49:00 DEBUG --- status code --- 18:49:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:49:00 DEBUG --- http response --- 18:49:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:49:00 DEBUG ----- output ----- 18:49:00 DEBUG { 18:49:00 DEBUG "status": "success", 18:49:00 DEBUG "data": { 18:49:00 DEBUG "resultType": "vector", 18:49:00 DEBUG "result": [ 18:49:00 DEBUG { 18:49:00 DEBUG "metric": { 18:49:00 DEBUG "instance": "10.142.0.11:9100" 18:49:00 DEBUG }, 18:49:00 DEBUG "value": [ 18:49:00 DEBUG 18:49:00 DEBUG 18:49:00 DEBUG 18:49:00 DEBUG [...] 18:49:00 DEBUG 18:49:00 DEBUG 18:49:00 DEBUG 18:49:00 DEBUG "instance": "10.142.0.51:9100" 18:49:00 DEBUG }, 18:49:00 DEBUG "value": [ 18:49:00 INFO [http_cmd]: http status code OK 18:49:00 DEBUG 1734461340, 18:49:00 INFO [http_cmd]: http status code OK 18:49:00 INFO [http_cmd]: http status code OK 18:49:00 DEBUG --- status code --- 18:49:00 INFO [http_cmd]: http status code OK 18:49:00 INFO [http_cmd]: http status code OK 18:49:00 INFO [http_cmd]: http status code OK 18:49:00 DEBUG "0.0004333333333358762" 18:49:00 DEBUG --- status code --- 18:49:00 DEBUG --- status code --- 18:49:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:49:00 DEBUG --- status code --- 18:49:00 DEBUG --- status code --- 18:49:00 DEBUG --- status code --- 18:49:00 DEBUG ] 18:49:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:49:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:49:00 DEBUG --- http response --- 18:49:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:49:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:49:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:49:00 DEBUG } 18:49:00 DEBUG --- http response --- 18:49:00 DEBUG --- http response --- 18:49:00 DEBUG --- http response --- 18:49:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:49:00 DEBUG --- http response --- 18:49:00 DEBUG --- http response --- 18:49:00 DEBUG ] 18:49:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:49:00 DEBUG ----- output ----- 18:49:00 DEBUG ----- output ----- 18:49:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:49:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:49:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:49:00 DEBUG } 18:49:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:49:00 DEBUG { 18:49:00 DEBUG { 18:49:00 DEBUG ----- output ----- 18:49:00 DEBUG ----- output ----- 18:49:00 DEBUG ----- output ----- 18:49:00 DEBUG } 18:49:00 DEBUG ----- output ----- 18:49:00 DEBUG "status": "success", 18:49:00 DEBUG "status": "success", 18:49:00 DEBUG { 18:49:00 DEBUG { 18:49:00 DEBUG { 18:49:00 DEBUG { 18:49:00 DEBUG "data": { 18:49:00 DEBUG "data": { 18:49:00 DEBUG "status": "success", 18:49:00 DEBUG "status": "success", 18:49:00 DEBUG "status": "success", 18:49:00 DEBUG "status": "success", 18:49:00 DEBUG "resultType": "vector", 18:49:00 DEBUG "resultType": "vector", 18:49:00 INFO [http_cmd]: http status code OK 18:49:00 DEBUG "data": { 18:49:00 DEBUG "data": { 18:49:00 DEBUG "data": { 18:49:00 DEBUG "data": { 18:49:00 DEBUG "result": [ 18:49:00 DEBUG "result": [ 18:49:00 DEBUG --- status code --- 18:49:00 DEBUG "resultType": "vector", 18:49:00 DEBUG "resultType": "vector", 18:49:00 DEBUG "resultType": "vector", 18:49:00 DEBUG "resultType": "vector", 18:49:00 DEBUG { 18:49:00 DEBUG { 18:49:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:49:00 DEBUG "result": [ 18:49:00 DEBUG "result": [ 18:49:00 DEBUG "result": [ 18:49:00 DEBUG "result": [ 18:49:00 DEBUG "metric": { 18:49:00 DEBUG "metric": { 18:49:00 DEBUG --- http response --- 18:49:00 DEBUG { 18:49:00 DEBUG { 18:49:00 DEBUG { 18:49:00 DEBUG { 18:49:00 DEBUG "pod": "ds-cts-0" 18:49:00 DEBUG "instance": "10.142.0.11:9100" 18:49:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:49:00 DEBUG "metric": { 18:49:00 DEBUG "metric": { 18:49:00 DEBUG "metric": { 18:49:00 DEBUG "metric": { 18:49:00 DEBUG }, 18:49:00 DEBUG }, 18:49:00 DEBUG ----- output ----- 18:49:00 DEBUG "pod": "ds-cts-0" 18:49:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:49:00 DEBUG "instance": "10.142.0.11:9100" 18:49:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:49:00 DEBUG "value": [ 18:49:00 DEBUG "value": [ 18:49:00 DEBUG { 18:49:00 DEBUG }, 18:49:00 DEBUG }, 18:49:00 DEBUG }, 18:49:00 DEBUG "value": [ 18:49:00 DEBUG 18:49:00 DEBUG 18:49:00 DEBUG "status": "success", 18:49:00 DEBUG "value": [ 18:49:00 DEBUG "value": [ 18:49:00 DEBUG }, 18:49:00 DEBUG 18:49:00 DEBUG 18:49:00 DEBUG 18:49:00 DEBUG "data": { 18:49:00 DEBUG 18:49:00 DEBUG 18:49:00 DEBUG "value": [ 18:49:00 DEBUG 18:49:00 DEBUG 18:49:00 DEBUG 18:49:00 DEBUG "resultType": "vector", 18:49:00 DEBUG 18:49:00 DEBUG 18:49:00 DEBUG 18:49:00 DEBUG 18:49:00 DEBUG [...] 18:49:00 DEBUG [...] 18:49:00 DEBUG "result": [ 18:49:00 DEBUG 18:49:00 DEBUG 18:49:00 DEBUG 18:49:00 DEBUG [...] 18:49:00 DEBUG 18:49:00 DEBUG 18:49:00 DEBUG { 18:49:00 DEBUG [...] 18:49:00 DEBUG [...] 18:49:00 DEBUG 18:49:00 DEBUG 18:49:00 DEBUG 18:49:00 DEBUG 18:49:00 DEBUG "metric": { 18:49:00 DEBUG 18:49:00 DEBUG 18:49:00 DEBUG [...] 18:49:00 DEBUG 18:49:00 DEBUG 18:49:00 DEBUG 18:49:00 DEBUG "pod": "ds-cts-0" 18:49:00 DEBUG 18:49:00 DEBUG 18:49:00 DEBUG 18:49:00 DEBUG 18:49:00 DEBUG "instance": "10.142.0.51:9100" 18:49:00 DEBUG "instance": "10.142.0.51:9100" 18:49:00 DEBUG }, 18:49:00 DEBUG 18:49:00 DEBUG 18:49:00 DEBUG 18:49:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:49:00 DEBUG }, 18:49:00 DEBUG }, 18:49:00 DEBUG "value": [ 18:49:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:49:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:49:00 DEBUG 18:49:00 DEBUG }, 18:49:00 DEBUG "value": [ 18:49:00 DEBUG "value": [ 18:49:00 DEBUG 18:49:00 DEBUG }, 18:49:00 DEBUG }, 18:49:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:49:00 DEBUG "value": [ 18:49:00 DEBUG 1734461340, 18:49:00 DEBUG 1734461340, 18:49:00 DEBUG 18:49:00 DEBUG "value": [ 18:49:00 DEBUG "value": [ 18:49:00 DEBUG }, 18:49:00 DEBUG 1734461340, 18:49:00 DEBUG "29081.600000000006" 18:49:00 DEBUG "79107.90000000002" 18:49:00 DEBUG 18:49:00 DEBUG 1734461340, 18:49:00 DEBUG 1734461340, 18:49:00 DEBUG "value": [ 18:49:00 DEBUG "0" 18:49:00 DEBUG ] 18:49:00 DEBUG ] 18:49:00 DEBUG [...] 18:49:00 DEBUG "714141696" 18:49:00 DEBUG "1.7782878971270324" 18:49:00 DEBUG 1734461340, 18:49:00 DEBUG ] 18:49:00 DEBUG } 18:49:00 DEBUG } 18:49:00 DEBUG 18:49:00 DEBUG ] 18:49:00 DEBUG ] 18:49:00 DEBUG "2529193984" 18:49:00 DEBUG } 18:49:00 DEBUG ] 18:49:00 DEBUG ] 18:49:00 DEBUG 18:49:00 DEBUG } 18:49:00 DEBUG } 18:49:00 DEBUG ] 18:49:00 DEBUG ] 18:49:00 DEBUG } 18:49:00 DEBUG } 18:49:00 DEBUG 18:49:00 DEBUG ] 18:49:00 DEBUG ] 18:49:00 DEBUG } 18:49:00 DEBUG } 18:49:00 DEBUG } 18:49:00 DEBUG } 18:49:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:49:00 DEBUG } 18:49:00 DEBUG } 18:49:00 DEBUG ] 18:49:00 DEBUG } 18:49:00 DEBUG }, 18:49:00 DEBUG } 18:49:00 DEBUG } 18:49:00 DEBUG } 18:49:00 DEBUG "value": [ 18:49:00 DEBUG } 18:49:00 DEBUG 1734461340, 18:49:00 DEBUG "1570.0402339897191" 18:49:00 DEBUG ] 18:49:00 DEBUG } 18:49:00 DEBUG ] 18:49:00 DEBUG } 18:49:00 DEBUG } 18:49:02 INFO 18:49:02 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:49:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:49:02 INFO [loop_until]: OK (rc = 0) 18:49:02 DEBUG --- stdout --- 18:49:02 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 543Mi ds-idrepo-0 7m 732Mi rcs-5d696ff5b-7ttvr 3m 296Mi rcs-5d696ff5b-wwlbr 2m 340Mi 18:49:02 DEBUG --- stderr --- 18:49:02 DEBUG 18:49:02 INFO 18:49:02 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:49:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:49:02 INFO [loop_until]: OK (rc = 0) 18:49:02 DEBUG --- stdout --- 18:49:02 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 98m 0% 2291Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1847m 5% 3622Mi 12% 18:49:02 DEBUG --- stderr --- 18:49:02 DEBUG 18:49:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461399" 18:49:59 INFO [http_cmd]: http status code OK 18:49:59 DEBUG --- status code --- 18:49:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:49:59 DEBUG --- http response --- 18:49:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:49:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461399" 18:49:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734461399" 18:49:59 INFO [http_cmd]: http status code OK 18:49:59 DEBUG --- status code --- 18:49:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:49:59 DEBUG --- http response --- 18:49:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:49:59 INFO [http_cmd]: http status code OK 18:49:59 DEBUG --- status code --- 18:49:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:49:59 DEBUG --- http response --- 18:49:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:49:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461399" 18:49:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461399" 18:49:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461399" 18:49:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461399" 18:49:59 INFO [http_cmd]: http status code OK 18:49:59 DEBUG --- status code --- 18:49:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:49:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461399" 18:49:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461399" 18:49:59 INFO [http_cmd]: http status code OK 18:49:59 DEBUG --- http response --- 18:49:59 DEBUG --- status code --- 18:49:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:49:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:49:59 DEBUG --- http response --- 18:49:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:49:59 INFO [http_cmd]: http status code OK 18:49:59 DEBUG --- status code --- 18:49:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:49:59 DEBUG --- http response --- 18:49:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:49:59 INFO [http_cmd]: http status code OK 18:49:59 DEBUG --- status code --- 18:49:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:49:59 DEBUG --- http response --- 18:49:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:49:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461399" 18:49:59 INFO [http_cmd]: http status code OK 18:49:59 DEBUG --- status code --- 18:49:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:49:59 DEBUG --- http response --- 18:49:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:49:59 INFO [http_cmd]: http status code OK 18:49:59 DEBUG --- status code --- 18:49:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:49:59 DEBUG --- http response --- 18:49:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:49:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461399" 18:49:59 INFO [http_cmd]: http status code OK 18:49:59 DEBUG --- status code --- 18:49:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:49:59 DEBUG --- http response --- 18:49:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:49:59 INFO [http_cmd]: http status code OK 18:49:59 DEBUG --- status code --- 18:49:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:49:59 DEBUG --- http response --- 18:49:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:49:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461399" 18:49:59 INFO [http_cmd]: http status code OK 18:49:59 DEBUG --- status code --- 18:49:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:49:59 DEBUG --- http response --- 18:49:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:49:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734461399" 18:49:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461399" 18:49:59 ERROR [http_cmd]: ERROR 18:49:59 ERROR --- status code --- 18:49:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:49:59 ERROR --- http response --- 18:49:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:49:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:49:59 INFO [http_cmd]: http status code OK 18:49:59 DEBUG --- status code --- 18:49:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:49:59 DEBUG --- http response --- 18:49:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:49:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461399" 18:49:59 INFO [http_cmd]: http status code OK 18:49:59 DEBUG --- status code --- 18:49:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:49:59 DEBUG --- http response --- 18:49:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:49:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461399" 18:49:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461399" 18:49:59 INFO [http_cmd]: http status code OK 18:49:59 DEBUG --- status code --- 18:49:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:49:59 DEBUG --- http response --- 18:49:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:49:59 INFO [http_cmd]: http status code OK 18:49:59 DEBUG --- status code --- 18:49:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:49:59 DEBUG --- http response --- 18:49:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:49:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461399" 18:49:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461399" 18:49:59 INFO [http_cmd]: http status code OK 18:49:59 DEBUG --- status code --- 18:49:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:49:59 DEBUG --- http response --- 18:49:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:49:59 INFO [http_cmd]: http status code OK 18:49:59 DEBUG --- status code --- 18:49:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:49:59 DEBUG --- http response --- 18:49:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:49:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734461399" 18:49:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461399" 18:49:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734461399" 18:49:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461399" 18:49:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734461399" 18:49:59 INFO [http_cmd]: http status code OK 18:49:59 INFO [http_cmd]: http status code OK 18:49:59 DEBUG --- status code --- 18:49:59 DEBUG --- status code --- 18:49:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:49:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:49:59 DEBUG --- http response --- 18:49:59 DEBUG --- http response --- 18:49:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:49:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:49:59 INFO [http_cmd]: http status code OK 18:49:59 DEBUG --- status code --- 18:49:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:49:59 DEBUG --- http response --- 18:49:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:49:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461399" 18:49:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461399" 18:49:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461399" 18:49:59 INFO [http_cmd]: http status code OK 18:49:59 DEBUG --- status code --- 18:49:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:49:59 INFO [http_cmd]: http status code OK 18:49:59 DEBUG --- http response --- 18:49:59 DEBUG --- status code --- 18:49:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:49:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:49:59 DEBUG --- http response --- 18:49:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:49:59 INFO [http_cmd]: http status code OK 18:49:59 DEBUG --- status code --- 18:49:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:49:59 DEBUG --- http response --- 18:49:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:49:59 INFO [http_cmd]: http status code OK 18:49:59 DEBUG --- status code --- 18:49:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:49:59 DEBUG --- http response --- 18:49:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:49:59 INFO [http_cmd]: http status code OK 18:49:59 DEBUG --- status code --- 18:49:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:49:59 DEBUG --- http response --- 18:49:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:49:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461399" 18:49:59 INFO [http_cmd]: http status code OK 18:49:59 DEBUG --- status code --- 18:49:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:49:59 DEBUG --- http response --- 18:49:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:49:59 DEBUG ----- output ----- 18:49:59 DEBUG { 18:49:59 DEBUG "status": "success", 18:49:59 DEBUG "data": { 18:49:59 DEBUG "resultType": "vector", 18:49:59 DEBUG "result": [ 18:49:59 DEBUG { 18:49:59 DEBUG "metric": { 18:49:59 DEBUG "pod": "ds-cts-0" 18:49:59 DEBUG }, 18:49:59 DEBUG "value": [ 18:49:59 DEBUG 18:49:59 DEBUG 18:49:59 DEBUG 18:49:59 DEBUG [...] 18:49:59 DEBUG 18:49:59 DEBUG 18:49:59 DEBUG 18:49:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:49:59 DEBUG }, 18:49:59 DEBUG "value": [ 18:49:59 DEBUG 1734461399, 18:49:59 DEBUG "0.01598348335222971" 18:49:59 DEBUG ] 18:49:59 DEBUG } 18:49:59 DEBUG ] 18:49:59 DEBUG } 18:49:59 DEBUG } 18:50:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461400" 18:50:00 INFO [http_cmd]: http status code OK 18:50:00 DEBUG --- status code --- 18:50:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:50:00 DEBUG --- http response --- 18:50:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:50:00 DEBUG ----- output ----- 18:50:00 DEBUG { 18:50:00 DEBUG "status": "success", 18:50:00 DEBUG "data": { 18:50:00 DEBUG "resultType": "vector", 18:50:00 DEBUG "result": [ 18:50:00 DEBUG { 18:50:00 DEBUG "metric": { 18:50:00 DEBUG "instance": "10.142.0.11:9100" 18:50:00 DEBUG }, 18:50:00 DEBUG "value": [ 18:50:00 DEBUG 18:50:00 DEBUG 18:50:00 DEBUG 18:50:00 DEBUG [...] 18:50:00 DEBUG 18:50:00 DEBUG 18:50:00 DEBUG 18:50:00 DEBUG "instance": "10.142.0.51:9100" 18:50:00 DEBUG }, 18:50:00 DEBUG "value": [ 18:50:00 DEBUG 1734461400, 18:50:00 DEBUG "87622.63333333336" 18:50:00 DEBUG ] 18:50:00 DEBUG } 18:50:00 DEBUG ] 18:50:00 DEBUG } 18:50:00 DEBUG } 18:50:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461400" 18:50:00 INFO [http_cmd]: http status code OK 18:50:00 DEBUG --- status code --- 18:50:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:50:00 DEBUG --- http response --- 18:50:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:50:00 DEBUG ----- output ----- 18:50:00 DEBUG { 18:50:00 DEBUG "status": "success", 18:50:00 DEBUG "data": { 18:50:00 DEBUG "resultType": "vector", 18:50:00 DEBUG "result": [ 18:50:00 DEBUG { 18:50:00 DEBUG "metric": { 18:50:00 DEBUG "instance": "10.142.0.11:9100" 18:50:00 DEBUG }, 18:50:00 DEBUG "value": [ 18:50:00 DEBUG 18:50:00 DEBUG 18:50:00 DEBUG 18:50:00 DEBUG [...] 18:50:00 DEBUG 18:50:00 DEBUG 18:50:00 DEBUG 18:50:00 DEBUG "instance": "10.142.0.51:9100" 18:50:00 DEBUG }, 18:50:00 DEBUG "value": [ 18:50:00 DEBUG 1734461400, 18:50:00 DEBUG "0.000666666666666771" 18:50:00 DEBUG ] 18:50:00 DEBUG } 18:50:00 DEBUG ] 18:50:00 DEBUG } 18:50:00 DEBUG } 18:50:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461400" 18:50:00 INFO [http_cmd]: http status code OK 18:50:00 DEBUG --- status code --- 18:50:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:50:00 DEBUG --- http response --- 18:50:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:50:00 DEBUG ----- output ----- 18:50:00 DEBUG { 18:50:00 DEBUG "status": "success", 18:50:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461400" 18:50:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461400" 18:50:00 DEBUG "data": { 18:50:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461400" 18:50:00 DEBUG "resultType": "vector", 18:50:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734461400" 18:50:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734461400" 18:50:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461400" 18:50:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461400" 18:50:00 DEBUG "result": [ 18:50:00 DEBUG { 18:50:00 DEBUG "metric": { 18:50:00 DEBUG "instance": "10.142.0.11:9100" 18:50:00 DEBUG }, 18:50:00 DEBUG "value": [ 18:50:00 DEBUG 18:50:00 DEBUG 18:50:00 DEBUG 18:50:00 DEBUG [...] 18:50:00 DEBUG 18:50:00 DEBUG 18:50:00 DEBUG 18:50:00 DEBUG "instance": "10.142.0.51:9100" 18:50:00 DEBUG }, 18:50:00 DEBUG "value": [ 18:50:00 DEBUG 1734461400, 18:50:00 DEBUG "0.000533333333334459" 18:50:00 DEBUG ] 18:50:00 DEBUG } 18:50:00 DEBUG ] 18:50:00 DEBUG } 18:50:00 DEBUG } 18:50:00 INFO [http_cmd]: http status code OK 18:50:00 INFO [http_cmd]: http status code OK 18:50:00 DEBUG --- status code --- 18:50:00 INFO [http_cmd]: http status code OK 18:50:00 DEBUG --- status code --- 18:50:00 DEBUG --- status code --- 18:50:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:50:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:50:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:50:00 DEBUG --- http response --- 18:50:00 DEBUG --- http response --- 18:50:00 DEBUG --- http response --- 18:50:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:50:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:50:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:50:00 DEBUG ----- output ----- 18:50:00 DEBUG ----- output ----- 18:50:00 DEBUG ----- output ----- 18:50:00 DEBUG { 18:50:00 DEBUG { 18:50:00 DEBUG { 18:50:00 DEBUG "status": "success", 18:50:00 DEBUG "data": { 18:50:00 DEBUG "status": "success", 18:50:00 DEBUG "status": "success", 18:50:00 DEBUG "resultType": "vector", 18:50:00 DEBUG "data": { 18:50:00 DEBUG "resultType": "vector", 18:50:00 DEBUG "result": [ 18:50:00 DEBUG "data": { 18:50:00 DEBUG "result": [ 18:50:00 DEBUG { 18:50:00 DEBUG "resultType": "vector", 18:50:00 DEBUG { 18:50:00 DEBUG "metric": { 18:50:00 DEBUG "result": [ 18:50:00 DEBUG "metric": { 18:50:00 DEBUG "instance": "10.142.0.11:9100" 18:50:00 DEBUG { 18:50:00 DEBUG "instance": "10.142.0.11:9100" 18:50:00 DEBUG }, 18:50:00 DEBUG "metric": { 18:50:00 DEBUG }, 18:50:00 DEBUG "value": [ 18:50:00 DEBUG "pod": "ds-cts-0" 18:50:00 INFO [http_cmd]: http status code OK 18:50:00 DEBUG "value": [ 18:50:00 DEBUG 18:50:00 DEBUG }, 18:50:00 DEBUG --- status code --- 18:50:00 DEBUG 18:50:00 DEBUG 18:50:00 DEBUG "value": [ 18:50:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:50:00 DEBUG 18:50:00 DEBUG 18:50:00 DEBUG 18:50:00 DEBUG --- http response --- 18:50:00 DEBUG 18:50:00 DEBUG [...] 18:50:00 INFO [http_cmd]: http status code OK 18:50:00 DEBUG 18:50:00 INFO [http_cmd]: http status code OK 18:50:00 DEBUG [...] 18:50:00 DEBUG 18:50:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:50:00 DEBUG --- status code --- 18:50:00 DEBUG 18:50:00 DEBUG --- status code --- 18:50:00 DEBUG 18:50:00 DEBUG 18:50:00 DEBUG ----- output ----- 18:50:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:50:00 DEBUG [...] 18:50:00 DEBUG 18:50:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:50:00 DEBUG 18:50:00 DEBUG 18:50:00 DEBUG { 18:50:00 DEBUG --- http response --- 18:50:00 INFO [http_cmd]: http status code OK 18:50:00 DEBUG 18:50:00 DEBUG --- http response --- 18:50:00 DEBUG 18:50:00 DEBUG "instance": "10.142.0.51:9100" 18:50:00 DEBUG "status": "success", 18:50:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:50:00 DEBUG --- status code --- 18:50:00 DEBUG 18:50:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:50:00 DEBUG "instance": "10.142.0.51:9100" 18:50:00 DEBUG }, 18:50:00 DEBUG "value": [ 18:50:00 DEBUG ----- output ----- 18:50:00 DEBUG { 18:50:00 DEBUG "status": "success", 18:50:00 DEBUG ----- output ----- 18:50:00 DEBUG }, 18:50:00 DEBUG "data": { 18:50:00 DEBUG 1734461400, 18:50:00 DEBUG "78551.60000000002" 18:50:00 DEBUG ] 18:50:00 DEBUG "data": { 18:50:00 DEBUG { 18:50:00 DEBUG "status": "success", 18:50:00 DEBUG "data": { 18:50:00 DEBUG "resultType": "vector", 18:50:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:50:00 DEBUG } 18:50:00 DEBUG "resultType": "vector", 18:50:00 DEBUG "value": [ 18:50:00 DEBUG "resultType": "vector", 18:50:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:50:00 DEBUG "result": [ 18:50:00 DEBUG { 18:50:00 DEBUG ] 18:50:00 DEBUG "result": [ 18:50:00 DEBUG 1734461400, 18:50:00 DEBUG "result": [ 18:50:00 DEBUG }, 18:50:00 DEBUG --- http response --- 18:50:00 DEBUG "metric": { 18:50:00 DEBUG } 18:50:00 DEBUG { 18:50:00 DEBUG "16793.600000000002" 18:50:00 DEBUG { 18:50:00 DEBUG "value": [ 18:50:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:50:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:50:00 DEBUG } 18:50:00 DEBUG "metric": { 18:50:00 DEBUG ] 18:50:00 DEBUG "metric": { 18:50:00 DEBUG 1734461400, 18:50:00 DEBUG ----- output ----- 18:50:00 DEBUG }, 18:50:00 DEBUG "pod": "ds-cts-0" 18:50:00 DEBUG } 18:50:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:50:00 DEBUG "0" 18:50:00 DEBUG { 18:50:00 DEBUG "value": [ 18:50:00 DEBUG }, 18:50:00 DEBUG ] 18:50:00 DEBUG }, 18:50:00 DEBUG ] 18:50:00 DEBUG "status": "success", 18:50:00 DEBUG 18:50:00 DEBUG "value": [ 18:50:00 DEBUG } 18:50:00 DEBUG "value": [ 18:50:00 DEBUG } 18:50:00 DEBUG "data": { 18:50:00 DEBUG 18:50:00 DEBUG 18:50:00 DEBUG } 18:50:00 DEBUG 18:50:00 DEBUG ] 18:50:00 DEBUG "resultType": "vector", 18:50:00 DEBUG 18:50:00 DEBUG 18:50:00 DEBUG 18:50:00 DEBUG } 18:50:00 DEBUG "result": [ 18:50:00 DEBUG [...] 18:50:00 DEBUG 18:50:00 DEBUG 18:50:00 DEBUG } 18:50:00 DEBUG { 18:50:00 DEBUG 18:50:00 DEBUG [...] 18:50:00 DEBUG [...] 18:50:00 DEBUG "metric": { 18:50:00 DEBUG 18:50:00 DEBUG 18:50:00 DEBUG 18:50:00 DEBUG "pod": "ds-cts-0" 18:50:00 DEBUG 18:50:00 DEBUG 18:50:00 DEBUG 18:50:00 DEBUG }, 18:50:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:50:00 DEBUG 18:50:00 DEBUG 18:50:00 DEBUG "value": [ 18:50:00 DEBUG }, 18:50:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:50:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:50:00 DEBUG 18:50:00 DEBUG "value": [ 18:50:00 DEBUG }, 18:50:00 DEBUG }, 18:50:00 DEBUG 18:50:00 DEBUG 1734461400, 18:50:00 DEBUG "value": [ 18:50:00 DEBUG "value": [ 18:50:00 DEBUG 18:50:00 DEBUG "1.7813120369032927" 18:50:00 DEBUG 1734461400, 18:50:00 DEBUG 1734461400, 18:50:00 DEBUG [...] 18:50:00 DEBUG ] 18:50:00 DEBUG "714305536" 18:50:00 DEBUG "2532622336" 18:50:00 DEBUG 18:50:00 DEBUG } 18:50:00 DEBUG ] 18:50:00 DEBUG ] 18:50:00 DEBUG 18:50:00 DEBUG ] 18:50:00 DEBUG } 18:50:00 DEBUG } 18:50:00 DEBUG 18:50:00 DEBUG } 18:50:00 DEBUG ] 18:50:00 DEBUG ] 18:50:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:50:00 DEBUG } 18:50:00 DEBUG } 18:50:00 DEBUG } 18:50:00 DEBUG }, 18:50:00 DEBUG } 18:50:00 DEBUG } 18:50:00 DEBUG "value": [ 18:50:00 DEBUG 1734461400, 18:50:00 DEBUG "1294.3431434587512" 18:50:00 DEBUG ] 18:50:00 DEBUG } 18:50:00 DEBUG ] 18:50:00 DEBUG } 18:50:00 DEBUG } 18:50:02 INFO 18:50:02 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:50:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:50:02 INFO [loop_until]: OK (rc = 0) 18:50:02 DEBUG --- stdout --- 18:50:02 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 544Mi ds-idrepo-0 7m 732Mi rcs-5d696ff5b-7ttvr 3m 296Mi rcs-5d696ff5b-wwlbr 13m 340Mi 18:50:02 DEBUG --- stderr --- 18:50:02 DEBUG 18:50:02 INFO 18:50:02 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:50:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:50:02 INFO [loop_until]: OK (rc = 0) 18:50:02 DEBUG --- stdout --- 18:50:02 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 113m 0% 2302Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1851m 5% 3624Mi 12% 18:50:02 DEBUG --- stderr --- 18:50:02 DEBUG 18:50:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461459" 18:50:59 INFO [http_cmd]: http status code OK 18:50:59 DEBUG --- status code --- 18:50:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:50:59 DEBUG --- http response --- 18:50:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:50:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461459" 18:50:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734461459" 18:50:59 INFO [http_cmd]: http status code OK 18:50:59 DEBUG --- status code --- 18:50:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:50:59 DEBUG --- http response --- 18:50:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:50:59 INFO [http_cmd]: http status code OK 18:50:59 DEBUG --- status code --- 18:50:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:50:59 DEBUG --- http response --- 18:50:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:50:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461459" 18:50:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461459" 18:50:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461459" 18:50:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461459" 18:50:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461459" 18:50:59 INFO [http_cmd]: http status code OK 18:50:59 DEBUG --- status code --- 18:50:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:50:59 DEBUG --- http response --- 18:50:59 INFO [http_cmd]: http status code OK 18:50:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:50:59 DEBUG --- status code --- 18:50:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:50:59 DEBUG --- http response --- 18:50:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:50:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461459" 18:50:59 INFO [http_cmd]: http status code OK 18:50:59 DEBUG --- status code --- 18:50:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:50:59 DEBUG --- http response --- 18:50:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:50:59 INFO [http_cmd]: http status code OK 18:50:59 DEBUG --- status code --- 18:50:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:50:59 DEBUG --- http response --- 18:50:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:50:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461459" 18:50:59 INFO [http_cmd]: http status code OK 18:50:59 DEBUG --- status code --- 18:50:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:50:59 DEBUG --- http response --- 18:50:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:50:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461459" 18:50:59 INFO [http_cmd]: http status code OK 18:50:59 DEBUG --- status code --- 18:50:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:50:59 DEBUG --- http response --- 18:50:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:50:59 INFO [http_cmd]: http status code OK 18:50:59 DEBUG --- status code --- 18:50:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:50:59 DEBUG --- http response --- 18:50:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:50:59 INFO [http_cmd]: http status code OK 18:50:59 DEBUG --- status code --- 18:50:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:50:59 DEBUG --- http response --- 18:50:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:50:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461459" 18:50:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734461459" 18:50:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461459" 18:50:59 INFO [http_cmd]: http status code OK 18:50:59 DEBUG --- status code --- 18:50:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:50:59 DEBUG --- http response --- 18:50:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:50:59 INFO [http_cmd]: http status code OK 18:50:59 DEBUG --- status code --- 18:50:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:50:59 DEBUG --- http response --- 18:50:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:50:59 ERROR [http_cmd]: ERROR 18:50:59 ERROR --- status code --- 18:50:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:50:59 ERROR --- http response --- 18:50:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:50:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:50:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461459" 18:50:59 INFO [http_cmd]: http status code OK 18:50:59 DEBUG --- status code --- 18:50:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:50:59 DEBUG --- http response --- 18:50:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:50:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461459" 18:50:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461459" 18:50:59 INFO [http_cmd]: http status code OK 18:50:59 DEBUG --- status code --- 18:50:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:50:59 DEBUG --- http response --- 18:50:59 INFO [http_cmd]: http status code OK 18:50:59 DEBUG --- status code --- 18:50:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:50:59 DEBUG --- http response --- 18:50:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:50:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:50:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461459" 18:50:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461459" 18:50:59 INFO [http_cmd]: http status code OK 18:50:59 DEBUG --- status code --- 18:50:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:50:59 DEBUG --- http response --- 18:50:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:50:59 INFO [http_cmd]: http status code OK 18:50:59 DEBUG --- status code --- 18:50:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:50:59 DEBUG --- http response --- 18:50:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:50:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734461459" 18:50:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461459" 18:50:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734461459" 18:50:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461459" 18:50:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734461459" 18:50:59 INFO [http_cmd]: http status code OK 18:50:59 DEBUG --- status code --- 18:50:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:50:59 DEBUG --- http response --- 18:50:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:50:59 INFO [http_cmd]: http status code OK 18:50:59 DEBUG --- status code --- 18:50:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:50:59 DEBUG --- http response --- 18:50:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:50:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461459" 18:50:59 INFO [http_cmd]: http status code OK 18:50:59 DEBUG --- status code --- 18:50:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:50:59 DEBUG --- http response --- 18:50:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:50:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461459" 18:50:59 INFO [http_cmd]: http status code OK 18:50:59 DEBUG --- status code --- 18:50:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:50:59 INFO [http_cmd]: http status code OK 18:50:59 DEBUG --- http response --- 18:50:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461459" 18:50:59 DEBUG --- status code --- 18:50:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:50:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:50:59 DEBUG --- http response --- 18:50:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:50:59 INFO [http_cmd]: http status code OK 18:50:59 DEBUG --- status code --- 18:50:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:50:59 DEBUG --- http response --- 18:50:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:50:59 INFO [http_cmd]: http status code OK 18:50:59 DEBUG --- status code --- 18:50:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:50:59 DEBUG --- http response --- 18:50:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:50:59 INFO [http_cmd]: http status code OK 18:50:59 DEBUG --- status code --- 18:50:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:50:59 DEBUG --- http response --- 18:50:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:50:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461459" 18:50:59 INFO [http_cmd]: http status code OK 18:50:59 DEBUG --- status code --- 18:50:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:50:59 DEBUG --- http response --- 18:50:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:50:59 DEBUG ----- output ----- 18:50:59 DEBUG { 18:50:59 DEBUG "status": "success", 18:50:59 DEBUG "data": { 18:50:59 DEBUG "resultType": "vector", 18:50:59 DEBUG "result": [ 18:50:59 DEBUG { 18:50:59 DEBUG "metric": { 18:50:59 DEBUG "pod": "ds-cts-0" 18:50:59 DEBUG }, 18:50:59 DEBUG "value": [ 18:50:59 DEBUG 18:50:59 DEBUG 18:50:59 DEBUG 18:50:59 DEBUG [...] 18:50:59 DEBUG 18:50:59 DEBUG 18:50:59 DEBUG 18:50:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:50:59 DEBUG }, 18:50:59 DEBUG "value": [ 18:50:59 DEBUG 1734461459, 18:50:59 DEBUG "0.016110922107227393" 18:50:59 DEBUG ] 18:50:59 DEBUG } 18:50:59 DEBUG ] 18:50:59 DEBUG } 18:50:59 DEBUG } 18:51:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461460" 18:51:00 INFO [http_cmd]: http status code OK 18:51:00 DEBUG --- status code --- 18:51:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:51:00 DEBUG --- http response --- 18:51:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:51:00 DEBUG ----- output ----- 18:51:00 DEBUG { 18:51:00 DEBUG "status": "success", 18:51:00 DEBUG "data": { 18:51:00 DEBUG "resultType": "vector", 18:51:00 DEBUG "result": [ 18:51:00 DEBUG { 18:51:00 DEBUG "metric": { 18:51:00 DEBUG "instance": "10.142.0.11:9100" 18:51:00 DEBUG }, 18:51:00 DEBUG "value": [ 18:51:00 DEBUG 18:51:00 DEBUG 18:51:00 DEBUG 18:51:00 DEBUG [...] 18:51:00 DEBUG 18:51:00 DEBUG 18:51:00 DEBUG 18:51:00 DEBUG "instance": "10.142.0.51:9100" 18:51:00 DEBUG }, 18:51:00 DEBUG "value": [ 18:51:00 DEBUG 1734461460, 18:51:00 DEBUG "87934.53333333337" 18:51:00 DEBUG ] 18:51:00 DEBUG } 18:51:00 DEBUG ] 18:51:00 DEBUG } 18:51:00 DEBUG } 18:51:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461460" 18:51:00 INFO [http_cmd]: http status code OK 18:51:00 DEBUG --- status code --- 18:51:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:51:00 DEBUG --- http response --- 18:51:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:51:00 DEBUG ----- output ----- 18:51:00 DEBUG { 18:51:00 DEBUG "status": "success", 18:51:00 DEBUG "data": { 18:51:00 DEBUG "resultType": "vector", 18:51:00 DEBUG "result": [ 18:51:00 DEBUG { 18:51:00 DEBUG "metric": { 18:51:00 DEBUG "instance": "10.142.0.11:9100" 18:51:00 DEBUG }, 18:51:00 DEBUG "value": [ 18:51:00 DEBUG 18:51:00 DEBUG 18:51:00 DEBUG 18:51:00 DEBUG [...] 18:51:00 DEBUG 18:51:00 DEBUG 18:51:00 DEBUG 18:51:00 DEBUG "instance": "10.142.0.51:9100" 18:51:00 DEBUG }, 18:51:00 DEBUG "value": [ 18:51:00 DEBUG 1734461460, 18:51:00 DEBUG "0.000666666666666771" 18:51:00 DEBUG ] 18:51:00 DEBUG } 18:51:00 DEBUG ] 18:51:00 DEBUG } 18:51:00 DEBUG } 18:51:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461460" 18:51:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461460" 18:51:00 INFO [http_cmd]: http status code OK 18:51:00 DEBUG --- status code --- 18:51:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:51:00 DEBUG --- http response --- 18:51:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:51:00 DEBUG ----- output ----- 18:51:00 DEBUG { 18:51:00 DEBUG "status": "success", 18:51:00 DEBUG "data": { 18:51:00 DEBUG "resultType": "vector", 18:51:00 DEBUG "result": [ 18:51:00 DEBUG { 18:51:00 DEBUG "metric": { 18:51:00 DEBUG "instance": "10.142.0.11:9100" 18:51:00 DEBUG }, 18:51:00 DEBUG "value": [ 18:51:00 DEBUG 18:51:00 DEBUG 18:51:00 DEBUG 18:51:00 DEBUG [...] 18:51:00 DEBUG 18:51:00 DEBUG 18:51:00 DEBUG 18:51:00 DEBUG "instance": "10.142.0.51:9100" 18:51:00 DEBUG }, 18:51:00 DEBUG "value": [ 18:51:00 DEBUG 1734461460, 18:51:00 DEBUG "0.00043333333333634994" 18:51:00 DEBUG ] 18:51:00 DEBUG } 18:51:00 DEBUG ] 18:51:00 DEBUG } 18:51:00 DEBUG } 18:51:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461460" 18:51:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461460" 18:51:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734461460" 18:51:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461460" 18:51:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461460" 18:51:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734461460" 18:51:00 INFO [http_cmd]: http status code OK 18:51:00 DEBUG --- status code --- 18:51:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:51:00 DEBUG --- http response --- 18:51:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:51:00 DEBUG ----- output ----- 18:51:00 DEBUG { 18:51:00 DEBUG "status": "success", 18:51:00 DEBUG "data": { 18:51:00 DEBUG "resultType": "vector", 18:51:00 DEBUG "result": [ 18:51:00 DEBUG { 18:51:00 DEBUG "metric": { 18:51:00 DEBUG "pod": "ds-cts-0" 18:51:00 DEBUG }, 18:51:00 DEBUG "value": [ 18:51:00 DEBUG 18:51:00 DEBUG 18:51:00 DEBUG 18:51:00 DEBUG [...] 18:51:00 INFO [http_cmd]: http status code OK 18:51:00 INFO [http_cmd]: http status code OK 18:51:00 DEBUG 18:51:00 DEBUG --- status code --- 18:51:00 DEBUG --- status code --- 18:51:00 DEBUG 18:51:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:51:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:51:00 DEBUG 18:51:00 DEBUG --- http response --- 18:51:00 DEBUG --- http response --- 18:51:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:51:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:51:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:51:00 DEBUG }, 18:51:00 DEBUG ----- output ----- 18:51:00 DEBUG ----- output ----- 18:51:00 DEBUG "value": [ 18:51:00 DEBUG { 18:51:00 DEBUG { 18:51:00 DEBUG 1734461460, 18:51:00 DEBUG "status": "success", 18:51:00 DEBUG "status": "success", 18:51:00 INFO [http_cmd]: http status code OK 18:51:00 INFO [http_cmd]: http status code OK 18:51:00 DEBUG "0" 18:51:00 DEBUG "data": { 18:51:00 DEBUG "data": { 18:51:00 DEBUG --- status code --- 18:51:00 DEBUG --- status code --- 18:51:00 DEBUG ] 18:51:00 DEBUG "resultType": "vector", 18:51:00 DEBUG "resultType": "vector", 18:51:00 INFO [http_cmd]: http status code OK 18:51:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:51:00 INFO [http_cmd]: http status code OK 18:51:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:51:00 DEBUG } 18:51:00 DEBUG "result": [ 18:51:00 DEBUG "result": [ 18:51:00 DEBUG --- status code --- 18:51:00 DEBUG --- http response --- 18:51:00 DEBUG --- status code --- 18:51:00 DEBUG --- http response --- 18:51:00 DEBUG ] 18:51:00 DEBUG { 18:51:00 DEBUG { 18:51:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:51:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:51:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:51:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:51:00 DEBUG } 18:51:00 DEBUG "metric": { 18:51:00 DEBUG "metric": { 18:51:00 DEBUG --- http response --- 18:51:00 DEBUG ----- output ----- 18:51:00 DEBUG --- http response --- 18:51:00 DEBUG ----- output ----- 18:51:00 DEBUG } 18:51:00 DEBUG "instance": "10.142.0.11:9100" 18:51:00 DEBUG "instance": "10.142.0.11:9100" 18:51:00 DEBUG { 18:51:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:51:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:51:00 DEBUG { 18:51:00 DEBUG }, 18:51:00 DEBUG }, 18:51:00 DEBUG "status": "success", 18:51:00 DEBUG ----- output ----- 18:51:00 DEBUG ----- output ----- 18:51:00 DEBUG "status": "success", 18:51:00 DEBUG "value": [ 18:51:00 DEBUG "value": [ 18:51:00 DEBUG "data": { 18:51:00 DEBUG { 18:51:00 DEBUG { 18:51:00 DEBUG "data": { 18:51:00 DEBUG 18:51:00 DEBUG 18:51:00 DEBUG "resultType": "vector", 18:51:00 DEBUG "status": "success", 18:51:00 DEBUG "status": "success", 18:51:00 DEBUG "resultType": "vector", 18:51:00 DEBUG 18:51:00 DEBUG 18:51:00 DEBUG "result": [ 18:51:00 DEBUG "data": { 18:51:00 DEBUG "data": { 18:51:00 DEBUG "result": [ 18:51:00 DEBUG 18:51:00 DEBUG 18:51:00 DEBUG { 18:51:00 DEBUG "resultType": "vector", 18:51:00 DEBUG "resultType": "vector", 18:51:00 DEBUG { 18:51:00 DEBUG [...] 18:51:00 DEBUG [...] 18:51:00 DEBUG "metric": { 18:51:00 DEBUG "result": [ 18:51:00 DEBUG "result": [ 18:51:00 DEBUG "metric": { 18:51:00 DEBUG 18:51:00 DEBUG 18:51:00 DEBUG "pod": "ds-cts-0" 18:51:00 DEBUG { 18:51:00 DEBUG "metric": { 18:51:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:51:00 DEBUG 18:51:00 DEBUG 18:51:00 DEBUG }, 18:51:00 DEBUG "value": [ 18:51:00 DEBUG "pod": "ds-cts-0" 18:51:00 DEBUG }, 18:51:00 DEBUG 18:51:00 DEBUG 18:51:00 DEBUG { 18:51:00 DEBUG 18:51:00 DEBUG }, 18:51:00 DEBUG "value": [ 18:51:00 DEBUG "instance": "10.142.0.51:9100" 18:51:00 DEBUG "instance": "10.142.0.51:9100" 18:51:00 DEBUG "metric": { 18:51:00 DEBUG 18:51:00 DEBUG "value": [ 18:51:00 DEBUG 18:51:00 DEBUG }, 18:51:00 DEBUG }, 18:51:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:51:00 DEBUG 18:51:00 DEBUG 18:51:00 DEBUG 18:51:00 DEBUG "value": [ 18:51:00 DEBUG "value": [ 18:51:00 DEBUG }, 18:51:00 DEBUG [...] 18:51:00 DEBUG 18:51:00 DEBUG 18:51:00 DEBUG 1734461460, 18:51:00 DEBUG 1734461460, 18:51:00 DEBUG "value": [ 18:51:00 DEBUG 18:51:00 DEBUG 18:51:00 DEBUG [...] 18:51:00 DEBUG "24029.866666666672" 18:51:00 DEBUG "72380.63333333336" 18:51:00 DEBUG 18:51:00 DEBUG 18:51:00 DEBUG [...] 18:51:00 DEBUG 18:51:00 DEBUG ] 18:51:00 DEBUG ] 18:51:00 DEBUG 18:51:00 DEBUG 18:51:00 DEBUG 18:51:00 DEBUG 18:51:00 DEBUG } 18:51:00 DEBUG } 18:51:00 DEBUG 18:51:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:51:00 DEBUG 18:51:00 DEBUG 18:51:00 DEBUG ] 18:51:00 DEBUG ] 18:51:00 DEBUG [...] 18:51:00 DEBUG }, 18:51:00 DEBUG 18:51:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:51:00 DEBUG } 18:51:00 DEBUG } 18:51:00 DEBUG 18:51:00 DEBUG "value": [ 18:51:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:51:00 DEBUG }, 18:51:00 DEBUG } 18:51:00 DEBUG } 18:51:00 DEBUG 18:51:00 DEBUG 1734461460, 18:51:00 DEBUG }, 18:51:00 DEBUG "value": [ 18:51:00 DEBUG 18:51:00 DEBUG "1165.8309102597395" 18:51:00 DEBUG "value": [ 18:51:00 DEBUG 1734461460, 18:51:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:51:00 DEBUG ] 18:51:00 DEBUG 1734461460, 18:51:00 DEBUG "695111680" 18:51:00 DEBUG }, 18:51:00 DEBUG } 18:51:00 DEBUG "2550956032" 18:51:00 DEBUG ] 18:51:00 DEBUG "value": [ 18:51:00 DEBUG ] 18:51:00 DEBUG ] 18:51:00 DEBUG } 18:51:00 DEBUG 1734461460, 18:51:00 DEBUG } 18:51:00 DEBUG } 18:51:00 DEBUG ] 18:51:00 DEBUG "1.769155403621873" 18:51:00 DEBUG } 18:51:00 DEBUG ] 18:51:00 DEBUG } 18:51:00 DEBUG ] 18:51:00 DEBUG } 18:51:00 DEBUG } 18:51:00 DEBUG } 18:51:00 DEBUG } 18:51:00 DEBUG ] 18:51:00 DEBUG } 18:51:00 DEBUG } 18:51:02 INFO 18:51:02 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:51:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:51:02 INFO [loop_until]: OK (rc = 0) 18:51:02 DEBUG --- stdout --- 18:51:02 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 7m 543Mi ds-idrepo-0 7m 734Mi rcs-5d696ff5b-7ttvr 5m 297Mi rcs-5d696ff5b-wwlbr 6m 331Mi 18:51:02 DEBUG --- stderr --- 18:51:02 DEBUG 18:51:02 INFO 18:51:02 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:51:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:51:02 INFO [loop_until]: OK (rc = 0) 18:51:02 DEBUG --- stdout --- 18:51:02 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 105m 0% 2287Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1848m 5% 3626Mi 12% 18:51:02 DEBUG --- stderr --- 18:51:02 DEBUG 18:51:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461519" 18:51:59 INFO [http_cmd]: http status code OK 18:51:59 DEBUG --- status code --- 18:51:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:51:59 DEBUG --- http response --- 18:51:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:51:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461519" 18:51:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734461519" 18:51:59 INFO [http_cmd]: http status code OK 18:51:59 DEBUG --- status code --- 18:51:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:51:59 DEBUG --- http response --- 18:51:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:51:59 INFO [http_cmd]: http status code OK 18:51:59 DEBUG --- status code --- 18:51:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:51:59 DEBUG --- http response --- 18:51:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:51:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461519" 18:51:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461519" 18:51:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461519" 18:51:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461519" 18:51:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461519" 18:51:59 INFO [http_cmd]: http status code OK 18:51:59 INFO [http_cmd]: http status code OK 18:51:59 DEBUG --- status code --- 18:51:59 INFO [http_cmd]: http status code OK 18:51:59 DEBUG --- status code --- 18:51:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:51:59 DEBUG --- status code --- 18:51:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:51:59 DEBUG --- http response --- 18:51:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:51:59 DEBUG --- http response --- 18:51:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:51:59 DEBUG --- http response --- 18:51:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:51:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:51:59 INFO [http_cmd]: http status code OK 18:51:59 DEBUG --- status code --- 18:51:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:51:59 DEBUG --- http response --- 18:51:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:51:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461519" 18:51:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461519" 18:51:59 INFO [http_cmd]: http status code OK 18:51:59 DEBUG --- status code --- 18:51:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:51:59 DEBUG --- http response --- 18:51:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:51:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461519" 18:51:59 INFO [http_cmd]: http status code OK 18:51:59 DEBUG --- status code --- 18:51:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:51:59 DEBUG --- http response --- 18:51:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:51:59 INFO [http_cmd]: http status code OK 18:51:59 DEBUG --- status code --- 18:51:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:51:59 DEBUG --- http response --- 18:51:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:51:59 INFO [http_cmd]: http status code OK 18:51:59 DEBUG --- status code --- 18:51:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:51:59 DEBUG --- http response --- 18:51:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:51:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461519" 18:51:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734461519" 18:51:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461519" 18:51:59 INFO [http_cmd]: http status code OK 18:51:59 DEBUG --- status code --- 18:51:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:51:59 DEBUG --- http response --- 18:51:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:51:59 INFO [http_cmd]: http status code OK 18:51:59 DEBUG --- status code --- 18:51:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:51:59 ERROR [http_cmd]: ERROR 18:51:59 ERROR --- status code --- 18:51:59 DEBUG --- http response --- 18:51:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:51:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:51:59 ERROR --- http response --- 18:51:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:51:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:51:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461519" 18:51:59 INFO [http_cmd]: http status code OK 18:51:59 DEBUG --- status code --- 18:51:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:51:59 DEBUG --- http response --- 18:51:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:51:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461519" 18:51:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461519" 18:51:59 INFO [http_cmd]: http status code OK 18:51:59 DEBUG --- status code --- 18:51:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:51:59 DEBUG --- http response --- 18:51:59 INFO [http_cmd]: http status code OK 18:51:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:51:59 DEBUG --- status code --- 18:51:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:51:59 DEBUG --- http response --- 18:51:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:51:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461519" 18:51:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461519" 18:51:59 INFO [http_cmd]: http status code OK 18:51:59 DEBUG --- status code --- 18:51:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:51:59 DEBUG --- http response --- 18:51:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:51:59 INFO [http_cmd]: http status code OK 18:51:59 DEBUG --- status code --- 18:51:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:51:59 DEBUG --- http response --- 18:51:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:51:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734461519" 18:51:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461519" 18:51:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734461519" 18:51:59 INFO [http_cmd]: http status code OK 18:51:59 DEBUG --- status code --- 18:51:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:51:59 DEBUG --- http response --- 18:51:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:51:59 INFO [http_cmd]: http status code OK 18:51:59 DEBUG --- status code --- 18:51:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:51:59 DEBUG --- http response --- 18:51:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:51:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461519" 18:51:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734461519" 18:51:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461519" 18:51:59 INFO [http_cmd]: http status code OK 18:51:59 DEBUG --- status code --- 18:51:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:51:59 DEBUG --- http response --- 18:51:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:51:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461519" 18:51:59 INFO [http_cmd]: http status code OK 18:51:59 DEBUG --- status code --- 18:51:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:51:59 DEBUG --- http response --- 18:51:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:51:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461519" 18:51:59 INFO [http_cmd]: http status code OK 18:51:59 DEBUG --- status code --- 18:51:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:51:59 DEBUG --- http response --- 18:51:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:51:59 INFO [http_cmd]: http status code OK 18:51:59 DEBUG --- status code --- 18:51:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:51:59 DEBUG --- http response --- 18:51:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:51:59 INFO [http_cmd]: http status code OK 18:51:59 DEBUG --- status code --- 18:51:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:51:59 DEBUG --- http response --- 18:51:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:51:59 INFO [http_cmd]: http status code OK 18:51:59 DEBUG --- status code --- 18:51:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:51:59 DEBUG --- http response --- 18:51:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:51:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461519" 18:51:59 INFO [http_cmd]: http status code OK 18:51:59 DEBUG --- status code --- 18:51:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:51:59 DEBUG --- http response --- 18:51:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:51:59 DEBUG ----- output ----- 18:51:59 DEBUG { 18:51:59 DEBUG "status": "success", 18:51:59 DEBUG "data": { 18:51:59 DEBUG "resultType": "vector", 18:51:59 DEBUG "result": [ 18:51:59 DEBUG { 18:51:59 DEBUG "metric": { 18:51:59 DEBUG "pod": "ds-cts-0" 18:51:59 DEBUG }, 18:51:59 DEBUG "value": [ 18:51:59 DEBUG 18:51:59 DEBUG 18:51:59 DEBUG 18:51:59 DEBUG [...] 18:51:59 DEBUG 18:51:59 DEBUG 18:51:59 DEBUG 18:51:59 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:51:59 DEBUG }, 18:51:59 DEBUG "value": [ 18:51:59 DEBUG 1734461519, 18:51:59 DEBUG "0.006595257110744634" 18:51:59 DEBUG ] 18:51:59 DEBUG } 18:51:59 DEBUG ] 18:51:59 DEBUG } 18:51:59 DEBUG } 18:52:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461520" 18:52:00 INFO [http_cmd]: http status code OK 18:52:00 DEBUG --- status code --- 18:52:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:52:00 DEBUG --- http response --- 18:52:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:52:00 DEBUG ----- output ----- 18:52:00 DEBUG { 18:52:00 DEBUG "status": "success", 18:52:00 DEBUG "data": { 18:52:00 DEBUG "resultType": "vector", 18:52:00 DEBUG "result": [ 18:52:00 DEBUG { 18:52:00 DEBUG "metric": { 18:52:00 DEBUG "instance": "10.142.0.11:9100" 18:52:00 DEBUG }, 18:52:00 DEBUG "value": [ 18:52:00 DEBUG 18:52:00 DEBUG 18:52:00 DEBUG 18:52:00 DEBUG [...] 18:52:00 DEBUG 18:52:00 DEBUG 18:52:00 DEBUG 18:52:00 DEBUG "instance": "10.142.0.51:9100" 18:52:00 DEBUG }, 18:52:00 DEBUG "value": [ 18:52:00 DEBUG 1734461520, 18:52:00 DEBUG "88669.50000000001" 18:52:00 DEBUG ] 18:52:00 DEBUG } 18:52:00 DEBUG ] 18:52:00 DEBUG } 18:52:00 DEBUG } 18:52:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461520" 18:52:00 INFO [http_cmd]: http status code OK 18:52:00 DEBUG --- status code --- 18:52:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:52:00 DEBUG --- http response --- 18:52:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:52:00 DEBUG ----- output ----- 18:52:00 DEBUG { 18:52:00 DEBUG "status": "success", 18:52:00 DEBUG "data": { 18:52:00 DEBUG "resultType": "vector", 18:52:00 DEBUG "result": [ 18:52:00 DEBUG { 18:52:00 DEBUG "metric": { 18:52:00 DEBUG "instance": "10.142.0.11:9100" 18:52:00 DEBUG }, 18:52:00 DEBUG "value": [ 18:52:00 DEBUG 18:52:00 DEBUG 18:52:00 DEBUG 18:52:00 DEBUG [...] 18:52:00 DEBUG 18:52:00 DEBUG 18:52:00 DEBUG 18:52:00 DEBUG "instance": "10.142.0.51:9100" 18:52:00 DEBUG }, 18:52:00 DEBUG "value": [ 18:52:00 DEBUG 1734461520, 18:52:00 DEBUG "0" 18:52:00 DEBUG ] 18:52:00 DEBUG } 18:52:00 DEBUG ] 18:52:00 DEBUG } 18:52:00 DEBUG } 18:52:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461520" 18:52:00 INFO [http_cmd]: http status code OK 18:52:00 DEBUG --- status code --- 18:52:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:52:00 DEBUG --- http response --- 18:52:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:52:00 DEBUG ----- output ----- 18:52:00 DEBUG { 18:52:00 DEBUG "status": "success", 18:52:00 DEBUG "data": { 18:52:00 DEBUG "resultType": "vector", 18:52:00 DEBUG "result": [ 18:52:00 DEBUG { 18:52:00 DEBUG "metric": { 18:52:00 DEBUG "instance": "10.142.0.11:9100" 18:52:00 DEBUG }, 18:52:00 DEBUG "value": [ 18:52:00 DEBUG 18:52:00 DEBUG 18:52:00 DEBUG 18:52:00 DEBUG [...] 18:52:00 DEBUG 18:52:00 DEBUG 18:52:00 DEBUG 18:52:00 DEBUG "instance": "10.142.0.51:9100" 18:52:00 DEBUG }, 18:52:00 DEBUG "value": [ 18:52:00 DEBUG 1734461520, 18:52:00 DEBUG "0.0005333333333330378" 18:52:00 DEBUG ] 18:52:00 DEBUG } 18:52:00 DEBUG ] 18:52:00 DEBUG } 18:52:00 DEBUG } 18:52:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461520" 18:52:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461520" 18:52:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461520" 18:52:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461520" 18:52:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734461520" 18:52:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461520" 18:52:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734461520" 18:52:00 INFO [http_cmd]: http status code OK 18:52:00 DEBUG --- status code --- 18:52:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:52:00 DEBUG --- http response --- 18:52:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:52:00 DEBUG ----- output ----- 18:52:00 DEBUG { 18:52:00 DEBUG "status": "success", 18:52:00 DEBUG "data": { 18:52:00 DEBUG "resultType": "vector", 18:52:00 DEBUG "result": [ 18:52:00 DEBUG { 18:52:00 DEBUG "metric": { 18:52:00 DEBUG "pod": "ds-cts-0" 18:52:00 DEBUG }, 18:52:00 DEBUG "value": [ 18:52:00 DEBUG 18:52:00 DEBUG 18:52:00 DEBUG 18:52:00 DEBUG [...] 18:52:00 DEBUG 18:52:00 DEBUG 18:52:00 DEBUG 18:52:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:52:00 DEBUG }, 18:52:00 DEBUG "value": [ 18:52:00 DEBUG 1734461520, 18:52:00 INFO [http_cmd]: http status code OK 18:52:00 INFO [http_cmd]: http status code OK 18:52:00 DEBUG "0" 18:52:00 DEBUG --- status code --- 18:52:00 DEBUG --- status code --- 18:52:00 DEBUG ] 18:52:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:52:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:52:00 DEBUG } 18:52:00 DEBUG --- http response --- 18:52:00 DEBUG --- http response --- 18:52:00 DEBUG ] 18:52:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:52:00 INFO [http_cmd]: http status code OK 18:52:00 INFO [http_cmd]: http status code OK 18:52:00 INFO [http_cmd]: http status code OK 18:52:00 INFO [http_cmd]: http status code OK 18:52:00 DEBUG } 18:52:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:52:00 DEBUG ----- output ----- 18:52:00 DEBUG --- status code --- 18:52:00 DEBUG --- status code --- 18:52:00 DEBUG --- status code --- 18:52:00 DEBUG --- status code --- 18:52:00 DEBUG } 18:52:00 DEBUG ----- output ----- 18:52:00 DEBUG { 18:52:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:52:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:52:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:52:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:52:00 DEBUG { 18:52:00 DEBUG "status": "success", 18:52:00 DEBUG --- http response --- 18:52:00 DEBUG --- http response --- 18:52:00 DEBUG --- http response --- 18:52:00 DEBUG --- http response --- 18:52:00 DEBUG "status": "success", 18:52:00 DEBUG "data": { 18:52:00 DEBUG "resultType": "vector", 18:52:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:52:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:52:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:52:00 DEBUG "data": { 18:52:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:52:00 DEBUG "result": [ 18:52:00 DEBUG ----- output ----- 18:52:00 DEBUG ----- output ----- 18:52:00 DEBUG ----- output ----- 18:52:00 DEBUG "resultType": "vector", 18:52:00 DEBUG ----- output ----- 18:52:00 DEBUG { 18:52:00 DEBUG { 18:52:00 DEBUG { 18:52:00 DEBUG { 18:52:00 DEBUG "result": [ 18:52:00 DEBUG { 18:52:00 DEBUG "metric": { 18:52:00 DEBUG "status": "success", 18:52:00 DEBUG "status": "success", 18:52:00 DEBUG "status": "success", 18:52:00 DEBUG { 18:52:00 DEBUG "status": "success", 18:52:00 DEBUG "instance": "10.142.0.11:9100" 18:52:00 DEBUG "data": { 18:52:00 DEBUG "data": { 18:52:00 DEBUG "data": { 18:52:00 DEBUG "metric": { 18:52:00 DEBUG "data": { 18:52:00 DEBUG }, 18:52:00 DEBUG "resultType": "vector", 18:52:00 DEBUG "resultType": "vector", 18:52:00 DEBUG "resultType": "vector", 18:52:00 DEBUG "instance": "10.142.0.11:9100" 18:52:00 DEBUG "resultType": "vector", 18:52:00 DEBUG "value": [ 18:52:00 DEBUG "result": [ 18:52:00 DEBUG "result": [ 18:52:00 DEBUG "result": [ 18:52:00 DEBUG }, 18:52:00 DEBUG "result": [ 18:52:00 DEBUG 18:52:00 DEBUG { 18:52:00 DEBUG { 18:52:00 DEBUG { 18:52:00 DEBUG "value": [ 18:52:00 DEBUG 18:52:00 DEBUG 18:52:00 DEBUG "metric": { 18:52:00 DEBUG "metric": { 18:52:00 DEBUG "metric": { 18:52:00 DEBUG { 18:52:00 DEBUG 18:52:00 DEBUG 18:52:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:52:00 DEBUG "pod": "ds-cts-0" 18:52:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:52:00 DEBUG "metric": { 18:52:00 DEBUG 18:52:00 DEBUG [...] 18:52:00 DEBUG }, 18:52:00 DEBUG }, 18:52:00 DEBUG }, 18:52:00 DEBUG "pod": "ds-cts-0" 18:52:00 DEBUG [...] 18:52:00 DEBUG 18:52:00 DEBUG "value": [ 18:52:00 DEBUG "value": [ 18:52:00 DEBUG "value": [ 18:52:00 DEBUG }, 18:52:00 DEBUG 18:52:00 DEBUG 18:52:00 DEBUG 18:52:00 DEBUG 18:52:00 DEBUG 18:52:00 DEBUG "value": [ 18:52:00 DEBUG 18:52:00 DEBUG 18:52:00 DEBUG 18:52:00 DEBUG 18:52:00 DEBUG 18:52:00 DEBUG 18:52:00 DEBUG 18:52:00 DEBUG "instance": "10.142.0.51:9100" 18:52:00 DEBUG 18:52:00 DEBUG 18:52:00 DEBUG 18:52:00 DEBUG 18:52:00 DEBUG "instance": "10.142.0.51:9100" 18:52:00 DEBUG }, 18:52:00 DEBUG [...] 18:52:00 DEBUG [...] 18:52:00 DEBUG [...] 18:52:00 DEBUG 18:52:00 DEBUG }, 18:52:00 DEBUG "value": [ 18:52:00 DEBUG 18:52:00 DEBUG 18:52:00 DEBUG 18:52:00 DEBUG [...] 18:52:00 DEBUG 18:52:00 DEBUG 1734461520, 18:52:00 DEBUG 18:52:00 DEBUG 18:52:00 DEBUG 18:52:00 DEBUG 18:52:00 DEBUG 18:52:00 DEBUG "76007.83333333333" 18:52:00 DEBUG 18:52:00 DEBUG 18:52:00 DEBUG "value": [ 18:52:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:52:00 DEBUG 18:52:00 DEBUG ] 18:52:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:52:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:52:00 DEBUG }, 18:52:00 DEBUG }, 18:52:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:52:00 DEBUG } 18:52:00 DEBUG }, 18:52:00 DEBUG "value": [ 18:52:00 DEBUG "value": [ 18:52:00 DEBUG "value": [ 18:52:00 DEBUG }, 18:52:00 DEBUG ] 18:52:00 DEBUG } 18:52:00 DEBUG 1734461520, 18:52:00 DEBUG 1734461520, 18:52:00 DEBUG 1734461520, 18:52:00 DEBUG "value": [ 18:52:00 DEBUG 1734461520, 18:52:00 DEBUG } 18:52:00 DEBUG "19387.733333333337" 18:52:00 DEBUG "695267328" 18:52:00 DEBUG "1.7811608896591196" 18:52:00 DEBUG 1734461520, 18:52:00 DEBUG "2573250560" 18:52:00 DEBUG ] 18:52:00 DEBUG } 18:52:00 DEBUG ] 18:52:00 DEBUG "937.6776584375093" 18:52:00 DEBUG ] 18:52:00 DEBUG ] 18:52:00 DEBUG ] 18:52:00 DEBUG } 18:52:00 DEBUG ] 18:52:00 DEBUG } 18:52:00 DEBUG } 18:52:00 DEBUG } 18:52:00 DEBUG ] 18:52:00 DEBUG } 18:52:00 DEBUG ] 18:52:00 DEBUG ] 18:52:00 DEBUG } 18:52:00 DEBUG } 18:52:00 DEBUG ] 18:52:00 DEBUG } 18:52:00 DEBUG } 18:52:00 DEBUG } 18:52:00 DEBUG } 18:52:00 DEBUG } 18:52:00 DEBUG } 18:52:00 DEBUG } 18:52:02 INFO 18:52:02 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:52:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:52:02 INFO [loop_until]: OK (rc = 0) 18:52:02 DEBUG --- stdout --- 18:52:02 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 544Mi ds-idrepo-0 7m 736Mi rcs-5d696ff5b-7ttvr 7m 297Mi rcs-5d696ff5b-wwlbr 5m 331Mi 18:52:02 DEBUG --- stderr --- 18:52:02 DEBUG 18:52:02 INFO 18:52:02 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:52:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:52:02 INFO [loop_until]: OK (rc = 0) 18:52:02 DEBUG --- stdout --- 18:52:02 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 106m 0% 2267Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1909m 5% 3630Mi 12% 18:52:02 DEBUG --- stderr --- 18:52:02 DEBUG 18:52:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461579" 18:52:59 INFO [http_cmd]: http status code OK 18:52:59 DEBUG --- status code --- 18:52:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:52:59 DEBUG --- http response --- 18:52:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:52:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734461579" 18:52:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461579" 18:52:59 INFO [http_cmd]: http status code OK 18:52:59 DEBUG --- status code --- 18:52:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:52:59 DEBUG --- http response --- 18:52:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:52:59 INFO [http_cmd]: http status code OK 18:52:59 DEBUG --- status code --- 18:52:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:52:59 DEBUG --- http response --- 18:52:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:52:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461579" 18:52:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461579" 18:52:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461579" 18:52:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461579" 18:52:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461579" 18:52:59 INFO [http_cmd]: http status code OK 18:52:59 INFO [http_cmd]: http status code OK 18:52:59 DEBUG --- status code --- 18:52:59 DEBUG --- status code --- 18:52:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:52:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:52:59 DEBUG --- http response --- 18:52:59 DEBUG --- http response --- 18:52:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:52:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:52:59 INFO [http_cmd]: http status code OK 18:52:59 INFO [http_cmd]: http status code OK 18:52:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461579" 18:52:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461579" 18:52:59 INFO [http_cmd]: http status code OK 18:52:59 DEBUG --- status code --- 18:52:59 DEBUG --- status code --- 18:52:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461579" 18:52:59 DEBUG --- status code --- 18:52:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:52:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:52:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:52:59 DEBUG --- http response --- 18:52:59 DEBUG --- http response --- 18:52:59 DEBUG --- http response --- 18:52:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:52:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:52:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:52:59 INFO [http_cmd]: http status code OK 18:52:59 INFO [http_cmd]: http status code OK 18:52:59 INFO [http_cmd]: http status code OK 18:52:59 DEBUG --- status code --- 18:52:59 DEBUG --- status code --- 18:52:59 DEBUG --- status code --- 18:52:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:52:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:52:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:52:59 DEBUG --- http response --- 18:52:59 DEBUG --- http response --- 18:52:59 DEBUG --- http response --- 18:52:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:52:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:52:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:52:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461579" 18:52:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734461579" 18:52:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461579" 18:52:59 INFO [http_cmd]: http status code OK 18:52:59 DEBUG --- status code --- 18:52:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:52:59 DEBUG --- http response --- 18:52:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:52:59 INFO [http_cmd]: http status code OK 18:52:59 DEBUG --- status code --- 18:52:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:52:59 DEBUG --- http response --- 18:52:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:52:59 ERROR [http_cmd]: ERROR 18:52:59 ERROR --- status code --- 18:52:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:52:59 ERROR --- http response --- 18:52:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:52:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:52:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461579" 18:52:59 INFO [http_cmd]: http status code OK 18:52:59 DEBUG --- status code --- 18:52:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:52:59 DEBUG --- http response --- 18:52:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:52:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461579" 18:52:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461579" 18:52:59 INFO [http_cmd]: http status code OK 18:52:59 INFO [http_cmd]: http status code OK 18:52:59 DEBUG --- status code --- 18:52:59 DEBUG --- status code --- 18:52:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:52:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:52:59 DEBUG --- http response --- 18:52:59 DEBUG --- http response --- 18:52:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:52:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:52:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461579" 18:52:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461579" 18:52:59 INFO [http_cmd]: http status code OK 18:52:59 DEBUG --- status code --- 18:52:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:52:59 INFO [http_cmd]: http status code OK 18:52:59 DEBUG --- http response --- 18:52:59 DEBUG --- status code --- 18:52:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:52:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:52:59 DEBUG --- http response --- 18:52:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:52:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734461579" 18:52:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461579" 18:52:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734461579" 18:52:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734461579" 18:52:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461579" 18:52:59 INFO [http_cmd]: http status code OK 18:52:59 INFO [http_cmd]: http status code OK 18:52:59 DEBUG --- status code --- 18:52:59 INFO [http_cmd]: http status code OK 18:52:59 DEBUG --- status code --- 18:52:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:52:59 DEBUG --- status code --- 18:52:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:52:59 DEBUG --- http response --- 18:52:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:52:59 DEBUG --- http response --- 18:52:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:52:59 DEBUG --- http response --- 18:52:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:52:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:52:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461579" 18:52:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461579" 18:52:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461579" 18:52:59 INFO [http_cmd]: http status code OK 18:52:59 INFO [http_cmd]: http status code OK 18:52:59 DEBUG --- status code --- 18:52:59 DEBUG --- status code --- 18:52:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:52:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:52:59 DEBUG --- http response --- 18:52:59 DEBUG --- http response --- 18:52:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:52:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:52:59 INFO [http_cmd]: http status code OK 18:52:59 DEBUG --- status code --- 18:52:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:52:59 DEBUG --- http response --- 18:52:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:52:59 INFO [http_cmd]: http status code OK 18:52:59 DEBUG --- status code --- 18:52:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:52:59 DEBUG --- http response --- 18:52:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:52:59 INFO [http_cmd]: http status code OK 18:52:59 DEBUG --- status code --- 18:52:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:52:59 DEBUG --- http response --- 18:52:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:52:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461579" 18:53:00 INFO [http_cmd]: http status code OK 18:53:00 DEBUG --- status code --- 18:53:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:53:00 DEBUG --- http response --- 18:53:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:53:00 DEBUG ----- output ----- 18:53:00 DEBUG { 18:53:00 DEBUG "status": "success", 18:53:00 DEBUG "data": { 18:53:00 DEBUG "resultType": "vector", 18:53:00 DEBUG "result": [ 18:53:00 DEBUG { 18:53:00 DEBUG "metric": { 18:53:00 DEBUG "pod": "ds-cts-0" 18:53:00 DEBUG }, 18:53:00 DEBUG "value": [ 18:53:00 DEBUG 18:53:00 DEBUG 18:53:00 DEBUG 18:53:00 DEBUG [...] 18:53:00 DEBUG 18:53:00 DEBUG 18:53:00 DEBUG 18:53:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:53:00 DEBUG }, 18:53:00 DEBUG "value": [ 18:53:00 DEBUG 1734461579, 18:53:00 DEBUG "0.002531788821552007" 18:53:00 DEBUG ] 18:53:00 DEBUG } 18:53:00 DEBUG ] 18:53:00 DEBUG } 18:53:00 DEBUG } 18:53:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461580" 18:53:00 INFO [http_cmd]: http status code OK 18:53:00 DEBUG --- status code --- 18:53:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:53:00 DEBUG --- http response --- 18:53:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:53:00 DEBUG ----- output ----- 18:53:00 DEBUG { 18:53:00 DEBUG "status": "success", 18:53:00 DEBUG "data": { 18:53:00 DEBUG "resultType": "vector", 18:53:00 DEBUG "result": [ 18:53:00 DEBUG { 18:53:00 DEBUG "metric": { 18:53:00 DEBUG "instance": "10.142.0.11:9100" 18:53:00 DEBUG }, 18:53:00 DEBUG "value": [ 18:53:00 DEBUG 18:53:00 DEBUG 18:53:00 DEBUG 18:53:00 DEBUG [...] 18:53:00 DEBUG 18:53:00 DEBUG 18:53:00 DEBUG 18:53:00 DEBUG "instance": "10.142.0.51:9100" 18:53:00 DEBUG }, 18:53:00 DEBUG "value": [ 18:53:00 DEBUG 1734461580, 18:53:00 DEBUG "88131.5666666667" 18:53:00 DEBUG ] 18:53:00 DEBUG } 18:53:00 DEBUG ] 18:53:00 DEBUG } 18:53:00 DEBUG } 18:53:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461580" 18:53:00 INFO [http_cmd]: http status code OK 18:53:00 DEBUG --- status code --- 18:53:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:53:00 DEBUG --- http response --- 18:53:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:53:00 DEBUG ----- output ----- 18:53:00 DEBUG { 18:53:00 DEBUG "status": "success", 18:53:00 DEBUG "data": { 18:53:00 DEBUG "resultType": "vector", 18:53:00 DEBUG "result": [ 18:53:00 DEBUG { 18:53:00 DEBUG "metric": { 18:53:00 DEBUG "instance": "10.142.0.11:9100" 18:53:00 DEBUG }, 18:53:00 DEBUG "value": [ 18:53:00 DEBUG 18:53:00 DEBUG 18:53:00 DEBUG 18:53:00 DEBUG [...] 18:53:00 DEBUG 18:53:00 DEBUG 18:53:00 DEBUG 18:53:00 DEBUG "instance": "10.142.0.51:9100" 18:53:00 DEBUG }, 18:53:00 DEBUG "value": [ 18:53:00 DEBUG 1734461580, 18:53:00 DEBUG "0" 18:53:00 DEBUG ] 18:53:00 DEBUG } 18:53:00 DEBUG ] 18:53:00 DEBUG } 18:53:00 DEBUG } 18:53:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461580" 18:53:00 INFO [http_cmd]: http status code OK 18:53:00 DEBUG --- status code --- 18:53:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:53:00 DEBUG --- http response --- 18:53:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:53:00 DEBUG ----- output ----- 18:53:00 DEBUG { 18:53:00 DEBUG "status": "success", 18:53:00 DEBUG "data": { 18:53:00 DEBUG "resultType": "vector", 18:53:00 DEBUG "result": [ 18:53:00 DEBUG { 18:53:00 DEBUG "metric": { 18:53:00 DEBUG "instance": "10.142.0.11:9100" 18:53:00 DEBUG }, 18:53:00 DEBUG "value": [ 18:53:00 DEBUG 18:53:00 DEBUG 18:53:00 DEBUG 18:53:00 DEBUG [...] 18:53:00 DEBUG 18:53:00 DEBUG 18:53:00 DEBUG 18:53:00 DEBUG "instance": "10.142.0.51:9100" 18:53:00 DEBUG }, 18:53:00 DEBUG "value": [ 18:53:00 DEBUG 1734461580, 18:53:00 DEBUG "0.00043333333333634994" 18:53:00 DEBUG ] 18:53:00 DEBUG } 18:53:00 DEBUG ] 18:53:00 DEBUG } 18:53:00 DEBUG } 18:53:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461580" 18:53:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461580" 18:53:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734461580" 18:53:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734461580" 18:53:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461580" 18:53:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461580" 18:53:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461580" 18:53:00 INFO [http_cmd]: http status code OK 18:53:00 DEBUG --- status code --- 18:53:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:53:00 DEBUG --- http response --- 18:53:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:53:00 DEBUG ----- output ----- 18:53:00 DEBUG { 18:53:00 DEBUG "status": "success", 18:53:00 DEBUG "data": { 18:53:00 DEBUG "resultType": "vector", 18:53:00 DEBUG "result": [ 18:53:00 DEBUG { 18:53:00 DEBUG "metric": { 18:53:00 DEBUG "pod": "ds-cts-0" 18:53:00 DEBUG }, 18:53:00 DEBUG "value": [ 18:53:00 DEBUG 18:53:00 DEBUG 18:53:00 DEBUG 18:53:00 DEBUG [...] 18:53:00 DEBUG 18:53:00 DEBUG 18:53:00 DEBUG 18:53:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:53:00 DEBUG }, 18:53:00 DEBUG "value": [ 18:53:00 DEBUG 1734461580, 18:53:00 DEBUG "0" 18:53:00 DEBUG ] 18:53:00 DEBUG } 18:53:00 DEBUG ] 18:53:00 DEBUG } 18:53:00 DEBUG } 18:53:00 INFO [http_cmd]: http status code OK 18:53:00 DEBUG --- status code --- 18:53:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:53:00 DEBUG --- http response --- 18:53:00 INFO [http_cmd]: http status code OK 18:53:00 DEBUG --- status code --- 18:53:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:53:00 DEBUG --- http response --- 18:53:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:53:00 DEBUG ----- output ----- 18:53:00 DEBUG { 18:53:00 DEBUG "status": "success", 18:53:00 DEBUG "data": { 18:53:00 DEBUG "resultType": "vector", 18:53:00 DEBUG "result": [ 18:53:00 DEBUG { 18:53:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:53:00 INFO [http_cmd]: http status code OK 18:53:00 DEBUG --- status code --- 18:53:00 DEBUG ----- output ----- 18:53:00 DEBUG "metric": { 18:53:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:53:00 DEBUG { 18:53:00 DEBUG "instance": "10.142.0.11:9100" 18:53:00 DEBUG --- http response --- 18:53:00 DEBUG "status": "success", 18:53:00 DEBUG }, 18:53:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:53:00 DEBUG "data": { 18:53:00 DEBUG "value": [ 18:53:00 DEBUG ----- output ----- 18:53:00 INFO [http_cmd]: http status code OK 18:53:00 DEBUG "resultType": "vector", 18:53:00 DEBUG "result": [ 18:53:00 DEBUG { 18:53:00 DEBUG 18:53:00 DEBUG --- status code --- 18:53:00 INFO [http_cmd]: http status code OK 18:53:00 DEBUG "metric": { 18:53:00 DEBUG { 18:53:00 DEBUG 18:53:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:53:00 DEBUG --- status code --- 18:53:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:53:00 DEBUG "status": "success", 18:53:00 DEBUG 18:53:00 DEBUG --- http response --- 18:53:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:53:00 DEBUG }, 18:53:00 DEBUG "data": { 18:53:00 DEBUG [...] 18:53:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:53:00 DEBUG --- http response --- 18:53:00 DEBUG "value": [ 18:53:00 DEBUG "resultType": "vector", 18:53:00 DEBUG 18:53:00 DEBUG ----- output ----- 18:53:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:53:00 DEBUG 18:53:00 DEBUG "result": [ 18:53:00 DEBUG 18:53:00 DEBUG { 18:53:00 DEBUG ----- output ----- 18:53:00 DEBUG 18:53:00 DEBUG { 18:53:00 DEBUG 18:53:00 DEBUG "status": "success", 18:53:00 DEBUG { 18:53:00 DEBUG 18:53:00 DEBUG "metric": { 18:53:00 DEBUG "instance": "10.142.0.51:9100" 18:53:00 DEBUG "data": { 18:53:00 DEBUG "status": "success", 18:53:00 DEBUG [...] 18:53:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:53:00 DEBUG }, 18:53:00 DEBUG "resultType": "vector", 18:53:00 DEBUG "data": { 18:53:00 DEBUG 18:53:00 INFO [http_cmd]: http status code OK 18:53:00 DEBUG }, 18:53:00 DEBUG "value": [ 18:53:00 DEBUG "result": [ 18:53:00 DEBUG "resultType": "vector", 18:53:00 DEBUG 18:53:00 DEBUG --- status code --- 18:53:00 DEBUG "value": [ 18:53:00 DEBUG 1734461580, 18:53:00 DEBUG { 18:53:00 DEBUG "metric": { 18:53:00 DEBUG 18:53:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:53:00 DEBUG 18:53:00 DEBUG "72166.90000000002" 18:53:00 DEBUG "result": [ 18:53:00 DEBUG "pod": "ds-cts-0" 18:53:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:53:00 DEBUG --- http response --- 18:53:00 DEBUG 18:53:00 DEBUG ] 18:53:00 DEBUG { 18:53:00 DEBUG }, 18:53:00 DEBUG }, 18:53:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:53:00 DEBUG 18:53:00 DEBUG } 18:53:00 DEBUG "metric": { 18:53:00 DEBUG "value": [ 18:53:00 DEBUG "value": [ 18:53:00 DEBUG ----- output ----- 18:53:00 DEBUG [...] 18:53:00 DEBUG ] 18:53:00 DEBUG "pod": "ds-cts-0" 18:53:00 DEBUG 18:53:00 DEBUG 1734461580, 18:53:00 DEBUG { 18:53:00 DEBUG 18:53:00 DEBUG } 18:53:00 DEBUG }, 18:53:00 DEBUG 18:53:00 DEBUG "1.7783519913193286" 18:53:00 DEBUG "status": "success", 18:53:00 DEBUG 18:53:00 DEBUG } 18:53:00 DEBUG "value": [ 18:53:00 DEBUG 18:53:00 DEBUG ] 18:53:00 DEBUG "data": { 18:53:00 DEBUG 18:53:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:53:00 DEBUG }, 18:53:00 DEBUG "value": [ 18:53:00 DEBUG 1734461580, 18:53:00 DEBUG "2583183360" 18:53:00 DEBUG ] 18:53:00 DEBUG } 18:53:00 DEBUG "resultType": "vector", 18:53:00 DEBUG 18:53:00 DEBUG [...] 18:53:00 DEBUG } 18:53:00 DEBUG ] 18:53:00 DEBUG "result": [ 18:53:00 DEBUG 18:53:00 DEBUG 18:53:00 DEBUG ] 18:53:00 DEBUG } 18:53:00 DEBUG { 18:53:00 DEBUG "metric": { 18:53:00 DEBUG "instance": "10.142.0.11:9100" 18:53:00 DEBUG }, 18:53:00 DEBUG "value": [ 18:53:00 DEBUG 18:53:00 DEBUG 18:53:00 DEBUG } 18:53:00 DEBUG } 18:53:00 DEBUG 18:53:00 DEBUG [...] 18:53:00 DEBUG 18:53:00 DEBUG } 18:53:00 DEBUG 18:53:00 DEBUG 18:53:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:53:00 DEBUG 18:53:00 DEBUG 18:53:00 DEBUG }, 18:53:00 DEBUG [...] 18:53:00 DEBUG 18:53:00 DEBUG "value": [ 18:53:00 DEBUG 18:53:00 DEBUG 18:53:00 DEBUG 1734461580, 18:53:00 DEBUG "695308288" 18:53:00 DEBUG 18:53:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:53:00 DEBUG ] 18:53:00 DEBUG "instance": "10.142.0.51:9100" 18:53:00 DEBUG }, 18:53:00 DEBUG } 18:53:00 DEBUG }, 18:53:00 DEBUG "value": [ 18:53:00 DEBUG ] 18:53:00 DEBUG "value": [ 18:53:00 DEBUG 1734461580, 18:53:00 DEBUG } 18:53:00 DEBUG "20070.4" 18:53:00 DEBUG 1734461580, 18:53:00 DEBUG } 18:53:00 DEBUG ] 18:53:00 DEBUG "0" 18:53:00 DEBUG } 18:53:00 DEBUG ] 18:53:00 DEBUG ] 18:53:00 DEBUG } 18:53:00 DEBUG } 18:53:00 DEBUG ] 18:53:00 DEBUG } 18:53:00 DEBUG } 18:53:00 DEBUG } 18:53:02 INFO 18:53:02 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:53:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:53:02 INFO [loop_until]: OK (rc = 0) 18:53:02 DEBUG --- stdout --- 18:53:02 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 7m 544Mi ds-idrepo-0 7m 736Mi rcs-5d696ff5b-7ttvr 4m 297Mi rcs-5d696ff5b-wwlbr 2m 331Mi 18:53:02 DEBUG --- stderr --- 18:53:02 DEBUG 18:53:02 INFO 18:53:02 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:53:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:53:02 INFO [loop_until]: OK (rc = 0) 18:53:02 DEBUG --- stdout --- 18:53:02 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 100m 0% 2279Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1861m 5% 3629Mi 12% 18:53:02 DEBUG --- stderr --- 18:53:02 DEBUG 18:53:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461639" 18:53:59 INFO [http_cmd]: http status code OK 18:53:59 DEBUG --- status code --- 18:53:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:53:59 DEBUG --- http response --- 18:53:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:53:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734461639" 18:53:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461639" 18:53:59 INFO [http_cmd]: http status code OK 18:53:59 INFO [http_cmd]: http status code OK 18:53:59 DEBUG --- status code --- 18:53:59 DEBUG --- status code --- 18:53:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:53:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:53:59 DEBUG --- http response --- 18:53:59 DEBUG --- http response --- 18:53:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:53:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:53:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461639" 18:53:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461639" 18:53:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461639" 18:53:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461639" 18:53:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461639" 18:53:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461639" 18:53:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461639" 18:53:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461639" 18:53:59 INFO [http_cmd]: http status code OK 18:53:59 DEBUG --- status code --- 18:53:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:53:59 DEBUG --- http response --- 18:53:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:53:59 INFO [http_cmd]: http status code OK 18:53:59 DEBUG --- status code --- 18:53:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:53:59 DEBUG --- http response --- 18:53:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:53:59 INFO [http_cmd]: http status code OK 18:53:59 INFO [http_cmd]: http status code OK 18:53:59 DEBUG --- status code --- 18:53:59 DEBUG --- status code --- 18:53:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:53:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:53:59 DEBUG --- http response --- 18:53:59 DEBUG --- http response --- 18:53:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:53:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:53:59 INFO [http_cmd]: http status code OK 18:53:59 INFO [http_cmd]: http status code OK 18:53:59 DEBUG --- status code --- 18:53:59 INFO [http_cmd]: http status code OK 18:53:59 DEBUG --- status code --- 18:53:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:53:59 DEBUG --- status code --- 18:53:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:53:59 DEBUG --- http response --- 18:53:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:53:59 DEBUG --- http response --- 18:53:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:53:59 DEBUG --- http response --- 18:53:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:53:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:53:59 INFO [http_cmd]: http status code OK 18:53:59 DEBUG --- status code --- 18:53:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:53:59 DEBUG --- http response --- 18:53:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:53:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461639" 18:53:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461639" 18:53:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734461639" 18:53:59 INFO [http_cmd]: http status code OK 18:53:59 DEBUG --- status code --- 18:53:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:53:59 DEBUG --- http response --- 18:53:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:53:59 INFO [http_cmd]: http status code OK 18:53:59 DEBUG --- status code --- 18:53:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:53:59 DEBUG --- http response --- 18:53:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:53:59 ERROR [http_cmd]: ERROR 18:53:59 ERROR --- status code --- 18:53:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:53:59 ERROR --- http response --- 18:53:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:53:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:53:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461639" 18:53:59 INFO [http_cmd]: http status code OK 18:53:59 DEBUG --- status code --- 18:53:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:53:59 DEBUG --- http response --- 18:53:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:53:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461639" 18:53:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461639" 18:53:59 INFO [http_cmd]: http status code OK 18:53:59 DEBUG --- status code --- 18:53:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:53:59 DEBUG --- http response --- 18:53:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:53:59 INFO [http_cmd]: http status code OK 18:53:59 DEBUG --- status code --- 18:53:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:53:59 DEBUG --- http response --- 18:53:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:53:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461639" 18:53:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461639" 18:53:59 INFO [http_cmd]: http status code OK 18:53:59 DEBUG --- status code --- 18:53:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:53:59 DEBUG --- http response --- 18:53:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:53:59 INFO [http_cmd]: http status code OK 18:53:59 DEBUG --- status code --- 18:53:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:53:59 DEBUG --- http response --- 18:53:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:53:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734461639" 18:53:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734461639" 18:53:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461639" 18:53:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461639" 18:53:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734461639" 18:53:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461639" 18:53:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461639" 18:53:59 INFO [http_cmd]: http status code OK 18:53:59 INFO [http_cmd]: http status code OK 18:53:59 INFO [http_cmd]: http status code OK 18:53:59 DEBUG --- status code --- 18:53:59 INFO [http_cmd]: http status code OK 18:53:59 DEBUG --- status code --- 18:53:59 DEBUG --- status code --- 18:53:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:53:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:53:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:53:59 DEBUG --- status code --- 18:53:59 DEBUG --- http response --- 18:53:59 DEBUG --- http response --- 18:53:59 DEBUG --- http response --- 18:53:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:53:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:53:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:53:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:53:59 DEBUG --- http response --- 18:53:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:53:59 INFO [http_cmd]: http status code OK 18:53:59 DEBUG --- status code --- 18:53:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:53:59 DEBUG --- http response --- 18:53:59 INFO [http_cmd]: http status code OK 18:53:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461639" 18:53:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:53:59 INFO [http_cmd]: http status code OK 18:53:59 DEBUG --- status code --- 18:53:59 DEBUG --- status code --- 18:53:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:53:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:53:59 DEBUG --- http response --- 18:53:59 DEBUG --- http response --- 18:53:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:53:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:53:59 INFO [http_cmd]: http status code OK 18:53:59 DEBUG --- status code --- 18:53:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:53:59 DEBUG --- http response --- 18:53:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:54:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461640" 18:54:00 INFO [http_cmd]: http status code OK 18:54:00 DEBUG --- status code --- 18:54:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:54:00 DEBUG --- http response --- 18:54:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:54:00 DEBUG ----- output ----- 18:54:00 DEBUG { 18:54:00 DEBUG "status": "success", 18:54:00 DEBUG "data": { 18:54:00 DEBUG "resultType": "vector", 18:54:00 DEBUG "result": [ 18:54:00 DEBUG { 18:54:00 DEBUG "metric": { 18:54:00 DEBUG "pod": "ds-cts-0" 18:54:00 DEBUG }, 18:54:00 DEBUG "value": [ 18:54:00 DEBUG 18:54:00 DEBUG 18:54:00 DEBUG 18:54:00 DEBUG [...] 18:54:00 DEBUG 18:54:00 DEBUG 18:54:00 DEBUG 18:54:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:54:00 DEBUG }, 18:54:00 DEBUG "value": [ 18:54:00 DEBUG 1734461640, 18:54:00 DEBUG "0.003881094454207908" 18:54:00 DEBUG ] 18:54:00 DEBUG } 18:54:00 DEBUG ] 18:54:00 DEBUG } 18:54:00 DEBUG } 18:54:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461640" 18:54:00 INFO [http_cmd]: http status code OK 18:54:00 DEBUG --- status code --- 18:54:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:54:00 DEBUG --- http response --- 18:54:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:54:00 DEBUG ----- output ----- 18:54:00 DEBUG { 18:54:00 DEBUG "status": "success", 18:54:00 DEBUG "data": { 18:54:00 DEBUG "resultType": "vector", 18:54:00 DEBUG "result": [ 18:54:00 DEBUG { 18:54:00 DEBUG "metric": { 18:54:00 DEBUG "instance": "10.142.0.11:9100" 18:54:00 DEBUG }, 18:54:00 DEBUG "value": [ 18:54:00 DEBUG 18:54:00 DEBUG 18:54:00 DEBUG 18:54:00 DEBUG [...] 18:54:00 DEBUG 18:54:00 DEBUG 18:54:00 DEBUG 18:54:00 DEBUG "instance": "10.142.0.51:9100" 18:54:00 DEBUG }, 18:54:00 DEBUG "value": [ 18:54:00 DEBUG 1734461640, 18:54:00 DEBUG "89790.43333333335" 18:54:00 DEBUG ] 18:54:00 DEBUG } 18:54:00 DEBUG ] 18:54:00 DEBUG } 18:54:00 DEBUG } 18:54:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461640" 18:54:00 INFO [http_cmd]: http status code OK 18:54:00 DEBUG --- status code --- 18:54:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:54:00 DEBUG --- http response --- 18:54:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:54:00 DEBUG ----- output ----- 18:54:00 DEBUG { 18:54:00 DEBUG "status": "success", 18:54:00 DEBUG "data": { 18:54:00 DEBUG "resultType": "vector", 18:54:00 DEBUG "result": [ 18:54:00 DEBUG { 18:54:00 DEBUG "metric": { 18:54:00 DEBUG "instance": "10.142.0.11:9100" 18:54:00 DEBUG }, 18:54:00 DEBUG "value": [ 18:54:00 DEBUG 18:54:00 DEBUG 18:54:00 DEBUG 18:54:00 DEBUG [...] 18:54:00 DEBUG 18:54:00 DEBUG 18:54:00 DEBUG 18:54:00 DEBUG "instance": "10.142.0.51:9100" 18:54:00 DEBUG }, 18:54:00 DEBUG "value": [ 18:54:00 DEBUG 1734461640, 18:54:00 DEBUG "0.0003333333333332671" 18:54:00 DEBUG ] 18:54:00 DEBUG } 18:54:00 DEBUG ] 18:54:00 DEBUG } 18:54:00 DEBUG } 18:54:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461640" 18:54:00 INFO [http_cmd]: http status code OK 18:54:00 DEBUG --- status code --- 18:54:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:54:00 DEBUG --- http response --- 18:54:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:54:00 DEBUG ----- output ----- 18:54:00 DEBUG { 18:54:00 DEBUG "status": "success", 18:54:00 DEBUG "data": { 18:54:00 DEBUG "resultType": "vector", 18:54:00 DEBUG "result": [ 18:54:00 DEBUG { 18:54:00 DEBUG "metric": { 18:54:00 DEBUG "instance": "10.142.0.11:9100" 18:54:00 DEBUG }, 18:54:00 DEBUG "value": [ 18:54:00 DEBUG 18:54:00 DEBUG 18:54:00 DEBUG 18:54:00 DEBUG [...] 18:54:00 DEBUG 18:54:00 DEBUG 18:54:00 DEBUG 18:54:00 DEBUG "instance": "10.142.0.51:9100" 18:54:00 DEBUG }, 18:54:00 DEBUG "value": [ 18:54:00 DEBUG 1734461640, 18:54:00 DEBUG "0.00023333333333302642" 18:54:00 DEBUG ] 18:54:00 DEBUG } 18:54:00 DEBUG ] 18:54:00 DEBUG } 18:54:00 DEBUG } 18:54:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461640" 18:54:00 INFO [http_cmd]: http status code OK 18:54:00 DEBUG --- status code --- 18:54:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:54:00 DEBUG --- http response --- 18:54:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:54:00 DEBUG ----- output ----- 18:54:00 DEBUG { 18:54:00 DEBUG "status": "success", 18:54:00 DEBUG "data": { 18:54:00 DEBUG "resultType": "vector", 18:54:00 DEBUG "result": [ 18:54:00 DEBUG { 18:54:00 DEBUG "metric": { 18:54:00 DEBUG "pod": "ds-cts-0" 18:54:00 DEBUG }, 18:54:00 DEBUG "value": [ 18:54:00 DEBUG 18:54:00 DEBUG 18:54:00 DEBUG 18:54:00 DEBUG [...] 18:54:00 DEBUG 18:54:00 DEBUG 18:54:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461640" 18:54:00 DEBUG 18:54:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:54:00 DEBUG }, 18:54:00 DEBUG "value": [ 18:54:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734461640" 18:54:00 DEBUG 1734461640, 18:54:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734461640" 18:54:00 DEBUG "0" 18:54:00 DEBUG ] 18:54:00 DEBUG } 18:54:00 DEBUG ] 18:54:00 DEBUG } 18:54:00 DEBUG } 18:54:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461640" 18:54:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461640" 18:54:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461640" 18:54:00 INFO [http_cmd]: http status code OK 18:54:00 DEBUG --- status code --- 18:54:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:54:00 DEBUG --- http response --- 18:54:00 INFO [http_cmd]: http status code OK 18:54:00 INFO [http_cmd]: http status code OK 18:54:00 DEBUG --- status code --- 18:54:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:54:00 DEBUG --- status code --- 18:54:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:54:00 DEBUG ----- output ----- 18:54:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:54:00 DEBUG --- http response --- 18:54:00 DEBUG { 18:54:00 DEBUG --- http response --- 18:54:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:54:00 DEBUG "status": "success", 18:54:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:54:00 DEBUG ----- output ----- 18:54:00 DEBUG "data": { 18:54:00 DEBUG ----- output ----- 18:54:00 DEBUG { 18:54:00 DEBUG "resultType": "vector", 18:54:00 DEBUG { 18:54:00 DEBUG "status": "success", 18:54:00 DEBUG "result": [ 18:54:00 DEBUG "status": "success", 18:54:00 DEBUG "data": { 18:54:00 DEBUG { 18:54:00 DEBUG "data": { 18:54:00 DEBUG "resultType": "vector", 18:54:00 DEBUG "metric": { 18:54:00 DEBUG "resultType": "vector", 18:54:00 DEBUG "result": [ 18:54:00 DEBUG "instance": "10.142.0.11:9100" 18:54:00 DEBUG "result": [ 18:54:00 DEBUG { 18:54:00 DEBUG }, 18:54:00 DEBUG { 18:54:00 DEBUG "metric": { 18:54:00 DEBUG "value": [ 18:54:00 DEBUG "metric": { 18:54:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:54:00 DEBUG 18:54:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:54:00 DEBUG }, 18:54:00 DEBUG 18:54:00 DEBUG }, 18:54:00 DEBUG "value": [ 18:54:00 DEBUG 18:54:00 DEBUG "value": [ 18:54:00 INFO [http_cmd]: http status code OK 18:54:00 DEBUG 18:54:00 DEBUG [...] 18:54:00 DEBUG 18:54:00 DEBUG --- status code --- 18:54:00 DEBUG 18:54:00 DEBUG 18:54:00 DEBUG 18:54:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:54:00 DEBUG 18:54:00 DEBUG 18:54:00 DEBUG 18:54:00 DEBUG --- http response --- 18:54:00 DEBUG [...] 18:54:00 DEBUG 18:54:00 DEBUG [...] 18:54:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:54:00 DEBUG 18:54:00 DEBUG "instance": "10.142.0.51:9100" 18:54:00 DEBUG 18:54:00 DEBUG ----- output ----- 18:54:00 DEBUG 18:54:00 DEBUG }, 18:54:00 DEBUG 18:54:00 DEBUG { 18:54:00 DEBUG 18:54:00 DEBUG "value": [ 18:54:00 DEBUG 18:54:00 DEBUG "status": "success", 18:54:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:54:00 DEBUG 1734461640, 18:54:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:54:00 DEBUG "data": { 18:54:00 DEBUG }, 18:54:00 DEBUG "76283.43333333338" 18:54:00 DEBUG }, 18:54:00 DEBUG "resultType": "vector", 18:54:00 DEBUG "value": [ 18:54:00 INFO [http_cmd]: http status code OK 18:54:00 DEBUG ] 18:54:00 DEBUG "value": [ 18:54:00 DEBUG "result": [ 18:54:00 DEBUG 1734461640, 18:54:00 DEBUG --- status code --- 18:54:00 DEBUG } 18:54:00 DEBUG 1734461640, 18:54:00 DEBUG { 18:54:00 DEBUG "1.763856159179848" 18:54:00 DEBUG ] 18:54:00 DEBUG } 18:54:00 DEBUG "2599350272" 18:54:00 DEBUG "metric": { 18:54:00 DEBUG ] 18:54:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:54:00 DEBUG ] 18:54:00 DEBUG ] 18:54:00 DEBUG "pod": "ds-cts-0" 18:54:00 DEBUG } 18:54:00 DEBUG --- http response --- 18:54:00 DEBUG } 18:54:00 DEBUG } 18:54:00 DEBUG }, 18:54:00 DEBUG "value": [ 18:54:00 DEBUG 18:54:00 DEBUG } 18:54:00 DEBUG ] 18:54:00 DEBUG } 18:54:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:54:00 INFO [http_cmd]: http status code OK 18:54:00 DEBUG 18:54:00 DEBUG } 18:54:00 DEBUG } 18:54:00 DEBUG --- status code --- 18:54:00 DEBUG 18:54:00 DEBUG ----- output ----- 18:54:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:54:00 DEBUG [...] 18:54:00 DEBUG { 18:54:00 DEBUG --- http response --- 18:54:00 DEBUG 18:54:00 DEBUG "status": "success", 18:54:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:54:00 DEBUG 18:54:00 DEBUG "data": { 18:54:00 DEBUG ----- output ----- 18:54:00 DEBUG 18:54:00 DEBUG "resultType": "vector", 18:54:00 DEBUG { 18:54:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:54:00 DEBUG "result": [ 18:54:00 DEBUG "status": "success", 18:54:00 DEBUG }, 18:54:00 DEBUG { 18:54:00 DEBUG "data": { 18:54:00 DEBUG "value": [ 18:54:00 DEBUG "metric": { 18:54:00 DEBUG "resultType": "vector", 18:54:00 DEBUG 1734461640, 18:54:00 DEBUG "instance": "10.142.0.11:9100" 18:54:00 DEBUG "result": [ 18:54:00 DEBUG "695320576" 18:54:00 DEBUG }, 18:54:00 DEBUG { 18:54:00 DEBUG ] 18:54:00 DEBUG "value": [ 18:54:00 DEBUG "metric": { 18:54:00 DEBUG } 18:54:00 DEBUG 18:54:00 DEBUG "pod": "ds-cts-0" 18:54:00 DEBUG ] 18:54:00 DEBUG 18:54:00 DEBUG }, 18:54:00 DEBUG } 18:54:00 DEBUG 18:54:00 DEBUG "value": [ 18:54:00 DEBUG } 18:54:00 DEBUG [...] 18:54:00 DEBUG 18:54:00 DEBUG 18:54:00 DEBUG 18:54:00 DEBUG 18:54:00 DEBUG 18:54:00 DEBUG 18:54:00 DEBUG [...] 18:54:00 DEBUG "instance": "10.142.0.51:9100" 18:54:00 DEBUG 18:54:00 DEBUG }, 18:54:00 DEBUG 18:54:00 DEBUG "value": [ 18:54:00 DEBUG 18:54:00 DEBUG 1734461640, 18:54:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:54:00 DEBUG "6826.666666666668" 18:54:00 DEBUG }, 18:54:00 DEBUG ] 18:54:00 DEBUG "value": [ 18:54:00 DEBUG } 18:54:00 DEBUG 1734461640, 18:54:00 DEBUG ] 18:54:00 DEBUG "1852.771507442801" 18:54:00 DEBUG } 18:54:00 DEBUG ] 18:54:00 DEBUG } 18:54:00 DEBUG } 18:54:00 DEBUG ] 18:54:00 DEBUG } 18:54:00 DEBUG } 18:54:02 INFO 18:54:02 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:54:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:54:02 INFO [loop_until]: OK (rc = 0) 18:54:02 DEBUG --- stdout --- 18:54:02 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 544Mi ds-idrepo-0 8m 737Mi rcs-5d696ff5b-7ttvr 4m 297Mi rcs-5d696ff5b-wwlbr 3m 331Mi 18:54:02 DEBUG --- stderr --- 18:54:02 DEBUG 18:54:02 INFO 18:54:02 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:54:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:54:02 INFO [loop_until]: OK (rc = 0) 18:54:02 DEBUG --- stdout --- 18:54:02 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 99m 0% 2280Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1853m 5% 3632Mi 12% 18:54:02 DEBUG --- stderr --- 18:54:02 DEBUG 18:54:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461699" 18:54:59 INFO [http_cmd]: http status code OK 18:54:59 DEBUG --- status code --- 18:54:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:54:59 DEBUG --- http response --- 18:54:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:54:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461699" 18:54:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734461699" 18:54:59 INFO [http_cmd]: http status code OK 18:54:59 DEBUG --- status code --- 18:54:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:54:59 INFO [http_cmd]: http status code OK 18:54:59 DEBUG --- http response --- 18:54:59 DEBUG --- status code --- 18:54:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:54:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:54:59 DEBUG --- http response --- 18:54:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:54:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461699" 18:54:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461699" 18:54:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461699" 18:54:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461699" 18:54:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461699" 18:54:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461699" 18:54:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461699" 18:54:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461699" 18:54:59 INFO [http_cmd]: http status code OK 18:54:59 INFO [http_cmd]: http status code OK 18:54:59 DEBUG --- status code --- 18:54:59 INFO [http_cmd]: http status code OK 18:54:59 DEBUG --- status code --- 18:54:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:54:59 DEBUG --- status code --- 18:54:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:54:59 DEBUG --- http response --- 18:54:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:54:59 DEBUG --- http response --- 18:54:59 DEBUG --- http response --- 18:54:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:54:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:54:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:54:59 INFO [http_cmd]: http status code OK 18:54:59 INFO [http_cmd]: http status code OK 18:54:59 INFO [http_cmd]: http status code OK 18:54:59 DEBUG --- status code --- 18:54:59 INFO [http_cmd]: http status code OK 18:54:59 DEBUG --- status code --- 18:54:59 DEBUG --- status code --- 18:54:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:54:59 DEBUG --- status code --- 18:54:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:54:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:54:59 DEBUG --- http response --- 18:54:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:54:59 DEBUG --- http response --- 18:54:59 DEBUG --- http response --- 18:54:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:54:59 INFO [http_cmd]: http status code OK 18:54:59 DEBUG --- http response --- 18:54:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:54:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:54:59 DEBUG --- status code --- 18:54:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:54:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:54:59 DEBUG --- http response --- 18:54:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:54:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461699" 18:54:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461699" 18:54:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734461699" 18:54:59 INFO [http_cmd]: http status code OK 18:54:59 DEBUG --- status code --- 18:54:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:54:59 DEBUG --- http response --- 18:54:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:54:59 INFO [http_cmd]: http status code OK 18:54:59 ERROR [http_cmd]: ERROR 18:54:59 DEBUG --- status code --- 18:54:59 ERROR --- status code --- 18:54:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:54:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:54:59 DEBUG --- http response --- 18:54:59 ERROR --- http response --- 18:54:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:54:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:54:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:54:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461699" 18:54:59 INFO [http_cmd]: http status code OK 18:54:59 DEBUG --- status code --- 18:54:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:54:59 DEBUG --- http response --- 18:54:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:54:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461699" 18:54:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461699" 18:54:59 INFO [http_cmd]: http status code OK 18:54:59 DEBUG --- status code --- 18:54:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:54:59 DEBUG --- http response --- 18:54:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:54:59 INFO [http_cmd]: http status code OK 18:54:59 DEBUG --- status code --- 18:54:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:54:59 DEBUG --- http response --- 18:54:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:54:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461699" 18:54:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461699" 18:54:59 INFO [http_cmd]: http status code OK 18:54:59 DEBUG --- status code --- 18:54:59 INFO [http_cmd]: http status code OK 18:54:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:54:59 DEBUG --- status code --- 18:54:59 DEBUG --- http response --- 18:54:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:54:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:54:59 DEBUG --- http response --- 18:54:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:54:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461699" 18:54:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734461699" 18:54:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461699" 18:54:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734461699" 18:54:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734461699" 18:54:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461699" 18:54:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461699" 18:54:59 INFO [http_cmd]: http status code OK 18:54:59 DEBUG --- status code --- 18:54:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:54:59 DEBUG --- http response --- 18:54:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:54:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461699" 18:54:59 INFO [http_cmd]: http status code OK 18:54:59 DEBUG --- status code --- 18:54:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:54:59 INFO [http_cmd]: http status code OK 18:54:59 INFO [http_cmd]: http status code OK 18:54:59 INFO [http_cmd]: http status code OK 18:54:59 DEBUG --- http response --- 18:54:59 DEBUG --- status code --- 18:54:59 DEBUG --- status code --- 18:54:59 DEBUG --- status code --- 18:54:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:54:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:54:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:54:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:54:59 DEBUG --- http response --- 18:54:59 INFO [http_cmd]: http status code OK 18:54:59 DEBUG --- http response --- 18:54:59 DEBUG --- http response --- 18:54:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:54:59 DEBUG --- status code --- 18:54:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:54:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:54:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:54:59 INFO [http_cmd]: http status code OK 18:54:59 DEBUG --- http response --- 18:54:59 DEBUG --- status code --- 18:54:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:54:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:54:59 DEBUG --- http response --- 18:54:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:54:59 INFO [http_cmd]: http status code OK 18:54:59 DEBUG --- status code --- 18:54:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:54:59 DEBUG --- http response --- 18:54:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:55:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461700" 18:55:00 INFO [http_cmd]: http status code OK 18:55:00 DEBUG --- status code --- 18:55:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:55:00 DEBUG --- http response --- 18:55:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:55:00 DEBUG ----- output ----- 18:55:00 DEBUG { 18:55:00 DEBUG "status": "success", 18:55:00 DEBUG "data": { 18:55:00 DEBUG "resultType": "vector", 18:55:00 DEBUG "result": [ 18:55:00 DEBUG { 18:55:00 DEBUG "metric": { 18:55:00 DEBUG "pod": "ds-cts-0" 18:55:00 DEBUG }, 18:55:00 DEBUG "value": [ 18:55:00 DEBUG 18:55:00 DEBUG 18:55:00 DEBUG 18:55:00 DEBUG [...] 18:55:00 DEBUG 18:55:00 DEBUG 18:55:00 DEBUG 18:55:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:55:00 DEBUG }, 18:55:00 DEBUG "value": [ 18:55:00 DEBUG 1734461700, 18:55:00 DEBUG "0.009822723547387804" 18:55:00 DEBUG ] 18:55:00 DEBUG } 18:55:00 DEBUG ] 18:55:00 DEBUG } 18:55:00 DEBUG } 18:55:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461700" 18:55:00 INFO [http_cmd]: http status code OK 18:55:00 DEBUG --- status code --- 18:55:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:55:00 DEBUG --- http response --- 18:55:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:55:00 DEBUG ----- output ----- 18:55:00 DEBUG { 18:55:00 DEBUG "status": "success", 18:55:00 DEBUG "data": { 18:55:00 DEBUG "resultType": "vector", 18:55:00 DEBUG "result": [ 18:55:00 DEBUG { 18:55:00 DEBUG "metric": { 18:55:00 DEBUG "instance": "10.142.0.11:9100" 18:55:00 DEBUG }, 18:55:00 DEBUG "value": [ 18:55:00 DEBUG 18:55:00 DEBUG 18:55:00 DEBUG 18:55:00 DEBUG [...] 18:55:00 DEBUG 18:55:00 DEBUG 18:55:00 DEBUG 18:55:00 DEBUG "instance": "10.142.0.51:9100" 18:55:00 DEBUG }, 18:55:00 DEBUG "value": [ 18:55:00 DEBUG 1734461700, 18:55:00 DEBUG "88068.8666666667" 18:55:00 DEBUG ] 18:55:00 DEBUG } 18:55:00 DEBUG ] 18:55:00 DEBUG } 18:55:00 DEBUG } 18:55:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461700" 18:55:00 INFO [http_cmd]: http status code OK 18:55:00 DEBUG --- status code --- 18:55:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:55:00 DEBUG --- http response --- 18:55:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:55:00 DEBUG ----- output ----- 18:55:00 DEBUG { 18:55:00 DEBUG "status": "success", 18:55:00 DEBUG "data": { 18:55:00 DEBUG "resultType": "vector", 18:55:00 DEBUG "result": [ 18:55:00 DEBUG { 18:55:00 DEBUG "metric": { 18:55:00 DEBUG "instance": "10.142.0.11:9100" 18:55:00 DEBUG }, 18:55:00 DEBUG "value": [ 18:55:00 DEBUG 18:55:00 DEBUG 18:55:00 DEBUG 18:55:00 DEBUG [...] 18:55:00 DEBUG 18:55:00 DEBUG 18:55:00 DEBUG 18:55:00 DEBUG "instance": "10.142.0.51:9100" 18:55:00 DEBUG }, 18:55:00 DEBUG "value": [ 18:55:00 DEBUG 1734461700, 18:55:00 DEBUG "0" 18:55:00 DEBUG ] 18:55:00 DEBUG } 18:55:00 DEBUG ] 18:55:00 DEBUG } 18:55:00 DEBUG } 18:55:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461700" 18:55:00 INFO [http_cmd]: http status code OK 18:55:00 DEBUG --- status code --- 18:55:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:55:00 DEBUG --- http response --- 18:55:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:55:00 DEBUG ----- output ----- 18:55:00 DEBUG { 18:55:00 DEBUG "status": "success", 18:55:00 DEBUG "data": { 18:55:00 DEBUG "resultType": "vector", 18:55:00 DEBUG "result": [ 18:55:00 DEBUG { 18:55:00 DEBUG "metric": { 18:55:00 DEBUG "instance": "10.142.0.11:9100" 18:55:00 DEBUG }, 18:55:00 DEBUG "value": [ 18:55:00 DEBUG 18:55:00 DEBUG 18:55:00 DEBUG 18:55:00 DEBUG [...] 18:55:00 DEBUG 18:55:00 DEBUG 18:55:00 DEBUG 18:55:00 DEBUG "instance": "10.142.0.51:9100" 18:55:00 DEBUG }, 18:55:00 DEBUG "value": [ 18:55:00 DEBUG 1734461700, 18:55:00 DEBUG "0.00046666666666605284" 18:55:00 DEBUG ] 18:55:00 DEBUG } 18:55:00 DEBUG ] 18:55:00 DEBUG } 18:55:00 DEBUG } 18:55:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461700" 18:55:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734461700" 18:55:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461700" 18:55:00 INFO [http_cmd]: http status code OK 18:55:00 DEBUG --- status code --- 18:55:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:55:00 DEBUG --- http response --- 18:55:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:55:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461700" 18:55:00 DEBUG ----- output ----- 18:55:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734461700" 18:55:00 DEBUG { 18:55:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461700" 18:55:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461700" 18:55:00 DEBUG "status": "success", 18:55:00 DEBUG "data": { 18:55:00 DEBUG "resultType": "vector", 18:55:00 DEBUG "result": [ 18:55:00 DEBUG { 18:55:00 DEBUG "metric": { 18:55:00 DEBUG "pod": "ds-cts-0" 18:55:00 DEBUG }, 18:55:00 DEBUG "value": [ 18:55:00 DEBUG 18:55:00 DEBUG 18:55:00 DEBUG 18:55:00 DEBUG [...] 18:55:00 DEBUG 18:55:00 DEBUG 18:55:00 DEBUG 18:55:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:55:00 DEBUG }, 18:55:00 DEBUG "value": [ 18:55:00 DEBUG 1734461700, 18:55:00 DEBUG "0" 18:55:00 DEBUG ] 18:55:00 DEBUG } 18:55:00 DEBUG ] 18:55:00 DEBUG } 18:55:00 DEBUG } 18:55:00 INFO [http_cmd]: http status code OK 18:55:00 DEBUG --- status code --- 18:55:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:55:00 INFO [http_cmd]: http status code OK 18:55:00 DEBUG --- http response --- 18:55:00 DEBUG --- status code --- 18:55:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:55:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:55:00 DEBUG ----- output ----- 18:55:00 DEBUG --- http response --- 18:55:00 DEBUG { 18:55:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:55:00 DEBUG "status": "success", 18:55:00 DEBUG ----- output ----- 18:55:00 DEBUG "data": { 18:55:00 DEBUG { 18:55:00 DEBUG "resultType": "vector", 18:55:00 DEBUG "status": "success", 18:55:00 DEBUG "result": [ 18:55:00 INFO [http_cmd]: http status code OK 18:55:00 DEBUG "data": { 18:55:00 INFO [http_cmd]: http status code OK 18:55:00 DEBUG { 18:55:00 DEBUG --- status code --- 18:55:00 DEBUG "resultType": "vector", 18:55:00 INFO [http_cmd]: http status code OK 18:55:00 DEBUG --- status code --- 18:55:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:55:00 DEBUG --- http response --- 18:55:00 INFO [http_cmd]: http status code OK 18:55:00 DEBUG "result": [ 18:55:00 DEBUG { 18:55:00 DEBUG "metric": { 18:55:00 DEBUG "instance": "10.142.0.11:9100" 18:55:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:55:00 DEBUG --- status code --- 18:55:00 DEBUG --- status code --- 18:55:00 DEBUG "metric": { 18:55:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:55:00 DEBUG }, 18:55:00 DEBUG ----- output ----- 18:55:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:55:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:55:00 DEBUG --- http response --- 18:55:00 DEBUG --- http response --- 18:55:00 DEBUG "value": [ 18:55:00 DEBUG { 18:55:00 DEBUG --- http response --- 18:55:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:55:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:55:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:55:00 DEBUG 18:55:00 DEBUG "status": "success", 18:55:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:55:00 DEBUG }, 18:55:00 DEBUG ----- output ----- 18:55:00 DEBUG ----- output ----- 18:55:00 DEBUG 18:55:00 DEBUG "data": { 18:55:00 DEBUG ----- output ----- 18:55:00 DEBUG "value": [ 18:55:00 DEBUG { 18:55:00 DEBUG { 18:55:00 DEBUG 18:55:00 DEBUG "resultType": "vector", 18:55:00 DEBUG { 18:55:00 DEBUG 18:55:00 DEBUG "status": "success", 18:55:00 DEBUG "status": "success", 18:55:00 DEBUG [...] 18:55:00 DEBUG "result": [ 18:55:00 DEBUG "status": "success", 18:55:00 DEBUG 18:55:00 DEBUG "data": { 18:55:00 DEBUG "data": { 18:55:00 DEBUG 18:55:00 DEBUG { 18:55:00 DEBUG "data": { 18:55:00 DEBUG 18:55:00 DEBUG "resultType": "vector", 18:55:00 DEBUG "resultType": "vector", 18:55:00 DEBUG 18:55:00 DEBUG "metric": { 18:55:00 DEBUG "resultType": "vector", 18:55:00 DEBUG [...] 18:55:00 DEBUG "result": [ 18:55:00 DEBUG "result": [ 18:55:00 DEBUG 18:55:00 DEBUG "instance": "10.142.0.11:9100" 18:55:00 DEBUG "result": [ 18:55:00 DEBUG 18:55:00 DEBUG { 18:55:00 DEBUG { 18:55:00 DEBUG "instance": "10.142.0.51:9100" 18:55:00 DEBUG }, 18:55:00 DEBUG { 18:55:00 DEBUG 18:55:00 DEBUG "metric": { 18:55:00 DEBUG "metric": { 18:55:00 DEBUG }, 18:55:00 DEBUG "value": [ 18:55:00 DEBUG "metric": { 18:55:00 DEBUG 18:55:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:55:00 DEBUG "pod": "ds-cts-0" 18:55:00 DEBUG "value": [ 18:55:00 DEBUG 18:55:00 DEBUG "pod": "ds-cts-0" 18:55:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:55:00 DEBUG }, 18:55:00 DEBUG }, 18:55:00 DEBUG 1734461700, 18:55:00 DEBUG 18:55:00 DEBUG }, 18:55:00 DEBUG }, 18:55:00 DEBUG "value": [ 18:55:00 DEBUG "value": [ 18:55:00 DEBUG "72456.33333333336" 18:55:00 DEBUG 18:55:00 DEBUG "value": [ 18:55:00 DEBUG "value": [ 18:55:00 DEBUG 18:55:00 DEBUG 18:55:00 DEBUG ] 18:55:00 DEBUG [...] 18:55:00 DEBUG 18:55:00 DEBUG 1734461700, 18:55:00 DEBUG 18:55:00 DEBUG 18:55:00 DEBUG } 18:55:00 DEBUG 18:55:00 DEBUG 18:55:00 DEBUG "1.7812026905655256" 18:55:00 DEBUG 18:55:00 DEBUG 18:55:00 DEBUG ] 18:55:00 DEBUG 18:55:00 DEBUG 18:55:00 DEBUG ] 18:55:00 DEBUG [...] 18:55:00 DEBUG [...] 18:55:00 DEBUG } 18:55:00 DEBUG 18:55:00 DEBUG [...] 18:55:00 DEBUG } 18:55:00 DEBUG 18:55:00 DEBUG 18:55:00 DEBUG } 18:55:00 DEBUG "instance": "10.142.0.51:9100" 18:55:00 DEBUG 18:55:00 DEBUG ] 18:55:00 DEBUG 18:55:00 DEBUG 18:55:00 DEBUG }, 18:55:00 DEBUG 18:55:00 DEBUG } 18:55:00 DEBUG 18:55:00 DEBUG 18:55:00 DEBUG "value": [ 18:55:00 DEBUG 18:55:00 DEBUG } 18:55:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:55:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:55:00 DEBUG }, 18:55:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:55:00 DEBUG }, 18:55:00 DEBUG 1734461700, 18:55:00 DEBUG "value": [ 18:55:00 DEBUG }, 18:55:00 DEBUG "value": [ 18:55:00 DEBUG "25258.66666666667" 18:55:00 DEBUG 1734461700, 18:55:00 DEBUG "value": [ 18:55:00 DEBUG 1734461700, 18:55:00 DEBUG ] 18:55:00 DEBUG "695463936" 18:55:00 DEBUG 1734461700, 18:55:00 DEBUG "2585767936" 18:55:00 DEBUG } 18:55:00 DEBUG ] 18:55:00 DEBUG } 18:55:00 DEBUG } 18:55:00 DEBUG ] 18:55:00 DEBUG "569.2644452937701" 18:55:00 DEBUG ] 18:55:00 DEBUG } 18:55:00 DEBUG ] 18:55:00 DEBUG } 18:55:00 DEBUG ] 18:55:00 DEBUG } 18:55:00 DEBUG ] 18:55:00 DEBUG } 18:55:00 DEBUG ] 18:55:00 DEBUG } 18:55:00 DEBUG } 18:55:00 DEBUG } 18:55:00 DEBUG } 18:55:00 DEBUG } 18:55:02 INFO 18:55:02 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:55:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:55:02 INFO [loop_until]: OK (rc = 0) 18:55:02 DEBUG --- stdout --- 18:55:02 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 543Mi ds-idrepo-0 7m 737Mi rcs-5d696ff5b-7ttvr 3m 297Mi rcs-5d696ff5b-wwlbr 2m 331Mi 18:55:02 DEBUG --- stderr --- 18:55:02 DEBUG 18:55:02 INFO 18:55:02 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:55:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:55:02 INFO [loop_until]: OK (rc = 0) 18:55:02 DEBUG --- stdout --- 18:55:02 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 100m 0% 2279Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1848m 5% 3630Mi 12% 18:55:02 DEBUG --- stderr --- 18:55:02 DEBUG 18:55:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461759" 18:55:59 INFO [http_cmd]: http status code OK 18:55:59 DEBUG --- status code --- 18:55:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:55:59 DEBUG --- http response --- 18:55:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:55:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461759" 18:55:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734461759" 18:55:59 INFO [http_cmd]: http status code OK 18:55:59 INFO [http_cmd]: http status code OK 18:55:59 DEBUG --- status code --- 18:55:59 DEBUG --- status code --- 18:55:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:55:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:55:59 DEBUG --- http response --- 18:55:59 DEBUG --- http response --- 18:55:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:55:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:55:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461759" 18:55:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461759" 18:55:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461759" 18:55:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461759" 18:55:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461759" 18:55:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461759" 18:55:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461759" 18:55:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461759" 18:55:59 INFO [http_cmd]: http status code OK 18:55:59 INFO [http_cmd]: http status code OK 18:55:59 INFO [http_cmd]: http status code OK 18:55:59 DEBUG --- status code --- 18:55:59 DEBUG --- status code --- 18:55:59 DEBUG --- status code --- 18:55:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:55:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:55:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:55:59 DEBUG --- http response --- 18:55:59 INFO [http_cmd]: http status code OK 18:55:59 DEBUG --- http response --- 18:55:59 DEBUG --- http response --- 18:55:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:55:59 DEBUG --- status code --- 18:55:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:55:59 INFO [http_cmd]: http status code OK 18:55:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:55:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:55:59 INFO [http_cmd]: http status code OK 18:55:59 DEBUG --- status code --- 18:55:59 DEBUG --- http response --- 18:55:59 DEBUG --- status code --- 18:55:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:55:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:55:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:55:59 DEBUG --- http response --- 18:55:59 DEBUG --- http response --- 18:55:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:55:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:55:59 INFO [http_cmd]: http status code OK 18:55:59 INFO [http_cmd]: http status code OK 18:55:59 DEBUG --- status code --- 18:55:59 DEBUG --- status code --- 18:55:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:55:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:55:59 DEBUG --- http response --- 18:55:59 DEBUG --- http response --- 18:55:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:55:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:55:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461759" 18:55:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734461759" 18:55:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461759" 18:55:59 INFO [http_cmd]: http status code OK 18:55:59 DEBUG --- status code --- 18:55:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:55:59 DEBUG --- http response --- 18:55:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:55:59 INFO [http_cmd]: http status code OK 18:55:59 ERROR [http_cmd]: ERROR 18:55:59 DEBUG --- status code --- 18:55:59 ERROR --- status code --- 18:55:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:55:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:55:59 DEBUG --- http response --- 18:55:59 ERROR --- http response --- 18:55:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:55:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:55:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:55:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461759" 18:55:59 INFO [http_cmd]: http status code OK 18:55:59 DEBUG --- status code --- 18:55:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:55:59 DEBUG --- http response --- 18:55:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:55:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461759" 18:55:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461759" 18:55:59 INFO [http_cmd]: http status code OK 18:55:59 DEBUG --- status code --- 18:55:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:55:59 DEBUG --- http response --- 18:55:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:55:59 INFO [http_cmd]: http status code OK 18:55:59 DEBUG --- status code --- 18:55:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:55:59 DEBUG --- http response --- 18:55:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:55:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461759" 18:55:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461759" 18:55:59 INFO [http_cmd]: http status code OK 18:55:59 DEBUG --- status code --- 18:55:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:55:59 DEBUG --- http response --- 18:55:59 INFO [http_cmd]: http status code OK 18:55:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:55:59 DEBUG --- status code --- 18:55:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:55:59 DEBUG --- http response --- 18:55:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:55:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461759" 18:55:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734461759" 18:55:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734461759" 18:55:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734461759" 18:55:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461759" 18:55:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461759" 18:55:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461759" 18:55:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461759" 18:55:59 INFO [http_cmd]: http status code OK 18:55:59 DEBUG --- status code --- 18:55:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:55:59 INFO [http_cmd]: http status code OK 18:55:59 INFO [http_cmd]: http status code OK 18:55:59 DEBUG --- http response --- 18:55:59 DEBUG --- status code --- 18:55:59 DEBUG --- status code --- 18:55:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:55:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:55:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:55:59 DEBUG --- http response --- 18:55:59 DEBUG --- http response --- 18:55:59 INFO [http_cmd]: http status code OK 18:55:59 DEBUG --- status code --- 18:55:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:55:59 DEBUG --- http response --- 18:55:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:55:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:55:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:55:59 INFO [http_cmd]: http status code OK 18:55:59 DEBUG --- status code --- 18:55:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:55:59 INFO [http_cmd]: http status code OK 18:55:59 DEBUG --- http response --- 18:55:59 DEBUG --- status code --- 18:55:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:55:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:55:59 DEBUG --- http response --- 18:55:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:55:59 INFO [http_cmd]: http status code OK 18:55:59 DEBUG --- status code --- 18:55:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:55:59 DEBUG --- http response --- 18:55:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:55:59 INFO [http_cmd]: http status code OK 18:55:59 DEBUG --- status code --- 18:55:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:55:59 DEBUG --- http response --- 18:55:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:56:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461760" 18:56:00 INFO [http_cmd]: http status code OK 18:56:00 DEBUG --- status code --- 18:56:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:56:00 DEBUG --- http response --- 18:56:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:56:00 DEBUG ----- output ----- 18:56:00 DEBUG { 18:56:00 DEBUG "status": "success", 18:56:00 DEBUG "data": { 18:56:00 DEBUG "resultType": "vector", 18:56:00 DEBUG "result": [ 18:56:00 DEBUG { 18:56:00 DEBUG "metric": { 18:56:00 DEBUG "pod": "ds-cts-0" 18:56:00 DEBUG }, 18:56:00 DEBUG "value": [ 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG [...] 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:56:00 DEBUG }, 18:56:00 DEBUG "value": [ 18:56:00 DEBUG 1734461760, 18:56:00 DEBUG "0.004138149158622434" 18:56:00 DEBUG ] 18:56:00 DEBUG } 18:56:00 DEBUG ] 18:56:00 DEBUG } 18:56:00 DEBUG } 18:56:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461760" 18:56:00 INFO [http_cmd]: http status code OK 18:56:00 DEBUG --- status code --- 18:56:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:56:00 DEBUG --- http response --- 18:56:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:56:00 DEBUG ----- output ----- 18:56:00 DEBUG { 18:56:00 DEBUG "status": "success", 18:56:00 DEBUG "data": { 18:56:00 DEBUG "resultType": "vector", 18:56:00 DEBUG "result": [ 18:56:00 DEBUG { 18:56:00 DEBUG "metric": { 18:56:00 DEBUG "instance": "10.142.0.11:9100" 18:56:00 DEBUG }, 18:56:00 DEBUG "value": [ 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG [...] 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG "instance": "10.142.0.51:9100" 18:56:00 DEBUG }, 18:56:00 DEBUG "value": [ 18:56:00 DEBUG 1734461760, 18:56:00 DEBUG "87389.60000000002" 18:56:00 DEBUG ] 18:56:00 DEBUG } 18:56:00 DEBUG ] 18:56:00 DEBUG } 18:56:00 DEBUG } 18:56:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461760" 18:56:00 INFO [http_cmd]: http status code OK 18:56:00 DEBUG --- status code --- 18:56:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:56:00 DEBUG --- http response --- 18:56:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:56:00 DEBUG ----- output ----- 18:56:00 DEBUG { 18:56:00 DEBUG "status": "success", 18:56:00 DEBUG "data": { 18:56:00 DEBUG "resultType": "vector", 18:56:00 DEBUG "result": [ 18:56:00 DEBUG { 18:56:00 DEBUG "metric": { 18:56:00 DEBUG "instance": "10.142.0.11:9100" 18:56:00 DEBUG }, 18:56:00 DEBUG "value": [ 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG [...] 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG "instance": "10.142.0.51:9100" 18:56:00 DEBUG }, 18:56:00 DEBUG "value": [ 18:56:00 DEBUG 1734461760, 18:56:00 DEBUG "0.0003333333333332671" 18:56:00 DEBUG ] 18:56:00 DEBUG } 18:56:00 DEBUG ] 18:56:00 DEBUG } 18:56:00 DEBUG } 18:56:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461760" 18:56:00 INFO [http_cmd]: http status code OK 18:56:00 DEBUG --- status code --- 18:56:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:56:00 DEBUG --- http response --- 18:56:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:56:00 DEBUG ----- output ----- 18:56:00 DEBUG { 18:56:00 DEBUG "status": "success", 18:56:00 DEBUG "data": { 18:56:00 DEBUG "resultType": "vector", 18:56:00 DEBUG "result": [ 18:56:00 DEBUG { 18:56:00 DEBUG "metric": { 18:56:00 DEBUG "instance": "10.142.0.11:9100" 18:56:00 DEBUG }, 18:56:00 DEBUG "value": [ 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG [...] 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG "instance": "10.142.0.51:9100" 18:56:00 DEBUG }, 18:56:00 DEBUG "value": [ 18:56:00 DEBUG 1734461760, 18:56:00 DEBUG "0.0003666666666674701" 18:56:00 DEBUG ] 18:56:00 DEBUG } 18:56:00 DEBUG ] 18:56:00 DEBUG } 18:56:00 DEBUG } 18:56:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461760" 18:56:00 INFO [http_cmd]: http status code OK 18:56:00 DEBUG --- status code --- 18:56:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:56:00 DEBUG --- http response --- 18:56:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:56:00 DEBUG ----- output ----- 18:56:00 DEBUG { 18:56:00 DEBUG "status": "success", 18:56:00 DEBUG "data": { 18:56:00 DEBUG "resultType": "vector", 18:56:00 DEBUG "result": [ 18:56:00 DEBUG { 18:56:00 DEBUG "metric": { 18:56:00 DEBUG "pod": "ds-cts-0" 18:56:00 DEBUG }, 18:56:00 DEBUG "value": [ 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG [...] 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:56:00 DEBUG }, 18:56:00 DEBUG "value": [ 18:56:00 DEBUG 1734461760, 18:56:00 DEBUG "0" 18:56:00 DEBUG ] 18:56:00 DEBUG } 18:56:00 DEBUG ] 18:56:00 DEBUG } 18:56:00 DEBUG } 18:56:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461760" 18:56:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734461760" 18:56:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461760" 18:56:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461760" 18:56:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461760" 18:56:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734461760" 18:56:00 INFO [http_cmd]: http status code OK 18:56:00 DEBUG --- status code --- 18:56:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:56:00 DEBUG --- http response --- 18:56:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:56:00 DEBUG ----- output ----- 18:56:00 DEBUG { 18:56:00 DEBUG "status": "success", 18:56:00 DEBUG "data": { 18:56:00 DEBUG "resultType": "vector", 18:56:00 DEBUG "result": [ 18:56:00 DEBUG { 18:56:00 DEBUG "metric": { 18:56:00 DEBUG "pod": "ds-cts-0" 18:56:00 DEBUG }, 18:56:00 DEBUG "value": [ 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG [...] 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:56:00 DEBUG }, 18:56:00 DEBUG "value": [ 18:56:00 DEBUG 1734461760, 18:56:00 DEBUG "1294.713717336473" 18:56:00 DEBUG ] 18:56:00 DEBUG } 18:56:00 DEBUG ] 18:56:00 DEBUG } 18:56:00 DEBUG } 18:56:00 INFO [http_cmd]: http status code OK 18:56:00 DEBUG --- status code --- 18:56:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:56:00 DEBUG --- http response --- 18:56:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:56:00 DEBUG ----- output ----- 18:56:00 DEBUG { 18:56:00 DEBUG "status": "success", 18:56:00 DEBUG "data": { 18:56:00 DEBUG "resultType": "vector", 18:56:00 DEBUG "result": [ 18:56:00 DEBUG { 18:56:00 DEBUG "metric": { 18:56:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:56:00 DEBUG }, 18:56:00 DEBUG "value": [ 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG [...] 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:56:00 DEBUG }, 18:56:00 DEBUG "value": [ 18:56:00 DEBUG 1734461760, 18:56:00 DEBUG "2560557056" 18:56:00 DEBUG ] 18:56:00 DEBUG } 18:56:00 DEBUG ] 18:56:00 DEBUG } 18:56:00 DEBUG } 18:56:00 INFO [http_cmd]: http status code OK 18:56:00 INFO [http_cmd]: http status code OK 18:56:00 INFO [http_cmd]: http status code OK 18:56:00 DEBUG --- status code --- 18:56:00 INFO [http_cmd]: http status code OK 18:56:00 DEBUG --- status code --- 18:56:00 DEBUG --- status code --- 18:56:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:56:00 DEBUG --- status code --- 18:56:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:56:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:56:00 DEBUG --- http response --- 18:56:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:56:00 DEBUG --- http response --- 18:56:00 DEBUG --- http response --- 18:56:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:56:00 DEBUG --- http response --- 18:56:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:56:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:56:00 DEBUG ----- output ----- 18:56:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:56:00 DEBUG ----- output ----- 18:56:00 DEBUG ----- output ----- 18:56:00 DEBUG { 18:56:00 DEBUG ----- output ----- 18:56:00 DEBUG { 18:56:00 DEBUG { 18:56:00 DEBUG "status": "success", 18:56:00 DEBUG { 18:56:00 DEBUG "status": "success", 18:56:00 DEBUG "status": "success", 18:56:00 DEBUG "data": { 18:56:00 DEBUG "data": { 18:56:00 DEBUG "data": { 18:56:00 DEBUG "status": "success", 18:56:00 DEBUG "resultType": "vector", 18:56:00 DEBUG "resultType": "vector", 18:56:00 DEBUG "resultType": "vector", 18:56:00 DEBUG "data": { 18:56:00 DEBUG "result": [ 18:56:00 DEBUG "result": [ 18:56:00 DEBUG "result": [ 18:56:00 DEBUG "resultType": "vector", 18:56:00 DEBUG { 18:56:00 DEBUG { 18:56:00 DEBUG { 18:56:00 DEBUG "result": [ 18:56:00 DEBUG "metric": { 18:56:00 DEBUG "metric": { 18:56:00 DEBUG "metric": { 18:56:00 DEBUG { 18:56:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:56:00 DEBUG "instance": "10.142.0.11:9100" 18:56:00 DEBUG "pod": "ds-cts-0" 18:56:00 DEBUG "metric": { 18:56:00 DEBUG }, 18:56:00 DEBUG }, 18:56:00 DEBUG }, 18:56:00 DEBUG "value": [ 18:56:00 DEBUG "value": [ 18:56:00 DEBUG "value": [ 18:56:00 DEBUG "instance": "10.142.0.11:9100" 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG }, 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG "value": [ 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG [...] 18:56:00 DEBUG [...] 18:56:00 DEBUG [...] 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG [...] 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG 18:56:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:56:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:56:00 DEBUG "instance": "10.142.0.51:9100" 18:56:00 DEBUG 18:56:00 DEBUG }, 18:56:00 DEBUG }, 18:56:00 DEBUG }, 18:56:00 DEBUG 18:56:00 DEBUG "value": [ 18:56:00 DEBUG "value": [ 18:56:00 DEBUG "value": [ 18:56:00 DEBUG "instance": "10.142.0.51:9100" 18:56:00 DEBUG 1734461760, 18:56:00 DEBUG 1734461760, 18:56:00 DEBUG 1734461760, 18:56:00 DEBUG }, 18:56:00 DEBUG "695492608" 18:56:00 DEBUG "1.776182991313331" 18:56:00 DEBUG "24712.53333333334" 18:56:00 DEBUG "value": [ 18:56:00 DEBUG ] 18:56:00 DEBUG ] 18:56:00 DEBUG ] 18:56:00 DEBUG 1734461760, 18:56:00 DEBUG } 18:56:00 DEBUG } 18:56:00 DEBUG } 18:56:00 DEBUG "71798.63333333335" 18:56:00 DEBUG ] 18:56:00 DEBUG ] 18:56:00 DEBUG ] 18:56:00 DEBUG ] 18:56:00 DEBUG } 18:56:00 DEBUG } 18:56:00 DEBUG } 18:56:00 DEBUG } 18:56:00 DEBUG } 18:56:00 DEBUG } 18:56:00 DEBUG } 18:56:00 DEBUG ] 18:56:00 DEBUG } 18:56:00 DEBUG } 18:56:02 INFO 18:56:02 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:56:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:56:02 INFO [loop_until]: OK (rc = 0) 18:56:02 DEBUG --- stdout --- 18:56:02 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 544Mi ds-idrepo-0 7m 738Mi rcs-5d696ff5b-7ttvr 5m 296Mi rcs-5d696ff5b-wwlbr 3m 331Mi 18:56:02 DEBUG --- stderr --- 18:56:02 DEBUG 18:56:02 INFO 18:56:02 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:56:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:56:02 INFO [loop_until]: OK (rc = 0) 18:56:02 DEBUG --- stdout --- 18:56:02 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 92m 0% 2279Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1853m 5% 3630Mi 12% 18:56:02 DEBUG --- stderr --- 18:56:02 DEBUG 18:56:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461819" 18:56:59 INFO [http_cmd]: http status code OK 18:56:59 DEBUG --- status code --- 18:56:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:56:59 DEBUG --- http response --- 18:56:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:56:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461819" 18:56:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734461819" 18:56:59 INFO [http_cmd]: http status code OK 18:56:59 INFO [http_cmd]: http status code OK 18:56:59 DEBUG --- status code --- 18:56:59 DEBUG --- status code --- 18:56:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:56:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:56:59 DEBUG --- http response --- 18:56:59 DEBUG --- http response --- 18:56:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:56:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:56:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461819" 18:56:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461819" 18:56:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461819" 18:56:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461819" 18:56:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461819" 18:56:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461819" 18:56:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461819" 18:56:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461819" 18:56:59 INFO [http_cmd]: http status code OK 18:56:59 INFO [http_cmd]: http status code OK 18:56:59 DEBUG --- status code --- 18:56:59 DEBUG --- status code --- 18:56:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:56:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:56:59 DEBUG --- http response --- 18:56:59 DEBUG --- http response --- 18:56:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:56:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:56:59 INFO [http_cmd]: http status code OK 18:56:59 DEBUG --- status code --- 18:56:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:56:59 DEBUG --- http response --- 18:56:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:56:59 INFO [http_cmd]: http status code OK 18:56:59 INFO [http_cmd]: http status code OK 18:56:59 INFO [http_cmd]: http status code OK 18:56:59 DEBUG --- status code --- 18:56:59 DEBUG --- status code --- 18:56:59 DEBUG --- status code --- 18:56:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:56:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:56:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:56:59 DEBUG --- http response --- 18:56:59 DEBUG --- http response --- 18:56:59 DEBUG --- http response --- 18:56:59 INFO [http_cmd]: http status code OK 18:56:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:56:59 INFO [http_cmd]: http status code OK 18:56:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:56:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:56:59 DEBUG --- status code --- 18:56:59 DEBUG --- status code --- 18:56:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:56:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:56:59 DEBUG --- http response --- 18:56:59 DEBUG --- http response --- 18:56:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:56:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:56:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461819" 18:56:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734461819" 18:56:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461819" 18:56:59 INFO [http_cmd]: http status code OK 18:56:59 DEBUG --- status code --- 18:56:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:56:59 DEBUG --- http response --- 18:56:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:56:59 INFO [http_cmd]: http status code OK 18:56:59 ERROR [http_cmd]: ERROR 18:56:59 DEBUG --- status code --- 18:56:59 ERROR --- status code --- 18:56:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:56:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:56:59 DEBUG --- http response --- 18:56:59 ERROR --- http response --- 18:56:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:56:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:56:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:56:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461819" 18:56:59 INFO [http_cmd]: http status code OK 18:56:59 DEBUG --- status code --- 18:56:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:56:59 DEBUG --- http response --- 18:56:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:56:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461819" 18:56:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461819" 18:56:59 INFO [http_cmd]: http status code OK 18:56:59 DEBUG --- status code --- 18:56:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:56:59 DEBUG --- http response --- 18:56:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:56:59 INFO [http_cmd]: http status code OK 18:56:59 DEBUG --- status code --- 18:56:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:56:59 DEBUG --- http response --- 18:56:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:56:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461819" 18:56:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461819" 18:56:59 INFO [http_cmd]: http status code OK 18:56:59 DEBUG --- status code --- 18:56:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:56:59 DEBUG --- http response --- 18:56:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:56:59 INFO [http_cmd]: http status code OK 18:56:59 DEBUG --- status code --- 18:56:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:56:59 DEBUG --- http response --- 18:56:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:56:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461819" 18:56:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734461819" 18:56:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734461819" 18:56:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734461819" 18:56:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461819" 18:56:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461819" 18:56:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461819" 18:56:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461819" 18:56:59 INFO [http_cmd]: http status code OK 18:56:59 DEBUG --- status code --- 18:56:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:56:59 DEBUG --- http response --- 18:56:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:56:59 INFO [http_cmd]: http status code OK 18:56:59 DEBUG --- status code --- 18:56:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:56:59 DEBUG --- http response --- 18:56:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:56:59 INFO [http_cmd]: http status code OK 18:56:59 DEBUG --- status code --- 18:56:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:56:59 DEBUG --- http response --- 18:56:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:56:59 INFO [http_cmd]: http status code OK 18:56:59 DEBUG --- status code --- 18:56:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:56:59 DEBUG --- http response --- 18:56:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:56:59 INFO [http_cmd]: http status code OK 18:56:59 DEBUG --- status code --- 18:56:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:56:59 DEBUG --- http response --- 18:56:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:56:59 INFO [http_cmd]: http status code OK 18:56:59 DEBUG --- status code --- 18:56:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:56:59 DEBUG --- http response --- 18:56:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:56:59 INFO [http_cmd]: http status code OK 18:56:59 DEBUG --- status code --- 18:56:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:56:59 DEBUG --- http response --- 18:56:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:56:59 INFO [http_cmd]: http status code OK 18:56:59 DEBUG --- status code --- 18:56:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:56:59 DEBUG --- http response --- 18:56:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:57:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461820" 18:57:00 INFO [http_cmd]: http status code OK 18:57:00 DEBUG --- status code --- 18:57:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:57:00 DEBUG --- http response --- 18:57:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:57:00 DEBUG ----- output ----- 18:57:00 DEBUG { 18:57:00 DEBUG "status": "success", 18:57:00 DEBUG "data": { 18:57:00 DEBUG "resultType": "vector", 18:57:00 DEBUG "result": [ 18:57:00 DEBUG { 18:57:00 DEBUG "metric": { 18:57:00 DEBUG "pod": "ds-cts-0" 18:57:00 DEBUG }, 18:57:00 DEBUG "value": [ 18:57:00 DEBUG 18:57:00 DEBUG 18:57:00 DEBUG 18:57:00 DEBUG [...] 18:57:00 DEBUG 18:57:00 DEBUG 18:57:00 DEBUG 18:57:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:57:00 DEBUG }, 18:57:00 DEBUG "value": [ 18:57:00 DEBUG 1734461820, 18:57:00 DEBUG "0.0036814086491325565" 18:57:00 DEBUG ] 18:57:00 DEBUG } 18:57:00 DEBUG ] 18:57:00 DEBUG } 18:57:00 DEBUG } 18:57:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461820" 18:57:00 INFO [http_cmd]: http status code OK 18:57:00 DEBUG --- status code --- 18:57:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:57:00 DEBUG --- http response --- 18:57:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:57:00 DEBUG ----- output ----- 18:57:00 DEBUG { 18:57:00 DEBUG "status": "success", 18:57:00 DEBUG "data": { 18:57:00 DEBUG "resultType": "vector", 18:57:00 DEBUG "result": [ 18:57:00 DEBUG { 18:57:00 DEBUG "metric": { 18:57:00 DEBUG "instance": "10.142.0.11:9100" 18:57:00 DEBUG }, 18:57:00 DEBUG "value": [ 18:57:00 DEBUG 18:57:00 DEBUG 18:57:00 DEBUG 18:57:00 DEBUG [...] 18:57:00 DEBUG 18:57:00 DEBUG 18:57:00 DEBUG 18:57:00 DEBUG "instance": "10.142.0.51:9100" 18:57:00 DEBUG }, 18:57:00 DEBUG "value": [ 18:57:00 DEBUG 1734461820, 18:57:00 DEBUG "90829.56666666667" 18:57:00 DEBUG ] 18:57:00 DEBUG } 18:57:00 DEBUG ] 18:57:00 DEBUG } 18:57:00 DEBUG } 18:57:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461820" 18:57:00 INFO [http_cmd]: http status code OK 18:57:00 DEBUG --- status code --- 18:57:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:57:00 DEBUG --- http response --- 18:57:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:57:00 DEBUG ----- output ----- 18:57:00 DEBUG { 18:57:00 DEBUG "status": "success", 18:57:00 DEBUG "data": { 18:57:00 DEBUG "resultType": "vector", 18:57:00 DEBUG "result": [ 18:57:00 DEBUG { 18:57:00 DEBUG "metric": { 18:57:00 DEBUG "instance": "10.142.0.11:9100" 18:57:00 DEBUG }, 18:57:00 DEBUG "value": [ 18:57:00 DEBUG 18:57:00 DEBUG 18:57:00 DEBUG 18:57:00 DEBUG [...] 18:57:00 DEBUG 18:57:00 DEBUG 18:57:00 DEBUG 18:57:00 DEBUG "instance": "10.142.0.51:9100" 18:57:00 DEBUG }, 18:57:00 DEBUG "value": [ 18:57:00 DEBUG 1734461820, 18:57:00 DEBUG "0.0006666666666666527" 18:57:00 DEBUG ] 18:57:00 DEBUG } 18:57:00 DEBUG ] 18:57:00 DEBUG } 18:57:00 DEBUG } 18:57:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461820" 18:57:00 INFO [http_cmd]: http status code OK 18:57:00 DEBUG --- status code --- 18:57:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:57:00 DEBUG --- http response --- 18:57:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:57:00 DEBUG ----- output ----- 18:57:00 DEBUG { 18:57:00 DEBUG "status": "success", 18:57:00 DEBUG "data": { 18:57:00 DEBUG "resultType": "vector", 18:57:00 DEBUG "result": [ 18:57:00 DEBUG { 18:57:00 DEBUG "metric": { 18:57:00 DEBUG "instance": "10.142.0.11:9100" 18:57:00 DEBUG }, 18:57:00 DEBUG "value": [ 18:57:00 DEBUG 18:57:00 DEBUG 18:57:00 DEBUG 18:57:00 DEBUG [...] 18:57:00 DEBUG 18:57:00 DEBUG 18:57:00 DEBUG 18:57:00 DEBUG "instance": "10.142.0.51:9100" 18:57:00 DEBUG }, 18:57:00 DEBUG "value": [ 18:57:00 DEBUG 1734461820, 18:57:00 DEBUG "0.0004333333333325604" 18:57:00 DEBUG ] 18:57:00 DEBUG } 18:57:00 DEBUG ] 18:57:00 DEBUG } 18:57:00 DEBUG } 18:57:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461820" 18:57:00 INFO [http_cmd]: http status code OK 18:57:00 DEBUG --- status code --- 18:57:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:57:00 DEBUG --- http response --- 18:57:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:57:00 DEBUG ----- output ----- 18:57:00 DEBUG { 18:57:00 DEBUG "status": "success", 18:57:00 DEBUG "data": { 18:57:00 DEBUG "resultType": "vector", 18:57:00 DEBUG "result": [ 18:57:00 DEBUG { 18:57:00 DEBUG "metric": { 18:57:00 DEBUG "pod": "ds-cts-0" 18:57:00 DEBUG }, 18:57:00 DEBUG "value": [ 18:57:00 DEBUG 18:57:00 DEBUG 18:57:00 DEBUG 18:57:00 DEBUG [...] 18:57:00 DEBUG 18:57:00 DEBUG 18:57:00 DEBUG 18:57:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:57:00 DEBUG }, 18:57:00 DEBUG "value": [ 18:57:00 DEBUG 1734461820, 18:57:00 DEBUG "0" 18:57:00 DEBUG ] 18:57:00 DEBUG } 18:57:00 DEBUG ] 18:57:00 DEBUG } 18:57:00 DEBUG } 18:57:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461820" 18:57:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734461820" 18:57:00 INFO [http_cmd]: http status code OK 18:57:00 DEBUG --- status code --- 18:57:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:57:00 DEBUG --- http response --- 18:57:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:57:00 DEBUG ----- output ----- 18:57:00 DEBUG { 18:57:00 DEBUG "status": "success", 18:57:00 DEBUG "data": { 18:57:00 DEBUG "resultType": "vector", 18:57:00 DEBUG "result": [ 18:57:00 DEBUG { 18:57:00 DEBUG "metric": { 18:57:00 DEBUG "pod": "ds-cts-0" 18:57:00 DEBUG }, 18:57:00 DEBUG "value": [ 18:57:00 DEBUG 18:57:00 DEBUG 18:57:00 DEBUG 18:57:00 DEBUG [...] 18:57:00 DEBUG 18:57:00 DEBUG 18:57:00 DEBUG 18:57:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:57:00 DEBUG }, 18:57:00 DEBUG "value": [ 18:57:00 DEBUG 1734461820, 18:57:00 DEBUG "1659.9624541913208" 18:57:00 DEBUG ] 18:57:00 DEBUG } 18:57:00 DEBUG ] 18:57:00 DEBUG } 18:57:00 DEBUG } 18:57:00 INFO [http_cmd]: http status code OK 18:57:00 DEBUG --- status code --- 18:57:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:57:00 DEBUG --- http response --- 18:57:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:57:00 DEBUG ----- output ----- 18:57:00 DEBUG { 18:57:00 DEBUG "status": "success", 18:57:00 DEBUG "data": { 18:57:00 DEBUG "resultType": "vector", 18:57:00 DEBUG "result": [ 18:57:00 DEBUG { 18:57:00 DEBUG "metric": { 18:57:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:57:00 DEBUG }, 18:57:00 DEBUG "value": [ 18:57:00 DEBUG 18:57:00 DEBUG 18:57:00 DEBUG 18:57:00 DEBUG [...] 18:57:00 DEBUG 18:57:00 DEBUG 18:57:00 DEBUG 18:57:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:57:00 DEBUG }, 18:57:00 DEBUG "value": [ 18:57:00 DEBUG 1734461820, 18:57:00 DEBUG "2557018112" 18:57:00 DEBUG ] 18:57:00 DEBUG } 18:57:00 DEBUG ] 18:57:00 DEBUG } 18:57:00 DEBUG } 18:57:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461820" 18:57:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734461820" 18:57:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461820" 18:57:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461820" 18:57:00 INFO [http_cmd]: http status code OK 18:57:00 DEBUG --- status code --- 18:57:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:57:00 DEBUG --- http response --- 18:57:00 INFO [http_cmd]: http status code OK 18:57:00 DEBUG --- status code --- 18:57:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:57:00 DEBUG --- http response --- 18:57:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:57:00 DEBUG ----- output ----- 18:57:00 DEBUG { 18:57:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:57:00 INFO [http_cmd]: http status code OK 18:57:00 DEBUG "status": "success", 18:57:00 DEBUG ----- output ----- 18:57:00 DEBUG --- status code --- 18:57:00 DEBUG "data": { 18:57:00 DEBUG { 18:57:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:57:00 DEBUG "resultType": "vector", 18:57:00 DEBUG "status": "success", 18:57:00 DEBUG --- http response --- 18:57:00 DEBUG "result": [ 18:57:00 DEBUG "data": { 18:57:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:57:00 DEBUG { 18:57:00 DEBUG "resultType": "vector", 18:57:00 DEBUG ----- output ----- 18:57:00 DEBUG "metric": { 18:57:00 DEBUG "result": [ 18:57:00 DEBUG { 18:57:00 INFO [http_cmd]: http status code OK 18:57:00 DEBUG "pod": "ds-cts-0" 18:57:00 DEBUG { 18:57:00 DEBUG "status": "success", 18:57:00 DEBUG --- status code --- 18:57:00 DEBUG }, 18:57:00 DEBUG "metric": { 18:57:00 DEBUG "data": { 18:57:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:57:00 DEBUG "value": [ 18:57:00 DEBUG "instance": "10.142.0.11:9100" 18:57:00 DEBUG "resultType": "vector", 18:57:00 DEBUG --- http response --- 18:57:00 DEBUG 18:57:00 DEBUG }, 18:57:00 DEBUG "result": [ 18:57:00 DEBUG 18:57:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:57:00 DEBUG "value": [ 18:57:00 DEBUG { 18:57:00 DEBUG 18:57:00 DEBUG ----- output ----- 18:57:00 DEBUG 18:57:00 DEBUG "metric": { 18:57:00 DEBUG [...] 18:57:00 DEBUG { 18:57:00 DEBUG 18:57:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:57:00 DEBUG 18:57:00 DEBUG "status": "success", 18:57:00 DEBUG 18:57:00 DEBUG }, 18:57:00 DEBUG 18:57:00 DEBUG "data": { 18:57:00 DEBUG [...] 18:57:00 DEBUG "value": [ 18:57:00 DEBUG 18:57:00 DEBUG "resultType": "vector", 18:57:00 DEBUG 18:57:00 DEBUG 18:57:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:57:00 DEBUG "result": [ 18:57:00 DEBUG 18:57:00 DEBUG 18:57:00 DEBUG }, 18:57:00 DEBUG { 18:57:00 DEBUG 18:57:00 DEBUG 18:57:00 DEBUG "value": [ 18:57:00 DEBUG "metric": { 18:57:00 DEBUG "instance": "10.142.0.51:9100" 18:57:00 DEBUG [...] 18:57:00 DEBUG 1734461820, 18:57:00 DEBUG "695529472" 18:57:00 DEBUG "instance": "10.142.0.11:9100" 18:57:00 DEBUG 18:57:00 DEBUG }, 18:57:00 DEBUG ] 18:57:00 DEBUG }, 18:57:00 DEBUG 18:57:00 DEBUG "value": [ 18:57:00 DEBUG } 18:57:00 DEBUG "value": [ 18:57:00 DEBUG 18:57:00 DEBUG 1734461820, 18:57:00 DEBUG ] 18:57:00 DEBUG 18:57:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:57:00 DEBUG "19797.333333333336" 18:57:00 DEBUG } 18:57:00 DEBUG 18:57:00 DEBUG }, 18:57:00 DEBUG ] 18:57:00 DEBUG } 18:57:00 DEBUG 18:57:00 DEBUG "value": [ 18:57:00 DEBUG } 18:57:00 DEBUG [...] 18:57:00 DEBUG 1734461820, 18:57:00 DEBUG ] 18:57:00 DEBUG 18:57:00 DEBUG "1.7590685641322426" 18:57:00 DEBUG } 18:57:00 DEBUG 18:57:00 DEBUG ] 18:57:00 DEBUG } 18:57:00 DEBUG 18:57:00 DEBUG } 18:57:00 DEBUG "instance": "10.142.0.51:9100" 18:57:00 DEBUG ] 18:57:00 DEBUG }, 18:57:00 DEBUG } 18:57:00 DEBUG "value": [ 18:57:00 DEBUG } 18:57:00 DEBUG 1734461820, 18:57:00 DEBUG "77109.73333333335" 18:57:00 DEBUG ] 18:57:00 DEBUG } 18:57:00 DEBUG ] 18:57:00 DEBUG } 18:57:00 DEBUG } 18:57:02 INFO 18:57:02 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:57:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:57:02 INFO [loop_until]: OK (rc = 0) 18:57:02 DEBUG --- stdout --- 18:57:02 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 544Mi ds-idrepo-0 7m 738Mi rcs-5d696ff5b-7ttvr 3m 297Mi rcs-5d696ff5b-wwlbr 3m 331Mi 18:57:02 DEBUG --- stderr --- 18:57:02 DEBUG 18:57:02 INFO 18:57:02 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:57:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:57:02 INFO [loop_until]: OK (rc = 0) 18:57:02 DEBUG --- stdout --- 18:57:02 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 94m 0% 2277Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1854m 5% 3628Mi 12% 18:57:02 DEBUG --- stderr --- 18:57:02 DEBUG 18:57:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461879" 18:57:59 INFO [http_cmd]: http status code OK 18:57:59 DEBUG --- status code --- 18:57:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:57:59 DEBUG --- http response --- 18:57:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:57:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461879" 18:57:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734461879" 18:57:59 INFO [http_cmd]: http status code OK 18:57:59 DEBUG --- status code --- 18:57:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:57:59 DEBUG --- http response --- 18:57:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:57:59 INFO [http_cmd]: http status code OK 18:57:59 DEBUG --- status code --- 18:57:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:57:59 DEBUG --- http response --- 18:57:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:57:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461879" 18:57:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461879" 18:57:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461879" 18:57:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461879" 18:57:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461879" 18:57:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461879" 18:57:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461879" 18:57:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461879" 18:57:59 INFO [http_cmd]: http status code OK 18:57:59 DEBUG --- status code --- 18:57:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:57:59 DEBUG --- http response --- 18:57:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:57:59 INFO [http_cmd]: http status code OK 18:57:59 DEBUG --- status code --- 18:57:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:57:59 DEBUG --- http response --- 18:57:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:57:59 INFO [http_cmd]: http status code OK 18:57:59 DEBUG --- status code --- 18:57:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:57:59 DEBUG --- http response --- 18:57:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:57:59 INFO [http_cmd]: http status code OK 18:57:59 DEBUG --- status code --- 18:57:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:57:59 DEBUG --- http response --- 18:57:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:57:59 INFO [http_cmd]: http status code OK 18:57:59 DEBUG --- status code --- 18:57:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:57:59 DEBUG --- http response --- 18:57:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:57:59 INFO [http_cmd]: http status code OK 18:57:59 DEBUG --- status code --- 18:57:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:57:59 DEBUG --- http response --- 18:57:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:57:59 INFO [http_cmd]: http status code OK 18:57:59 DEBUG --- status code --- 18:57:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:57:59 DEBUG --- http response --- 18:57:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:57:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461879" 18:57:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734461879" 18:57:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461879" 18:57:59 INFO [http_cmd]: http status code OK 18:57:59 DEBUG --- status code --- 18:57:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:57:59 DEBUG --- http response --- 18:57:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:57:59 ERROR [http_cmd]: ERROR 18:57:59 ERROR --- status code --- 18:57:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:57:59 ERROR --- http response --- 18:57:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:57:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:57:59 INFO [http_cmd]: http status code OK 18:57:59 DEBUG --- status code --- 18:57:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:57:59 DEBUG --- http response --- 18:57:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:57:59 INFO [http_cmd]: http status code OK 18:57:59 DEBUG --- status code --- 18:57:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:57:59 DEBUG --- http response --- 18:57:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:57:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461879" 18:57:59 INFO [http_cmd]: http status code OK 18:57:59 DEBUG --- status code --- 18:57:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:57:59 DEBUG --- http response --- 18:57:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:57:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461879" 18:57:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461879" 18:57:59 INFO [http_cmd]: http status code OK 18:57:59 DEBUG --- status code --- 18:57:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:57:59 DEBUG --- http response --- 18:57:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:57:59 INFO [http_cmd]: http status code OK 18:57:59 DEBUG --- status code --- 18:57:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:57:59 DEBUG --- http response --- 18:57:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:57:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461879" 18:57:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461879" 18:57:59 INFO [http_cmd]: http status code OK 18:57:59 DEBUG --- status code --- 18:57:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:57:59 INFO [http_cmd]: http status code OK 18:57:59 DEBUG --- http response --- 18:57:59 DEBUG --- status code --- 18:57:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:57:59 DEBUG --- http response --- 18:57:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:57:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:57:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734461879" 18:57:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734461879" 18:57:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734461879" 18:57:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461879" 18:57:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461879" 18:57:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461879" 18:57:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461879" 18:57:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461879" 18:57:59 INFO [http_cmd]: http status code OK 18:57:59 DEBUG --- status code --- 18:57:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:57:59 DEBUG --- http response --- 18:57:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:57:59 INFO [http_cmd]: http status code OK 18:57:59 DEBUG --- status code --- 18:57:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:57:59 DEBUG --- http response --- 18:57:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:57:59 INFO [http_cmd]: http status code OK 18:57:59 DEBUG --- status code --- 18:57:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:57:59 DEBUG --- http response --- 18:57:59 INFO [http_cmd]: http status code OK 18:57:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:58:00 DEBUG --- status code --- 18:58:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:58:00 DEBUG --- http response --- 18:58:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:58:00 INFO [http_cmd]: http status code OK 18:58:00 DEBUG --- status code --- 18:58:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:58:00 DEBUG --- http response --- 18:58:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:58:00 INFO [http_cmd]: http status code OK 18:58:00 INFO [http_cmd]: http status code OK 18:58:00 DEBUG --- status code --- 18:58:00 INFO [http_cmd]: http status code OK 18:58:00 DEBUG --- status code --- 18:58:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:58:00 DEBUG --- status code --- 18:58:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:58:00 DEBUG --- http response --- 18:58:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:58:00 DEBUG --- http response --- 18:58:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:58:00 DEBUG --- http response --- 18:58:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:58:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:58:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461880" 18:58:00 INFO [http_cmd]: http status code OK 18:58:00 DEBUG --- status code --- 18:58:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:58:00 DEBUG --- http response --- 18:58:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:58:00 DEBUG ----- output ----- 18:58:00 DEBUG { 18:58:00 DEBUG "status": "success", 18:58:00 DEBUG "data": { 18:58:00 DEBUG "resultType": "vector", 18:58:00 DEBUG "result": [ 18:58:00 DEBUG { 18:58:00 DEBUG "metric": { 18:58:00 DEBUG "pod": "ds-cts-0" 18:58:00 DEBUG }, 18:58:00 DEBUG "value": [ 18:58:00 DEBUG 18:58:00 DEBUG 18:58:00 DEBUG 18:58:00 DEBUG [...] 18:58:00 DEBUG 18:58:00 DEBUG 18:58:00 DEBUG 18:58:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:58:00 DEBUG }, 18:58:00 DEBUG "value": [ 18:58:00 DEBUG 1734461880, 18:58:00 DEBUG "0.002625034173852631" 18:58:00 DEBUG ] 18:58:00 DEBUG } 18:58:00 DEBUG ] 18:58:00 DEBUG } 18:58:00 DEBUG } 18:58:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461880" 18:58:00 INFO [http_cmd]: http status code OK 18:58:00 DEBUG --- status code --- 18:58:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:58:00 DEBUG --- http response --- 18:58:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:58:00 DEBUG ----- output ----- 18:58:00 DEBUG { 18:58:00 DEBUG "status": "success", 18:58:00 DEBUG "data": { 18:58:00 DEBUG "resultType": "vector", 18:58:00 DEBUG "result": [ 18:58:00 DEBUG { 18:58:00 DEBUG "metric": { 18:58:00 DEBUG "instance": "10.142.0.11:9100" 18:58:00 DEBUG }, 18:58:00 DEBUG "value": [ 18:58:00 DEBUG 18:58:00 DEBUG 18:58:00 DEBUG 18:58:00 DEBUG [...] 18:58:00 DEBUG 18:58:00 DEBUG 18:58:00 DEBUG 18:58:00 DEBUG "instance": "10.142.0.51:9100" 18:58:00 DEBUG }, 18:58:00 DEBUG "value": [ 18:58:00 DEBUG 1734461880, 18:58:00 DEBUG "87636.36666666668" 18:58:00 DEBUG ] 18:58:00 DEBUG } 18:58:00 DEBUG ] 18:58:00 DEBUG } 18:58:00 DEBUG } 18:58:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461880" 18:58:00 INFO [http_cmd]: http status code OK 18:58:00 DEBUG --- status code --- 18:58:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:58:00 DEBUG --- http response --- 18:58:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:58:00 DEBUG ----- output ----- 18:58:00 DEBUG { 18:58:00 DEBUG "status": "success", 18:58:00 DEBUG "data": { 18:58:00 DEBUG "resultType": "vector", 18:58:00 DEBUG "result": [ 18:58:00 DEBUG { 18:58:00 DEBUG "metric": { 18:58:00 DEBUG "instance": "10.142.0.11:9100" 18:58:00 DEBUG }, 18:58:00 DEBUG "value": [ 18:58:00 DEBUG 18:58:00 DEBUG 18:58:00 DEBUG 18:58:00 DEBUG [...] 18:58:00 DEBUG 18:58:00 DEBUG 18:58:00 DEBUG 18:58:00 DEBUG "instance": "10.142.0.51:9100" 18:58:00 DEBUG }, 18:58:00 DEBUG "value": [ 18:58:00 DEBUG 1734461880, 18:58:00 DEBUG "0.0006666666666666527" 18:58:00 DEBUG ] 18:58:00 DEBUG } 18:58:00 DEBUG ] 18:58:00 DEBUG } 18:58:00 DEBUG } 18:58:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461880" 18:58:00 INFO [http_cmd]: http status code OK 18:58:00 DEBUG --- status code --- 18:58:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:58:00 DEBUG --- http response --- 18:58:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:58:00 DEBUG ----- output ----- 18:58:00 DEBUG { 18:58:00 DEBUG "status": "success", 18:58:00 DEBUG "data": { 18:58:00 DEBUG "resultType": "vector", 18:58:00 DEBUG "result": [ 18:58:00 DEBUG { 18:58:00 DEBUG "metric": { 18:58:00 DEBUG "instance": "10.142.0.11:9100" 18:58:00 DEBUG }, 18:58:00 DEBUG "value": [ 18:58:00 DEBUG 18:58:00 DEBUG 18:58:00 DEBUG 18:58:00 DEBUG [...] 18:58:00 DEBUG 18:58:00 DEBUG 18:58:00 DEBUG 18:58:00 DEBUG "instance": "10.142.0.51:9100" 18:58:00 DEBUG }, 18:58:00 DEBUG "value": [ 18:58:00 DEBUG 1734461880, 18:58:00 DEBUG "0.0002666666666655715" 18:58:00 DEBUG ] 18:58:00 DEBUG } 18:58:00 DEBUG ] 18:58:00 DEBUG } 18:58:00 DEBUG } 18:58:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461880" 18:58:00 INFO [http_cmd]: http status code OK 18:58:00 DEBUG --- status code --- 18:58:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:58:00 DEBUG --- http response --- 18:58:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:58:00 DEBUG ----- output ----- 18:58:00 DEBUG { 18:58:00 DEBUG "status": "success", 18:58:00 DEBUG "data": { 18:58:00 DEBUG "resultType": "vector", 18:58:00 DEBUG "result": [ 18:58:00 DEBUG { 18:58:00 DEBUG "metric": { 18:58:00 DEBUG "pod": "ds-cts-0" 18:58:00 DEBUG }, 18:58:00 DEBUG "value": [ 18:58:00 DEBUG 18:58:00 DEBUG 18:58:00 DEBUG 18:58:00 DEBUG [...] 18:58:00 DEBUG 18:58:00 DEBUG 18:58:00 DEBUG 18:58:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:58:00 DEBUG }, 18:58:00 DEBUG "value": [ 18:58:00 DEBUG 1734461880, 18:58:00 DEBUG "0" 18:58:00 DEBUG ] 18:58:00 DEBUG } 18:58:00 DEBUG ] 18:58:00 DEBUG } 18:58:00 DEBUG } 18:58:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461880" 18:58:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734461880" 18:58:00 INFO [http_cmd]: http status code OK 18:58:00 DEBUG --- status code --- 18:58:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:58:00 DEBUG --- http response --- 18:58:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:58:00 DEBUG ----- output ----- 18:58:00 DEBUG { 18:58:00 DEBUG "status": "success", 18:58:00 DEBUG "data": { 18:58:00 DEBUG "resultType": "vector", 18:58:00 DEBUG "result": [ 18:58:00 DEBUG { 18:58:00 DEBUG "metric": { 18:58:00 DEBUG "pod": "ds-cts-0" 18:58:00 DEBUG }, 18:58:00 DEBUG "value": [ 18:58:00 DEBUG 18:58:00 DEBUG 18:58:00 DEBUG 18:58:00 DEBUG [...] 18:58:00 DEBUG 18:58:00 DEBUG 18:58:00 DEBUG 18:58:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:58:00 DEBUG }, 18:58:00 DEBUG "value": [ 18:58:00 DEBUG 1734461880, 18:58:00 DEBUG "1835.6732300833874" 18:58:00 DEBUG ] 18:58:00 DEBUG } 18:58:00 DEBUG ] 18:58:00 DEBUG } 18:58:00 DEBUG } 18:58:00 INFO [http_cmd]: http status code OK 18:58:00 DEBUG --- status code --- 18:58:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:58:00 DEBUG --- http response --- 18:58:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:58:00 DEBUG ----- output ----- 18:58:00 DEBUG { 18:58:00 DEBUG "status": "success", 18:58:00 DEBUG "data": { 18:58:00 DEBUG "resultType": "vector", 18:58:00 DEBUG "result": [ 18:58:00 DEBUG { 18:58:00 DEBUG "metric": { 18:58:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:58:00 DEBUG }, 18:58:00 DEBUG "value": [ 18:58:00 DEBUG 18:58:00 DEBUG 18:58:00 DEBUG 18:58:00 DEBUG [...] 18:58:00 DEBUG 18:58:00 DEBUG 18:58:00 DEBUG 18:58:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:58:00 DEBUG }, 18:58:00 DEBUG "value": [ 18:58:00 DEBUG 1734461880, 18:58:00 DEBUG "2573918208" 18:58:00 DEBUG ] 18:58:00 DEBUG } 18:58:00 DEBUG ] 18:58:00 DEBUG } 18:58:00 DEBUG } 18:58:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461880" 18:58:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461880" 18:58:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461880" 18:58:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734461880" 18:58:00 INFO [http_cmd]: http status code OK 18:58:00 DEBUG --- status code --- 18:58:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:58:00 DEBUG --- http response --- 18:58:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:58:00 DEBUG ----- output ----- 18:58:00 DEBUG { 18:58:00 DEBUG "status": "success", 18:58:00 DEBUG "data": { 18:58:00 DEBUG "resultType": "vector", 18:58:00 DEBUG "result": [ 18:58:00 DEBUG { 18:58:00 DEBUG "metric": { 18:58:00 DEBUG "instance": "10.142.0.11:9100" 18:58:00 DEBUG }, 18:58:00 DEBUG "value": [ 18:58:00 DEBUG 18:58:00 DEBUG 18:58:00 DEBUG 18:58:00 DEBUG [...] 18:58:00 DEBUG 18:58:00 DEBUG 18:58:00 DEBUG 18:58:00 DEBUG "instance": "10.142.0.51:9100" 18:58:00 DEBUG }, 18:58:00 DEBUG "value": [ 18:58:00 DEBUG 1734461880, 18:58:00 DEBUG "13107.200000000004" 18:58:00 DEBUG ] 18:58:00 INFO [http_cmd]: http status code OK 18:58:00 DEBUG } 18:58:00 INFO [http_cmd]: http status code OK 18:58:00 DEBUG --- status code --- 18:58:00 DEBUG ] 18:58:00 DEBUG --- status code --- 18:58:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:58:00 DEBUG } 18:58:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:58:00 DEBUG --- http response --- 18:58:00 DEBUG } 18:58:00 DEBUG --- http response --- 18:58:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:58:00 INFO [http_cmd]: http status code OK 18:58:00 DEBUG ----- output ----- 18:58:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:58:00 DEBUG --- status code --- 18:58:00 DEBUG { 18:58:00 DEBUG ----- output ----- 18:58:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:58:00 DEBUG "status": "success", 18:58:00 DEBUG { 18:58:00 DEBUG --- http response --- 18:58:00 DEBUG "data": { 18:58:00 DEBUG "status": "success", 18:58:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:58:00 DEBUG "resultType": "vector", 18:58:00 DEBUG "data": { 18:58:00 DEBUG ----- output ----- 18:58:00 DEBUG "result": [ 18:58:00 DEBUG "resultType": "vector", 18:58:00 DEBUG { 18:58:00 DEBUG "status": "success", 18:58:00 DEBUG "result": [ 18:58:00 DEBUG { 18:58:00 DEBUG "data": { 18:58:00 DEBUG { 18:58:00 DEBUG "metric": { 18:58:00 DEBUG "resultType": "vector", 18:58:00 DEBUG "metric": { 18:58:00 DEBUG "pod": "ds-cts-0" 18:58:00 DEBUG "result": [ 18:58:00 DEBUG "instance": "10.142.0.11:9100" 18:58:00 DEBUG }, 18:58:00 DEBUG { 18:58:00 DEBUG }, 18:58:00 DEBUG "value": [ 18:58:00 DEBUG "metric": { 18:58:00 DEBUG "value": [ 18:58:00 DEBUG 18:58:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:58:00 DEBUG 18:58:00 DEBUG 18:58:00 DEBUG }, 18:58:00 DEBUG 18:58:00 DEBUG 18:58:00 DEBUG "value": [ 18:58:00 DEBUG 18:58:00 DEBUG [...] 18:58:00 DEBUG 18:58:00 DEBUG [...] 18:58:00 DEBUG 18:58:00 DEBUG 18:58:00 DEBUG 18:58:00 DEBUG 18:58:00 DEBUG 18:58:00 DEBUG 18:58:00 DEBUG 18:58:00 DEBUG [...] 18:58:00 DEBUG 18:58:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:58:00 DEBUG 18:58:00 DEBUG "instance": "10.142.0.51:9100" 18:58:00 DEBUG }, 18:58:00 DEBUG 18:58:00 DEBUG }, 18:58:00 DEBUG "value": [ 18:58:00 DEBUG 18:58:00 DEBUG "value": [ 18:58:00 DEBUG 1734461880, 18:58:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:58:00 DEBUG 1734461880, 18:58:00 DEBUG "72080.4666666667" 18:58:00 DEBUG }, 18:58:00 DEBUG "695558144" 18:58:00 DEBUG ] 18:58:00 DEBUG "value": [ 18:58:00 DEBUG ] 18:58:00 DEBUG } 18:58:00 DEBUG 1734461880, 18:58:00 DEBUG } 18:58:00 DEBUG ] 18:58:00 DEBUG "1.7692136011282378" 18:58:00 DEBUG ] 18:58:00 DEBUG } 18:58:00 DEBUG ] 18:58:00 DEBUG } 18:58:00 DEBUG } 18:58:00 DEBUG } 18:58:00 DEBUG ] 18:58:00 DEBUG } 18:58:00 DEBUG } 18:58:00 DEBUG } 18:58:02 INFO 18:58:02 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:58:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:58:02 INFO [loop_until]: OK (rc = 0) 18:58:02 DEBUG --- stdout --- 18:58:02 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 544Mi ds-idrepo-0 7m 738Mi rcs-5d696ff5b-7ttvr 3m 296Mi rcs-5d696ff5b-wwlbr 3m 331Mi 18:58:02 DEBUG --- stderr --- 18:58:02 DEBUG 18:58:02 INFO 18:58:02 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:58:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:58:03 INFO [loop_until]: OK (rc = 0) 18:58:03 DEBUG --- stdout --- 18:58:03 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 95m 0% 2276Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1834m 5% 3629Mi 12% 18:58:03 DEBUG --- stderr --- 18:58:03 DEBUG 18:58:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461939" 18:58:59 INFO [http_cmd]: http status code OK 18:58:59 DEBUG --- status code --- 18:58:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:58:59 DEBUG --- http response --- 18:58:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:58:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734461939" 18:58:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461939" 18:58:59 INFO [http_cmd]: http status code OK 18:58:59 DEBUG --- status code --- 18:58:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:58:59 DEBUG --- http response --- 18:58:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:58:59 INFO [http_cmd]: http status code OK 18:58:59 DEBUG --- status code --- 18:58:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:58:59 DEBUG --- http response --- 18:58:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:58:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461939" 18:58:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461939" 18:58:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461939" 18:58:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461939" 18:58:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461939" 18:58:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461939" 18:58:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461939" 18:58:59 INFO [http_cmd]: http status code OK 18:58:59 DEBUG --- status code --- 18:58:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:58:59 DEBUG --- http response --- 18:58:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:58:59 INFO [http_cmd]: http status code OK 18:58:59 DEBUG --- status code --- 18:58:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:58:59 DEBUG --- http response --- 18:58:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:58:59 INFO [http_cmd]: http status code OK 18:58:59 DEBUG --- status code --- 18:58:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:58:59 DEBUG --- http response --- 18:58:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:58:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461939" 18:58:59 INFO [http_cmd]: http status code OK 18:58:59 DEBUG --- status code --- 18:58:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:58:59 INFO [http_cmd]: http status code OK 18:58:59 DEBUG --- status code --- 18:58:59 INFO [http_cmd]: http status code OK 18:58:59 DEBUG --- http response --- 18:58:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:58:59 INFO [http_cmd]: http status code OK 18:58:59 DEBUG --- status code --- 18:58:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:58:59 DEBUG --- http response --- 18:58:59 DEBUG --- status code --- 18:58:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:58:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:58:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:58:59 DEBUG --- http response --- 18:58:59 DEBUG --- http response --- 18:58:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:58:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734461939" 18:58:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:58:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461939" 18:58:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461939" 18:58:59 INFO [http_cmd]: http status code OK 18:58:59 DEBUG --- status code --- 18:58:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:58:59 DEBUG --- http response --- 18:58:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:58:59 INFO [http_cmd]: http status code OK 18:58:59 DEBUG --- status code --- 18:58:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:58:59 DEBUG --- http response --- 18:58:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:58:59 ERROR [http_cmd]: ERROR 18:58:59 ERROR --- status code --- 18:58:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:58:59 ERROR --- http response --- 18:58:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:58:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:58:59 INFO [http_cmd]: http status code OK 18:58:59 DEBUG --- status code --- 18:58:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:58:59 DEBUG --- http response --- 18:58:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:58:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461939" 18:58:59 INFO [http_cmd]: http status code OK 18:58:59 DEBUG --- status code --- 18:58:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:58:59 DEBUG --- http response --- 18:58:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:58:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461939" 18:58:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461939" 18:58:59 INFO [http_cmd]: http status code OK 18:58:59 DEBUG --- status code --- 18:58:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:58:59 DEBUG --- http response --- 18:58:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:58:59 INFO [http_cmd]: http status code OK 18:58:59 DEBUG --- status code --- 18:58:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:58:59 DEBUG --- http response --- 18:58:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:58:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461939" 18:58:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461939" 18:58:59 INFO [http_cmd]: http status code OK 18:58:59 DEBUG --- status code --- 18:58:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:58:59 INFO [http_cmd]: http status code OK 18:58:59 DEBUG --- http response --- 18:58:59 DEBUG --- status code --- 18:58:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:58:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:58:59 DEBUG --- http response --- 18:58:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461940" 18:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734461940" 18:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734461940" 18:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734461940" 18:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461940" 18:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461940" 18:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461940" 18:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461940" 18:59:00 INFO [http_cmd]: http status code OK 18:59:00 DEBUG --- status code --- 18:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:59:00 DEBUG --- http response --- 18:59:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:59:00 INFO [http_cmd]: http status code OK 18:59:00 INFO [http_cmd]: http status code OK 18:59:00 INFO [http_cmd]: http status code OK 18:59:00 DEBUG --- status code --- 18:59:00 DEBUG --- status code --- 18:59:00 DEBUG --- status code --- 18:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:59:00 DEBUG --- http response --- 18:59:00 DEBUG --- http response --- 18:59:00 DEBUG --- http response --- 18:59:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:59:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:59:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:59:00 INFO [http_cmd]: http status code OK 18:59:00 DEBUG --- status code --- 18:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:59:00 DEBUG --- http response --- 18:59:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:59:00 INFO [http_cmd]: http status code OK 18:59:00 DEBUG --- status code --- 18:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:59:00 DEBUG --- http response --- 18:59:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:59:00 INFO [http_cmd]: http status code OK 18:59:00 DEBUG --- status code --- 18:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:59:00 DEBUG --- http response --- 18:59:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:59:00 INFO [http_cmd]: http status code OK 18:59:00 DEBUG --- status code --- 18:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:59:00 DEBUG --- http response --- 18:59:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461940" 18:59:00 INFO [http_cmd]: http status code OK 18:59:00 DEBUG --- status code --- 18:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:59:00 DEBUG --- http response --- 18:59:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:59:00 DEBUG ----- output ----- 18:59:00 DEBUG { 18:59:00 DEBUG "status": "success", 18:59:00 DEBUG "data": { 18:59:00 DEBUG "resultType": "vector", 18:59:00 DEBUG "result": [ 18:59:00 DEBUG { 18:59:00 DEBUG "metric": { 18:59:00 DEBUG "pod": "ds-cts-0" 18:59:00 DEBUG }, 18:59:00 DEBUG "value": [ 18:59:00 DEBUG 18:59:00 DEBUG 18:59:00 DEBUG 18:59:00 DEBUG [...] 18:59:00 DEBUG 18:59:00 DEBUG 18:59:00 DEBUG 18:59:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:59:00 DEBUG }, 18:59:00 DEBUG "value": [ 18:59:00 DEBUG 1734461940, 18:59:00 DEBUG "0.006363623383418842" 18:59:00 DEBUG ] 18:59:00 DEBUG } 18:59:00 DEBUG ] 18:59:00 DEBUG } 18:59:00 DEBUG } 18:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461940" 18:59:00 INFO [http_cmd]: http status code OK 18:59:00 DEBUG --- status code --- 18:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:59:00 DEBUG --- http response --- 18:59:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:59:00 DEBUG ----- output ----- 18:59:00 DEBUG { 18:59:00 DEBUG "status": "success", 18:59:00 DEBUG "data": { 18:59:00 DEBUG "resultType": "vector", 18:59:00 DEBUG "result": [ 18:59:00 DEBUG { 18:59:00 DEBUG "metric": { 18:59:00 DEBUG "instance": "10.142.0.11:9100" 18:59:00 DEBUG }, 18:59:00 DEBUG "value": [ 18:59:00 DEBUG 18:59:00 DEBUG 18:59:00 DEBUG 18:59:00 DEBUG [...] 18:59:00 DEBUG 18:59:00 DEBUG 18:59:00 DEBUG 18:59:00 DEBUG "instance": "10.142.0.51:9100" 18:59:00 DEBUG }, 18:59:00 DEBUG "value": [ 18:59:00 DEBUG 1734461940, 18:59:00 DEBUG "90614.0666666667" 18:59:00 DEBUG ] 18:59:00 DEBUG } 18:59:00 DEBUG ] 18:59:00 DEBUG } 18:59:00 DEBUG } 18:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461940" 18:59:00 INFO [http_cmd]: http status code OK 18:59:00 DEBUG --- status code --- 18:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:59:00 DEBUG --- http response --- 18:59:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:59:00 DEBUG ----- output ----- 18:59:00 DEBUG { 18:59:00 DEBUG "status": "success", 18:59:00 DEBUG "data": { 18:59:00 DEBUG "resultType": "vector", 18:59:00 DEBUG "result": [ 18:59:00 DEBUG { 18:59:00 DEBUG "metric": { 18:59:00 DEBUG "instance": "10.142.0.11:9100" 18:59:00 DEBUG }, 18:59:00 DEBUG "value": [ 18:59:00 DEBUG 18:59:00 DEBUG 18:59:00 DEBUG 18:59:00 DEBUG [...] 18:59:00 DEBUG 18:59:00 DEBUG 18:59:00 DEBUG 18:59:00 DEBUG "instance": "10.142.0.51:9100" 18:59:00 DEBUG }, 18:59:00 DEBUG "value": [ 18:59:00 DEBUG 1734461940, 18:59:00 DEBUG "0" 18:59:00 DEBUG ] 18:59:00 DEBUG } 18:59:00 DEBUG ] 18:59:00 DEBUG } 18:59:00 DEBUG } 18:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461940" 18:59:00 INFO [http_cmd]: http status code OK 18:59:00 DEBUG --- status code --- 18:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:59:00 DEBUG --- http response --- 18:59:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:59:00 DEBUG ----- output ----- 18:59:00 DEBUG { 18:59:00 DEBUG "status": "success", 18:59:00 DEBUG "data": { 18:59:00 DEBUG "resultType": "vector", 18:59:00 DEBUG "result": [ 18:59:00 DEBUG { 18:59:00 DEBUG "metric": { 18:59:00 DEBUG "instance": "10.142.0.11:9100" 18:59:00 DEBUG }, 18:59:00 DEBUG "value": [ 18:59:00 DEBUG 18:59:00 DEBUG 18:59:00 DEBUG 18:59:00 DEBUG [...] 18:59:00 DEBUG 18:59:00 DEBUG 18:59:00 DEBUG 18:59:00 DEBUG "instance": "10.142.0.51:9100" 18:59:00 DEBUG }, 18:59:00 DEBUG "value": [ 18:59:00 DEBUG 1734461940, 18:59:00 DEBUG "0.00039999999999954156" 18:59:00 DEBUG ] 18:59:00 DEBUG } 18:59:00 DEBUG ] 18:59:00 DEBUG } 18:59:00 DEBUG } 18:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461940" 18:59:00 INFO [http_cmd]: http status code OK 18:59:00 DEBUG --- status code --- 18:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:59:00 DEBUG --- http response --- 18:59:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:59:00 DEBUG ----- output ----- 18:59:00 DEBUG { 18:59:00 DEBUG "status": "success", 18:59:00 DEBUG "data": { 18:59:00 DEBUG "resultType": "vector", 18:59:00 DEBUG "result": [ 18:59:00 DEBUG { 18:59:00 DEBUG "metric": { 18:59:00 DEBUG "pod": "ds-cts-0" 18:59:00 DEBUG }, 18:59:00 DEBUG "value": [ 18:59:00 DEBUG 18:59:00 DEBUG 18:59:00 DEBUG 18:59:00 DEBUG [...] 18:59:00 DEBUG 18:59:00 DEBUG 18:59:00 DEBUG 18:59:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:59:00 DEBUG }, 18:59:00 DEBUG "value": [ 18:59:00 DEBUG 1734461940, 18:59:00 DEBUG "0" 18:59:00 DEBUG ] 18:59:00 DEBUG } 18:59:00 DEBUG ] 18:59:00 DEBUG } 18:59:00 DEBUG } 18:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461940" 18:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734461940" 18:59:00 INFO [http_cmd]: http status code OK 18:59:00 DEBUG --- status code --- 18:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:59:00 DEBUG --- http response --- 18:59:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:59:00 DEBUG ----- output ----- 18:59:00 DEBUG { 18:59:00 DEBUG "status": "success", 18:59:00 DEBUG "data": { 18:59:00 DEBUG "resultType": "vector", 18:59:00 DEBUG "result": [ 18:59:00 DEBUG { 18:59:00 DEBUG "metric": { 18:59:00 DEBUG "pod": "ds-cts-0" 18:59:00 DEBUG }, 18:59:00 DEBUG "value": [ 18:59:00 DEBUG 18:59:00 DEBUG 18:59:00 DEBUG 18:59:00 DEBUG [...] 18:59:00 DEBUG 18:59:00 DEBUG 18:59:00 DEBUG 18:59:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:59:00 DEBUG }, 18:59:00 DEBUG "value": [ 18:59:00 DEBUG 1734461940, 18:59:00 DEBUG "1212.8566056804557" 18:59:00 DEBUG ] 18:59:00 DEBUG } 18:59:00 DEBUG ] 18:59:00 DEBUG } 18:59:00 DEBUG } 18:59:00 INFO [http_cmd]: http status code OK 18:59:00 DEBUG --- status code --- 18:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:59:00 DEBUG --- http response --- 18:59:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:59:00 DEBUG ----- output ----- 18:59:00 DEBUG { 18:59:00 DEBUG "status": "success", 18:59:00 DEBUG "data": { 18:59:00 DEBUG "resultType": "vector", 18:59:00 DEBUG "result": [ 18:59:00 DEBUG { 18:59:00 DEBUG "metric": { 18:59:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:59:00 DEBUG }, 18:59:00 DEBUG "value": [ 18:59:00 DEBUG 18:59:00 DEBUG 18:59:00 DEBUG 18:59:00 DEBUG [...] 18:59:00 DEBUG 18:59:00 DEBUG 18:59:00 DEBUG 18:59:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:59:00 DEBUG }, 18:59:00 DEBUG "value": [ 18:59:00 DEBUG 1734461940, 18:59:00 DEBUG "2575572992" 18:59:00 DEBUG ] 18:59:00 DEBUG } 18:59:00 DEBUG ] 18:59:00 DEBUG } 18:59:00 DEBUG } 18:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461940" 18:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734461940" 18:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461940" 18:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734461940" 18:59:00 INFO [http_cmd]: http status code OK 18:59:00 DEBUG --- status code --- 18:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:59:00 DEBUG --- http response --- 18:59:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:59:00 DEBUG ----- output ----- 18:59:00 DEBUG { 18:59:00 DEBUG "status": "success", 18:59:00 DEBUG "data": { 18:59:00 DEBUG "resultType": "vector", 18:59:00 DEBUG "result": [ 18:59:00 DEBUG { 18:59:00 DEBUG "metric": { 18:59:00 DEBUG "instance": "10.142.0.11:9100" 18:59:00 DEBUG }, 18:59:00 DEBUG "value": [ 18:59:00 DEBUG 18:59:00 DEBUG 18:59:00 DEBUG 18:59:00 DEBUG [...] 18:59:00 DEBUG 18:59:00 DEBUG 18:59:00 DEBUG 18:59:00 DEBUG "instance": "10.142.0.51:9100" 18:59:00 DEBUG }, 18:59:00 DEBUG "value": [ 18:59:00 DEBUG 1734461940, 18:59:00 DEBUG "24985.600000000006" 18:59:00 DEBUG ] 18:59:00 DEBUG } 18:59:00 DEBUG ] 18:59:00 DEBUG } 18:59:00 DEBUG } 18:59:00 INFO [http_cmd]: http status code OK 18:59:00 DEBUG --- status code --- 18:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:59:00 DEBUG --- http response --- 18:59:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:59:00 DEBUG ----- output ----- 18:59:00 DEBUG { 18:59:00 DEBUG "status": "success", 18:59:00 DEBUG "data": { 18:59:00 DEBUG "resultType": "vector", 18:59:00 DEBUG "result": [ 18:59:00 DEBUG { 18:59:00 DEBUG "metric": { 18:59:00 DEBUG "instance": "10.142.0.11:9100" 18:59:00 DEBUG }, 18:59:00 DEBUG "value": [ 18:59:00 DEBUG 18:59:00 DEBUG 18:59:00 DEBUG 18:59:00 DEBUG [...] 18:59:00 DEBUG 18:59:00 DEBUG 18:59:00 DEBUG 18:59:00 DEBUG "instance": "10.142.0.51:9100" 18:59:00 DEBUG }, 18:59:00 DEBUG "value": [ 18:59:00 DEBUG 1734461940, 18:59:00 DEBUG "76680.60000000002" 18:59:00 DEBUG ] 18:59:00 DEBUG } 18:59:00 DEBUG ] 18:59:00 DEBUG } 18:59:00 DEBUG } 18:59:00 INFO [http_cmd]: http status code OK 18:59:00 DEBUG --- status code --- 18:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:59:00 DEBUG --- http response --- 18:59:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 18:59:00 DEBUG ----- output ----- 18:59:00 DEBUG { 18:59:00 DEBUG "status": "success", 18:59:00 DEBUG "data": { 18:59:00 DEBUG "resultType": "vector", 18:59:00 INFO [http_cmd]: http status code OK 18:59:00 DEBUG "result": [ 18:59:00 DEBUG --- status code --- 18:59:00 DEBUG { 18:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:59:00 DEBUG "metric": { 18:59:00 DEBUG --- http response --- 18:59:00 DEBUG "pod": "ds-cts-0" 18:59:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 18:59:00 DEBUG }, 18:59:00 DEBUG ----- output ----- 18:59:00 DEBUG "value": [ 18:59:00 DEBUG { 18:59:00 DEBUG 18:59:00 DEBUG "status": "success", 18:59:00 DEBUG 18:59:00 DEBUG "data": { 18:59:00 DEBUG 18:59:00 DEBUG "resultType": "vector", 18:59:00 DEBUG [...] 18:59:00 DEBUG "result": [ 18:59:00 DEBUG 18:59:00 DEBUG { 18:59:00 DEBUG 18:59:00 DEBUG "metric": { 18:59:00 DEBUG 18:59:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 18:59:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 18:59:00 DEBUG }, 18:59:00 DEBUG }, 18:59:00 DEBUG "value": [ 18:59:00 DEBUG "value": [ 18:59:00 DEBUG 18:59:00 DEBUG 1734461940, 18:59:00 DEBUG 18:59:00 DEBUG "695623680" 18:59:00 DEBUG 18:59:00 DEBUG ] 18:59:00 DEBUG [...] 18:59:00 DEBUG } 18:59:00 DEBUG 18:59:00 DEBUG ] 18:59:00 DEBUG 18:59:00 DEBUG } 18:59:00 DEBUG 18:59:00 DEBUG } 18:59:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 18:59:00 DEBUG }, 18:59:00 DEBUG "value": [ 18:59:00 DEBUG 1734461940, 18:59:00 DEBUG "1.7631989149726972" 18:59:00 DEBUG ] 18:59:00 DEBUG } 18:59:00 DEBUG ] 18:59:00 DEBUG } 18:59:00 DEBUG } 18:59:02 INFO 18:59:02 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 18:59:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:59:02 INFO [loop_until]: OK (rc = 0) 18:59:02 DEBUG --- stdout --- 18:59:02 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 543Mi ds-idrepo-0 7m 739Mi rcs-5d696ff5b-7ttvr 3m 296Mi rcs-5d696ff5b-wwlbr 3m 331Mi 18:59:02 DEBUG --- stderr --- 18:59:02 DEBUG 18:59:03 INFO 18:59:03 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 18:59:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 18:59:03 INFO [loop_until]: OK (rc = 0) 18:59:03 DEBUG --- stdout --- 18:59:03 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 97m 0% 2268Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1842m 5% 3630Mi 12% 18:59:03 DEBUG --- stderr --- 18:59:03 DEBUG 18:59:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461999" 18:59:59 INFO [http_cmd]: http status code OK 18:59:59 DEBUG --- status code --- 18:59:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:59:59 DEBUG --- http response --- 18:59:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:59:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734461999" 18:59:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461999" 18:59:59 INFO [http_cmd]: http status code OK 18:59:59 DEBUG --- status code --- 18:59:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:59:59 INFO [http_cmd]: http status code OK 18:59:59 DEBUG --- http response --- 18:59:59 DEBUG --- status code --- 18:59:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:59:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:59:59 DEBUG --- http response --- 18:59:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:59:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461999" 18:59:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461999" 18:59:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461999" 18:59:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461999" 18:59:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461999" 18:59:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461999" 18:59:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461999" 18:59:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461999" 18:59:59 INFO [http_cmd]: http status code OK 18:59:59 DEBUG --- status code --- 18:59:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:59:59 DEBUG --- http response --- 18:59:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:59:59 INFO [http_cmd]: http status code OK 18:59:59 DEBUG --- status code --- 18:59:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:59:59 DEBUG --- http response --- 18:59:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:59:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461999" 18:59:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734461999" 18:59:59 INFO [http_cmd]: http status code OK 18:59:59 DEBUG --- status code --- 18:59:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:59:59 DEBUG --- http response --- 18:59:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:59:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461999" 18:59:59 INFO [http_cmd]: http status code OK 18:59:59 DEBUG --- status code --- 18:59:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:59:59 DEBUG --- http response --- 18:59:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:59:59 INFO [http_cmd]: http status code OK 18:59:59 DEBUG --- status code --- 18:59:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:59:59 DEBUG --- http response --- 18:59:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:59:59 ERROR [http_cmd]: ERROR 18:59:59 ERROR --- status code --- 18:59:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 18:59:59 ERROR --- http response --- 18:59:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 18:59:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 18:59:59 INFO [http_cmd]: http status code OK 18:59:59 DEBUG --- status code --- 18:59:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:59:59 DEBUG --- http response --- 18:59:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:59:59 INFO [http_cmd]: http status code OK 18:59:59 INFO [http_cmd]: http status code OK 18:59:59 INFO [http_cmd]: http status code OK 18:59:59 INFO [http_cmd]: http status code OK 18:59:59 DEBUG --- status code --- 18:59:59 DEBUG --- status code --- 18:59:59 DEBUG --- status code --- 18:59:59 DEBUG --- status code --- 18:59:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:59:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:59:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:59:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:59:59 DEBUG --- http response --- 18:59:59 DEBUG --- http response --- 18:59:59 DEBUG --- http response --- 18:59:59 DEBUG --- http response --- 18:59:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:59:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:59:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:59:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:59:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461999" 18:59:59 INFO [http_cmd]: http status code OK 18:59:59 DEBUG --- status code --- 18:59:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:59:59 DEBUG --- http response --- 18:59:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:59:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461999" 18:59:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461999" 18:59:59 INFO [http_cmd]: http status code OK 18:59:59 INFO [http_cmd]: http status code OK 18:59:59 DEBUG --- status code --- 18:59:59 DEBUG --- status code --- 18:59:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:59:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:59:59 DEBUG --- http response --- 18:59:59 DEBUG --- http response --- 18:59:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:59:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:59:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734461999" 18:59:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734461999" 18:59:59 INFO [http_cmd]: http status code OK 18:59:59 DEBUG --- status code --- 18:59:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:59:59 DEBUG --- http response --- 18:59:59 INFO [http_cmd]: http status code OK 18:59:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 18:59:59 DEBUG --- status code --- 18:59:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 18:59:59 DEBUG --- http response --- 18:59:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:00:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734462000" 19:00:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462000" 19:00:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734462000" 19:00:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462000" 19:00:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734462000" 19:00:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462000" 19:00:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462000" 19:00:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462000" 19:00:00 INFO [http_cmd]: http status code OK 19:00:00 DEBUG --- status code --- 19:00:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:00:00 DEBUG --- http response --- 19:00:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:00:00 INFO [http_cmd]: http status code OK 19:00:00 DEBUG --- status code --- 19:00:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:00:00 DEBUG --- http response --- 19:00:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:00:00 INFO [http_cmd]: http status code OK 19:00:00 DEBUG --- status code --- 19:00:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:00:00 DEBUG --- http response --- 19:00:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:00:00 INFO [http_cmd]: http status code OK 19:00:00 DEBUG --- status code --- 19:00:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:00:00 DEBUG --- http response --- 19:00:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:00:00 INFO [http_cmd]: http status code OK 19:00:00 DEBUG --- status code --- 19:00:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:00:00 DEBUG --- http response --- 19:00:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:00:00 INFO [http_cmd]: http status code OK 19:00:00 DEBUG --- status code --- 19:00:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:00:00 DEBUG --- http response --- 19:00:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:00:00 INFO [http_cmd]: http status code OK 19:00:00 DEBUG --- status code --- 19:00:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:00:00 DEBUG --- http response --- 19:00:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:00:00 INFO [http_cmd]: http status code OK 19:00:00 DEBUG --- status code --- 19:00:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:00:00 DEBUG --- http response --- 19:00:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:00:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462000" 19:00:00 INFO [http_cmd]: http status code OK 19:00:00 DEBUG --- status code --- 19:00:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:00:00 DEBUG --- http response --- 19:00:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:00:00 DEBUG ----- output ----- 19:00:00 DEBUG { 19:00:00 DEBUG "status": "success", 19:00:00 DEBUG "data": { 19:00:00 DEBUG "resultType": "vector", 19:00:00 DEBUG "result": [ 19:00:00 DEBUG { 19:00:00 DEBUG "metric": { 19:00:00 DEBUG "pod": "ds-cts-0" 19:00:00 DEBUG }, 19:00:00 DEBUG "value": [ 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG [...] 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:00:00 DEBUG }, 19:00:00 DEBUG "value": [ 19:00:00 DEBUG 1734462000, 19:00:00 DEBUG "0.0020490311895680095" 19:00:00 DEBUG ] 19:00:00 DEBUG } 19:00:00 DEBUG ] 19:00:00 DEBUG } 19:00:00 DEBUG } 19:00:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462000" 19:00:00 INFO [http_cmd]: http status code OK 19:00:00 DEBUG --- status code --- 19:00:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:00:00 DEBUG --- http response --- 19:00:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:00:00 DEBUG ----- output ----- 19:00:00 DEBUG { 19:00:00 DEBUG "status": "success", 19:00:00 DEBUG "data": { 19:00:00 DEBUG "resultType": "vector", 19:00:00 DEBUG "result": [ 19:00:00 DEBUG { 19:00:00 DEBUG "metric": { 19:00:00 DEBUG "instance": "10.142.0.11:9100" 19:00:00 DEBUG }, 19:00:00 DEBUG "value": [ 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG [...] 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG "instance": "10.142.0.51:9100" 19:00:00 DEBUG }, 19:00:00 DEBUG "value": [ 19:00:00 DEBUG 1734462000, 19:00:00 DEBUG "87732.23333333334" 19:00:00 DEBUG ] 19:00:00 DEBUG } 19:00:00 DEBUG ] 19:00:00 DEBUG } 19:00:00 DEBUG } 19:00:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462000" 19:00:00 INFO [http_cmd]: http status code OK 19:00:00 DEBUG --- status code --- 19:00:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:00:00 DEBUG --- http response --- 19:00:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:00:00 DEBUG ----- output ----- 19:00:00 DEBUG { 19:00:00 DEBUG "status": "success", 19:00:00 DEBUG "data": { 19:00:00 DEBUG "resultType": "vector", 19:00:00 DEBUG "result": [ 19:00:00 DEBUG { 19:00:00 DEBUG "metric": { 19:00:00 DEBUG "instance": "10.142.0.11:9100" 19:00:00 DEBUG }, 19:00:00 DEBUG "value": [ 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG [...] 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG "instance": "10.142.0.51:9100" 19:00:00 DEBUG }, 19:00:00 DEBUG "value": [ 19:00:00 DEBUG 1734462000, 19:00:00 DEBUG "0.0006666666666666527" 19:00:00 DEBUG ] 19:00:00 DEBUG } 19:00:00 DEBUG ] 19:00:00 DEBUG } 19:00:00 DEBUG } 19:00:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462000" 19:00:00 INFO [http_cmd]: http status code OK 19:00:00 DEBUG --- status code --- 19:00:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:00:00 DEBUG --- http response --- 19:00:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:00:00 DEBUG ----- output ----- 19:00:00 DEBUG { 19:00:00 DEBUG "status": "success", 19:00:00 DEBUG "data": { 19:00:00 DEBUG "resultType": "vector", 19:00:00 DEBUG "result": [ 19:00:00 DEBUG { 19:00:00 DEBUG "metric": { 19:00:00 DEBUG "instance": "10.142.0.11:9100" 19:00:00 DEBUG }, 19:00:00 DEBUG "value": [ 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG [...] 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG "instance": "10.142.0.51:9100" 19:00:00 DEBUG }, 19:00:00 DEBUG "value": [ 19:00:00 DEBUG 1734462000, 19:00:00 DEBUG "0.00039999999999954156" 19:00:00 DEBUG ] 19:00:00 DEBUG } 19:00:00 DEBUG ] 19:00:00 DEBUG } 19:00:00 DEBUG } 19:00:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462000" 19:00:00 INFO [http_cmd]: http status code OK 19:00:00 DEBUG --- status code --- 19:00:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:00:00 DEBUG --- http response --- 19:00:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:00:00 DEBUG ----- output ----- 19:00:00 DEBUG { 19:00:00 DEBUG "status": "success", 19:00:00 DEBUG "data": { 19:00:00 DEBUG "resultType": "vector", 19:00:00 DEBUG "result": [ 19:00:00 DEBUG { 19:00:00 DEBUG "metric": { 19:00:00 DEBUG "pod": "ds-cts-0" 19:00:00 DEBUG }, 19:00:00 DEBUG "value": [ 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG [...] 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:00:00 DEBUG }, 19:00:00 DEBUG "value": [ 19:00:00 DEBUG 1734462000, 19:00:00 DEBUG "0" 19:00:00 DEBUG ] 19:00:00 DEBUG } 19:00:00 DEBUG ] 19:00:00 DEBUG } 19:00:00 DEBUG } 19:00:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462000" 19:00:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734462000" 19:00:00 INFO [http_cmd]: http status code OK 19:00:00 DEBUG --- status code --- 19:00:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:00:00 DEBUG --- http response --- 19:00:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:00:00 DEBUG ----- output ----- 19:00:00 DEBUG { 19:00:00 DEBUG "status": "success", 19:00:00 DEBUG "data": { 19:00:00 DEBUG "resultType": "vector", 19:00:00 DEBUG "result": [ 19:00:00 DEBUG { 19:00:00 DEBUG "metric": { 19:00:00 DEBUG "pod": "ds-cts-0" 19:00:00 DEBUG }, 19:00:00 DEBUG "value": [ 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG [...] 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:00:00 DEBUG }, 19:00:00 DEBUG "value": [ 19:00:00 DEBUG 1734462000, 19:00:00 DEBUG "1764.8465425610314" 19:00:00 DEBUG ] 19:00:00 DEBUG } 19:00:00 DEBUG ] 19:00:00 DEBUG } 19:00:00 DEBUG } 19:00:00 INFO [http_cmd]: http status code OK 19:00:00 DEBUG --- status code --- 19:00:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:00:00 DEBUG --- http response --- 19:00:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:00:00 DEBUG ----- output ----- 19:00:00 DEBUG { 19:00:00 DEBUG "status": "success", 19:00:00 DEBUG "data": { 19:00:00 DEBUG "resultType": "vector", 19:00:00 DEBUG "result": [ 19:00:00 DEBUG { 19:00:00 DEBUG "metric": { 19:00:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 19:00:00 DEBUG }, 19:00:00 DEBUG "value": [ 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG [...] 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 19:00:00 DEBUG }, 19:00:00 DEBUG "value": [ 19:00:00 DEBUG 1734462000, 19:00:00 DEBUG "2579812352" 19:00:00 DEBUG ] 19:00:00 DEBUG } 19:00:00 DEBUG ] 19:00:00 DEBUG } 19:00:00 DEBUG } 19:00:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462000" 19:00:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462000" 19:00:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462000" 19:00:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734462000" 19:00:00 INFO [http_cmd]: http status code OK 19:00:00 DEBUG --- status code --- 19:00:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:00:00 DEBUG --- http response --- 19:00:00 INFO [http_cmd]: http status code OK 19:00:00 DEBUG --- status code --- 19:00:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:00:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:00:00 DEBUG --- http response --- 19:00:00 DEBUG ----- output ----- 19:00:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:00:00 DEBUG { 19:00:00 DEBUG ----- output ----- 19:00:00 DEBUG "status": "success", 19:00:00 DEBUG { 19:00:00 DEBUG "data": { 19:00:00 DEBUG "status": "success", 19:00:00 DEBUG "resultType": "vector", 19:00:00 DEBUG "data": { 19:00:00 DEBUG "result": [ 19:00:00 DEBUG "resultType": "vector", 19:00:00 DEBUG { 19:00:00 DEBUG "result": [ 19:00:00 DEBUG "metric": { 19:00:00 DEBUG { 19:00:00 DEBUG "instance": "10.142.0.11:9100" 19:00:00 DEBUG "metric": { 19:00:00 DEBUG }, 19:00:00 DEBUG "instance": "10.142.0.11:9100" 19:00:00 DEBUG "value": [ 19:00:00 DEBUG }, 19:00:00 DEBUG 19:00:00 DEBUG "value": [ 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG [...] 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG [...] 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG "instance": "10.142.0.51:9100" 19:00:00 DEBUG 19:00:00 DEBUG }, 19:00:00 DEBUG "instance": "10.142.0.51:9100" 19:00:00 DEBUG "value": [ 19:00:00 DEBUG }, 19:00:00 DEBUG 1734462000, 19:00:00 DEBUG "value": [ 19:00:00 DEBUG "71717.26666666669" 19:00:00 DEBUG 1734462000, 19:00:00 DEBUG ] 19:00:00 DEBUG "19660.800000000003" 19:00:00 DEBUG } 19:00:00 DEBUG ] 19:00:00 DEBUG ] 19:00:00 DEBUG } 19:00:00 DEBUG } 19:00:00 DEBUG ] 19:00:00 DEBUG } 19:00:00 DEBUG } 19:00:00 DEBUG } 19:00:00 INFO [http_cmd]: http status code OK 19:00:00 DEBUG --- status code --- 19:00:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:00:00 DEBUG --- http response --- 19:00:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:00:00 DEBUG ----- output ----- 19:00:00 DEBUG { 19:00:00 DEBUG "status": "success", 19:00:00 DEBUG "data": { 19:00:00 DEBUG "resultType": "vector", 19:00:00 DEBUG "result": [ 19:00:00 DEBUG { 19:00:00 DEBUG "metric": { 19:00:00 DEBUG "pod": "ds-cts-0" 19:00:00 DEBUG }, 19:00:00 DEBUG "value": [ 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG [...] 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:00:00 DEBUG }, 19:00:00 DEBUG "value": [ 19:00:00 DEBUG 1734462000, 19:00:00 DEBUG "695615488" 19:00:00 DEBUG ] 19:00:00 DEBUG } 19:00:00 DEBUG ] 19:00:00 DEBUG } 19:00:00 DEBUG } 19:00:00 INFO [http_cmd]: http status code OK 19:00:00 DEBUG --- status code --- 19:00:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:00:00 DEBUG --- http response --- 19:00:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:00:00 DEBUG ----- output ----- 19:00:00 DEBUG { 19:00:00 DEBUG "status": "success", 19:00:00 DEBUG "data": { 19:00:00 DEBUG "resultType": "vector", 19:00:00 DEBUG "result": [ 19:00:00 DEBUG { 19:00:00 DEBUG "metric": { 19:00:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 19:00:00 DEBUG }, 19:00:00 DEBUG "value": [ 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG [...] 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG 19:00:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 19:00:00 DEBUG }, 19:00:00 DEBUG "value": [ 19:00:00 DEBUG 1734462000, 19:00:00 DEBUG "1.776443551731878" 19:00:00 DEBUG ] 19:00:00 DEBUG } 19:00:00 DEBUG ] 19:00:00 DEBUG } 19:00:00 DEBUG } 19:00:02 INFO 19:00:02 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 19:00:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:00:02 INFO [loop_until]: OK (rc = 0) 19:00:02 DEBUG --- stdout --- 19:00:02 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 5m 544Mi ds-idrepo-0 6m 738Mi rcs-5d696ff5b-7ttvr 5m 296Mi rcs-5d696ff5b-wwlbr 3m 331Mi 19:00:02 DEBUG --- stderr --- 19:00:02 DEBUG 19:00:03 INFO 19:00:03 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 19:00:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:00:03 INFO [loop_until]: OK (rc = 0) 19:00:03 DEBUG --- stdout --- 19:00:03 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 95m 0% 2287Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1836m 5% 3633Mi 12% 19:00:03 DEBUG --- stderr --- 19:00:03 DEBUG 19:00:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462059" 19:00:59 INFO [http_cmd]: http status code OK 19:00:59 DEBUG --- status code --- 19:00:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:00:59 DEBUG --- http response --- 19:00:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:00:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734462059" 19:00:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462059" 19:00:59 INFO [http_cmd]: http status code OK 19:00:59 DEBUG --- status code --- 19:00:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:00:59 DEBUG --- http response --- 19:00:59 INFO [http_cmd]: http status code OK 19:00:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:00:59 DEBUG --- status code --- 19:00:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:00:59 DEBUG --- http response --- 19:00:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:00:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462059" 19:00:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462059" 19:00:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462059" 19:00:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462059" 19:00:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462059" 19:00:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734462059" 19:00:59 INFO [http_cmd]: http status code OK 19:00:59 DEBUG --- status code --- 19:00:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:00:59 DEBUG --- http response --- 19:00:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:00:59 INFO [http_cmd]: http status code OK 19:00:59 DEBUG --- status code --- 19:00:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:00:59 DEBUG --- http response --- 19:00:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:00:59 INFO [http_cmd]: http status code OK 19:00:59 DEBUG --- status code --- 19:00:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:00:59 DEBUG --- http response --- 19:00:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:00:59 INFO [http_cmd]: http status code OK 19:00:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462059" 19:00:59 DEBUG --- status code --- 19:00:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462059" 19:00:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462059" 19:00:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:00:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462059" 19:00:59 INFO [http_cmd]: http status code OK 19:00:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462059" 19:00:59 ERROR [http_cmd]: ERROR 19:00:59 DEBUG --- http response --- 19:00:59 DEBUG --- status code --- 19:00:59 ERROR --- status code --- 19:00:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:00:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:00:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:00:59 DEBUG --- http response --- 19:00:59 ERROR --- http response --- 19:00:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:00:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:00:59 INFO [http_cmd]: http status code OK 19:00:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:00:59 INFO [http_cmd]: http status code OK 19:00:59 DEBUG --- status code --- 19:00:59 DEBUG --- status code --- 19:00:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:00:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:00:59 DEBUG --- http response --- 19:00:59 DEBUG --- http response --- 19:00:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:00:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:00:59 INFO [http_cmd]: http status code OK 19:00:59 DEBUG --- status code --- 19:00:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:00:59 DEBUG --- http response --- 19:00:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:00:59 INFO [http_cmd]: http status code OK 19:00:59 DEBUG --- status code --- 19:00:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:00:59 DEBUG --- http response --- 19:00:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:00:59 INFO [http_cmd]: http status code OK 19:00:59 DEBUG --- status code --- 19:00:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:00:59 DEBUG --- http response --- 19:00:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:00:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462059" 19:00:59 INFO [http_cmd]: http status code OK 19:00:59 DEBUG --- status code --- 19:00:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:00:59 DEBUG --- http response --- 19:00:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:00:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462059" 19:00:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462059" 19:00:59 INFO [http_cmd]: http status code OK 19:00:59 INFO [http_cmd]: http status code OK 19:00:59 DEBUG --- status code --- 19:00:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:00:59 DEBUG --- status code --- 19:00:59 DEBUG --- http response --- 19:00:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:00:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:00:59 DEBUG --- http response --- 19:00:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:00:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462059" 19:00:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462059" 19:00:59 INFO [http_cmd]: http status code OK 19:00:59 DEBUG --- status code --- 19:00:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:00:59 INFO [http_cmd]: http status code OK 19:00:59 DEBUG --- status code --- 19:00:59 DEBUG --- http response --- 19:00:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:00:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:00:59 DEBUG --- http response --- 19:00:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:01:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734462060" 19:01:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462060" 19:01:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734462060" 19:01:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462060" 19:01:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462060" 19:01:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734462060" 19:01:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462060" 19:01:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462060" 19:01:00 INFO [http_cmd]: http status code OK 19:01:00 DEBUG --- status code --- 19:01:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:01:00 DEBUG --- http response --- 19:01:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:01:00 INFO [http_cmd]: http status code OK 19:01:00 DEBUG --- status code --- 19:01:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:01:00 INFO [http_cmd]: http status code OK 19:01:00 DEBUG --- http response --- 19:01:00 DEBUG --- status code --- 19:01:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:01:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:01:00 DEBUG --- http response --- 19:01:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:01:00 INFO [http_cmd]: http status code OK 19:01:00 INFO [http_cmd]: http status code OK 19:01:00 INFO [http_cmd]: http status code OK 19:01:00 DEBUG --- status code --- 19:01:00 DEBUG --- status code --- 19:01:00 DEBUG --- status code --- 19:01:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:01:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:01:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:01:00 DEBUG --- http response --- 19:01:00 DEBUG --- http response --- 19:01:00 DEBUG --- http response --- 19:01:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:01:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:01:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:01:00 INFO [http_cmd]: http status code OK 19:01:00 DEBUG --- status code --- 19:01:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:01:00 DEBUG --- http response --- 19:01:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:01:00 INFO [http_cmd]: http status code OK 19:01:00 DEBUG --- status code --- 19:01:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:01:00 DEBUG --- http response --- 19:01:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:01:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462060" 19:01:00 INFO [http_cmd]: http status code OK 19:01:00 DEBUG --- status code --- 19:01:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:01:00 DEBUG --- http response --- 19:01:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:01:00 DEBUG ----- output ----- 19:01:00 DEBUG { 19:01:00 DEBUG "status": "success", 19:01:00 DEBUG "data": { 19:01:00 DEBUG "resultType": "vector", 19:01:00 DEBUG "result": [ 19:01:00 DEBUG { 19:01:00 DEBUG "metric": { 19:01:00 DEBUG "pod": "ds-cts-0" 19:01:00 DEBUG }, 19:01:00 DEBUG "value": [ 19:01:00 DEBUG 19:01:00 DEBUG 19:01:00 DEBUG 19:01:00 DEBUG [...] 19:01:00 DEBUG 19:01:00 DEBUG 19:01:00 DEBUG 19:01:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:01:00 DEBUG }, 19:01:00 DEBUG "value": [ 19:01:00 DEBUG 1734462060, 19:01:00 DEBUG "0.0046608108723962365" 19:01:00 DEBUG ] 19:01:00 DEBUG } 19:01:00 DEBUG ] 19:01:00 DEBUG } 19:01:00 DEBUG } 19:01:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462060" 19:01:00 INFO [http_cmd]: http status code OK 19:01:00 DEBUG --- status code --- 19:01:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:01:00 DEBUG --- http response --- 19:01:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:01:00 DEBUG ----- output ----- 19:01:00 DEBUG { 19:01:00 DEBUG "status": "success", 19:01:00 DEBUG "data": { 19:01:00 DEBUG "resultType": "vector", 19:01:00 DEBUG "result": [ 19:01:00 DEBUG { 19:01:00 DEBUG "metric": { 19:01:00 DEBUG "instance": "10.142.0.11:9100" 19:01:00 DEBUG }, 19:01:00 DEBUG "value": [ 19:01:00 DEBUG 19:01:00 DEBUG 19:01:00 DEBUG 19:01:00 DEBUG [...] 19:01:00 DEBUG 19:01:00 DEBUG 19:01:00 DEBUG 19:01:00 DEBUG "instance": "10.142.0.51:9100" 19:01:00 DEBUG }, 19:01:00 DEBUG "value": [ 19:01:00 DEBUG 1734462060, 19:01:00 DEBUG "88416.33333333334" 19:01:00 DEBUG ] 19:01:00 DEBUG } 19:01:00 DEBUG ] 19:01:00 DEBUG } 19:01:00 DEBUG } 19:01:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462060" 19:01:00 INFO [http_cmd]: http status code OK 19:01:00 DEBUG --- status code --- 19:01:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:01:00 DEBUG --- http response --- 19:01:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:01:00 DEBUG ----- output ----- 19:01:00 DEBUG { 19:01:00 DEBUG "status": "success", 19:01:00 DEBUG "data": { 19:01:00 DEBUG "resultType": "vector", 19:01:00 DEBUG "result": [ 19:01:00 DEBUG { 19:01:00 DEBUG "metric": { 19:01:00 DEBUG "instance": "10.142.0.11:9100" 19:01:00 DEBUG }, 19:01:00 DEBUG "value": [ 19:01:00 DEBUG 19:01:00 DEBUG 19:01:00 DEBUG 19:01:00 DEBUG [...] 19:01:00 DEBUG 19:01:00 DEBUG 19:01:00 DEBUG 19:01:00 DEBUG "instance": "10.142.0.51:9100" 19:01:00 DEBUG }, 19:01:00 DEBUG "value": [ 19:01:00 DEBUG 1734462060, 19:01:00 DEBUG "0.0010000000000000382" 19:01:00 DEBUG ] 19:01:00 DEBUG } 19:01:00 DEBUG ] 19:01:00 DEBUG } 19:01:00 DEBUG } 19:01:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462060" 19:01:00 INFO [http_cmd]: http status code OK 19:01:00 DEBUG --- status code --- 19:01:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:01:00 DEBUG --- http response --- 19:01:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:01:00 DEBUG ----- output ----- 19:01:00 DEBUG { 19:01:00 DEBUG "status": "success", 19:01:00 DEBUG "data": { 19:01:00 DEBUG "resultType": "vector", 19:01:00 DEBUG "result": [ 19:01:00 DEBUG { 19:01:00 DEBUG "metric": { 19:01:00 DEBUG "instance": "10.142.0.11:9100" 19:01:00 DEBUG }, 19:01:00 DEBUG "value": [ 19:01:00 DEBUG 19:01:00 DEBUG 19:01:00 DEBUG 19:01:00 DEBUG [...] 19:01:00 DEBUG 19:01:00 DEBUG 19:01:00 DEBUG 19:01:00 DEBUG "instance": "10.142.0.51:9100" 19:01:00 DEBUG }, 19:01:00 DEBUG "value": [ 19:01:00 DEBUG 1734462060, 19:01:00 DEBUG "0.0009000000000000343" 19:01:00 DEBUG ] 19:01:00 DEBUG } 19:01:00 DEBUG ] 19:01:00 DEBUG } 19:01:00 DEBUG } 19:01:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462060" 19:01:00 INFO [http_cmd]: http status code OK 19:01:00 DEBUG --- status code --- 19:01:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:01:00 DEBUG --- http response --- 19:01:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:01:00 DEBUG ----- output ----- 19:01:00 DEBUG { 19:01:00 DEBUG "status": "success", 19:01:00 DEBUG "data": { 19:01:00 DEBUG "resultType": "vector", 19:01:00 DEBUG "result": [ 19:01:00 DEBUG { 19:01:00 DEBUG "metric": { 19:01:00 DEBUG "pod": "ds-cts-0" 19:01:00 DEBUG }, 19:01:00 DEBUG "value": [ 19:01:00 DEBUG 19:01:00 DEBUG 19:01:00 DEBUG 19:01:00 DEBUG [...] 19:01:00 DEBUG 19:01:00 DEBUG 19:01:00 DEBUG 19:01:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:01:00 DEBUG }, 19:01:00 DEBUG "value": [ 19:01:00 DEBUG 1734462060, 19:01:00 DEBUG "0" 19:01:00 DEBUG ] 19:01:00 DEBUG } 19:01:00 DEBUG ] 19:01:00 DEBUG } 19:01:00 DEBUG } 19:01:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462060" 19:01:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734462060" 19:01:00 INFO [http_cmd]: http status code OK 19:01:00 DEBUG --- status code --- 19:01:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:01:00 DEBUG --- http response --- 19:01:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:01:00 DEBUG ----- output ----- 19:01:00 DEBUG { 19:01:00 DEBUG "status": "success", 19:01:00 DEBUG "data": { 19:01:00 DEBUG "resultType": "vector", 19:01:00 DEBUG "result": [ 19:01:00 DEBUG { 19:01:00 DEBUG "metric": { 19:01:00 DEBUG "pod": "ds-cts-0" 19:01:00 DEBUG }, 19:01:00 DEBUG "value": [ 19:01:00 DEBUG 19:01:00 DEBUG 19:01:00 DEBUG 19:01:00 DEBUG [...] 19:01:00 DEBUG 19:01:00 DEBUG 19:01:00 DEBUG 19:01:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:01:00 DEBUG }, 19:01:00 DEBUG "value": [ 19:01:00 DEBUG 1734462060, 19:01:00 DEBUG "1194.2858644135176" 19:01:00 DEBUG ] 19:01:00 DEBUG } 19:01:00 DEBUG ] 19:01:00 DEBUG } 19:01:00 DEBUG } 19:01:00 INFO [http_cmd]: http status code OK 19:01:00 DEBUG --- status code --- 19:01:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:01:00 DEBUG --- http response --- 19:01:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:01:00 DEBUG ----- output ----- 19:01:00 DEBUG { 19:01:00 DEBUG "status": "success", 19:01:00 DEBUG "data": { 19:01:00 DEBUG "resultType": "vector", 19:01:00 DEBUG "result": [ 19:01:00 DEBUG { 19:01:00 DEBUG "metric": { 19:01:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 19:01:00 DEBUG }, 19:01:00 DEBUG "value": [ 19:01:00 DEBUG 19:01:00 DEBUG 19:01:00 DEBUG 19:01:00 DEBUG [...] 19:01:00 DEBUG 19:01:00 DEBUG 19:01:00 DEBUG 19:01:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 19:01:00 DEBUG }, 19:01:00 DEBUG "value": [ 19:01:00 DEBUG 1734462060, 19:01:00 DEBUG "2581106688" 19:01:00 DEBUG ] 19:01:00 DEBUG } 19:01:00 DEBUG ] 19:01:00 DEBUG } 19:01:00 DEBUG } 19:01:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462060" 19:01:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462060" 19:01:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462060" 19:01:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734462060" 19:01:00 INFO [http_cmd]: http status code OK 19:01:00 DEBUG --- status code --- 19:01:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:01:00 DEBUG --- http response --- 19:01:00 INFO [http_cmd]: http status code OK 19:01:00 DEBUG --- status code --- 19:01:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:01:00 DEBUG --- http response --- 19:01:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:01:00 DEBUG ----- output ----- 19:01:00 INFO [http_cmd]: http status code OK 19:01:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:01:00 DEBUG { 19:01:00 DEBUG --- status code --- 19:01:00 DEBUG ----- output ----- 19:01:00 DEBUG "status": "success", 19:01:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:01:00 DEBUG { 19:01:00 DEBUG "data": { 19:01:00 DEBUG --- http response --- 19:01:00 DEBUG "status": "success", 19:01:00 DEBUG "data": { 19:01:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:01:00 DEBUG "resultType": "vector", 19:01:00 DEBUG "resultType": "vector", 19:01:00 DEBUG ----- output ----- 19:01:00 DEBUG "result": [ 19:01:00 DEBUG "result": [ 19:01:00 DEBUG { 19:01:00 DEBUG { 19:01:00 DEBUG { 19:01:00 DEBUG "status": "success", 19:01:00 DEBUG "metric": { 19:01:00 DEBUG "metric": { 19:01:00 DEBUG "data": { 19:01:00 DEBUG "instance": "10.142.0.11:9100" 19:01:00 DEBUG "instance": "10.142.0.11:9100" 19:01:00 DEBUG "resultType": "vector", 19:01:00 DEBUG }, 19:01:00 DEBUG }, 19:01:00 DEBUG "result": [ 19:01:00 DEBUG "value": [ 19:01:00 DEBUG "value": [ 19:01:00 DEBUG { 19:01:00 DEBUG 19:01:00 DEBUG 19:01:00 DEBUG "metric": { 19:01:00 DEBUG 19:01:00 DEBUG 19:01:00 DEBUG "pod": "ds-cts-0" 19:01:00 INFO [http_cmd]: http status code OK 19:01:00 DEBUG 19:01:00 DEBUG 19:01:00 DEBUG }, 19:01:00 DEBUG --- status code --- 19:01:00 DEBUG [...] 19:01:00 DEBUG [...] 19:01:00 DEBUG "value": [ 19:01:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:01:00 DEBUG 19:01:00 DEBUG 19:01:00 DEBUG 19:01:00 DEBUG --- http response --- 19:01:00 DEBUG 19:01:00 DEBUG 19:01:00 DEBUG 19:01:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:01:00 DEBUG 19:01:00 DEBUG 19:01:00 DEBUG 19:01:00 DEBUG ----- output ----- 19:01:00 DEBUG "instance": "10.142.0.51:9100" 19:01:00 DEBUG "instance": "10.142.0.51:9100" 19:01:00 DEBUG [...] 19:01:00 DEBUG { 19:01:00 DEBUG }, 19:01:00 DEBUG }, 19:01:00 DEBUG 19:01:00 DEBUG "status": "success", 19:01:00 DEBUG "value": [ 19:01:00 DEBUG "value": [ 19:01:00 DEBUG 19:01:00 DEBUG "data": { 19:01:00 DEBUG 1734462060, 19:01:00 DEBUG 1734462060, 19:01:00 DEBUG 19:01:00 DEBUG "resultType": "vector", 19:01:00 DEBUG "169847.4666666667" 19:01:00 DEBUG "72232.46666666669" 19:01:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:01:00 DEBUG "result": [ 19:01:00 DEBUG ] 19:01:00 DEBUG ] 19:01:00 DEBUG }, 19:01:00 DEBUG { 19:01:00 DEBUG } 19:01:00 DEBUG } 19:01:00 DEBUG "value": [ 19:01:00 DEBUG "metric": { 19:01:00 DEBUG ] 19:01:00 DEBUG ] 19:01:00 DEBUG 1734462060, 19:01:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 19:01:00 DEBUG } 19:01:00 DEBUG } 19:01:00 DEBUG "695635968" 19:01:00 DEBUG }, 19:01:00 DEBUG } 19:01:00 DEBUG } 19:01:00 DEBUG ] 19:01:00 DEBUG } 19:01:00 DEBUG "value": [ 19:01:00 DEBUG ] 19:01:00 DEBUG 19:01:00 DEBUG } 19:01:00 DEBUG 19:01:00 DEBUG } 19:01:00 DEBUG 19:01:00 DEBUG [...] 19:01:00 DEBUG 19:01:00 DEBUG 19:01:00 DEBUG 19:01:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 19:01:00 DEBUG }, 19:01:00 DEBUG "value": [ 19:01:00 DEBUG 1734462060, 19:01:00 DEBUG "1.7715001643454973" 19:01:00 DEBUG ] 19:01:00 DEBUG } 19:01:00 DEBUG ] 19:01:00 DEBUG } 19:01:00 DEBUG } 19:01:02 INFO 19:01:02 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 19:01:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:01:02 INFO [loop_until]: OK (rc = 0) 19:01:02 DEBUG --- stdout --- 19:01:02 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 544Mi ds-idrepo-0 7m 737Mi rcs-5d696ff5b-7ttvr 4m 297Mi rcs-5d696ff5b-wwlbr 2m 331Mi 19:01:02 DEBUG --- stderr --- 19:01:02 DEBUG 19:01:03 INFO 19:01:03 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 19:01:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:01:03 INFO [loop_until]: OK (rc = 0) 19:01:03 DEBUG --- stdout --- 19:01:03 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 155m 0% 2220Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1842m 5% 3633Mi 12% 19:01:03 DEBUG --- stderr --- 19:01:03 DEBUG 19:01:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462119" 19:01:59 INFO [http_cmd]: http status code OK 19:01:59 DEBUG --- status code --- 19:01:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:01:59 DEBUG --- http response --- 19:01:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:01:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462119" 19:01:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734462119" 19:01:59 INFO [http_cmd]: http status code OK 19:01:59 DEBUG --- status code --- 19:01:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:01:59 INFO [http_cmd]: http status code OK 19:01:59 DEBUG --- http response --- 19:01:59 DEBUG --- status code --- 19:01:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:01:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:01:59 DEBUG --- http response --- 19:01:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:01:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462119" 19:01:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462119" 19:01:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462119" 19:01:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462119" 19:01:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734462119" 19:01:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462119" 19:01:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462119" 19:01:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462119" 19:01:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462119" 19:01:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462119" 19:01:59 INFO [http_cmd]: http status code OK 19:01:59 DEBUG --- status code --- 19:01:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:01:59 INFO [http_cmd]: http status code OK 19:01:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462119" 19:01:59 DEBUG --- http response --- 19:01:59 INFO [http_cmd]: http status code OK 19:01:59 DEBUG --- status code --- 19:01:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:01:59 DEBUG --- status code --- 19:01:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:01:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:01:59 DEBUG --- http response --- 19:01:59 DEBUG --- http response --- 19:01:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:01:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:01:59 INFO [http_cmd]: http status code OK 19:01:59 DEBUG --- status code --- 19:01:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:01:59 DEBUG --- http response --- 19:01:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:01:59 INFO [http_cmd]: http status code OK 19:01:59 DEBUG --- status code --- 19:01:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:01:59 DEBUG --- http response --- 19:01:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:01:59 INFO [http_cmd]: http status code OK 19:01:59 DEBUG --- status code --- 19:01:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:01:59 DEBUG --- http response --- 19:01:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:01:59 ERROR [http_cmd]: ERROR 19:01:59 ERROR --- status code --- 19:01:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:01:59 ERROR --- http response --- 19:01:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:01:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:01:59 INFO [http_cmd]: http status code OK 19:01:59 INFO [http_cmd]: http status code OK 19:01:59 INFO [http_cmd]: http status code OK 19:01:59 INFO [http_cmd]: http status code OK 19:01:59 DEBUG --- status code --- 19:01:59 DEBUG --- status code --- 19:01:59 DEBUG --- status code --- 19:01:59 DEBUG --- status code --- 19:01:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:01:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:01:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:01:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:01:59 DEBUG --- http response --- 19:01:59 DEBUG --- http response --- 19:01:59 DEBUG --- http response --- 19:01:59 DEBUG --- http response --- 19:01:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:01:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:01:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:01:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:01:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462119" 19:01:59 INFO [http_cmd]: http status code OK 19:01:59 DEBUG --- status code --- 19:01:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:01:59 DEBUG --- http response --- 19:01:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:01:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462119" 19:01:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462119" 19:01:59 INFO [http_cmd]: http status code OK 19:01:59 DEBUG --- status code --- 19:01:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:01:59 INFO [http_cmd]: http status code OK 19:01:59 DEBUG --- http response --- 19:01:59 DEBUG --- status code --- 19:01:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:01:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:01:59 DEBUG --- http response --- 19:01:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:01:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462119" 19:01:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462119" 19:01:59 INFO [http_cmd]: http status code OK 19:01:59 DEBUG --- status code --- 19:01:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:01:59 DEBUG --- http response --- 19:01:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:01:59 INFO [http_cmd]: http status code OK 19:01:59 DEBUG --- status code --- 19:01:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:01:59 DEBUG --- http response --- 19:01:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:02:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734462120" 19:02:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734462120" 19:02:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462120" 19:02:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734462120" 19:02:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462120" 19:02:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462120" 19:02:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462120" 19:02:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462120" 19:02:00 INFO [http_cmd]: http status code OK 19:02:00 DEBUG --- status code --- 19:02:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:02:00 DEBUG --- http response --- 19:02:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:02:00 INFO [http_cmd]: http status code OK 19:02:00 INFO [http_cmd]: http status code OK 19:02:00 INFO [http_cmd]: http status code OK 19:02:00 DEBUG --- status code --- 19:02:00 DEBUG --- status code --- 19:02:00 DEBUG --- status code --- 19:02:00 INFO [http_cmd]: http status code OK 19:02:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:02:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:02:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:02:00 DEBUG --- status code --- 19:02:00 DEBUG --- http response --- 19:02:00 DEBUG --- http response --- 19:02:00 DEBUG --- http response --- 19:02:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:02:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:02:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:02:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:02:00 DEBUG --- http response --- 19:02:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:02:00 INFO [http_cmd]: http status code OK 19:02:00 DEBUG --- status code --- 19:02:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:02:00 DEBUG --- http response --- 19:02:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:02:00 INFO [http_cmd]: http status code OK 19:02:00 INFO [http_cmd]: http status code OK 19:02:00 DEBUG --- status code --- 19:02:00 DEBUG --- status code --- 19:02:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:02:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:02:00 DEBUG --- http response --- 19:02:00 DEBUG --- http response --- 19:02:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:02:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:02:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462120" 19:02:00 INFO [http_cmd]: http status code OK 19:02:00 DEBUG --- status code --- 19:02:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:02:00 DEBUG --- http response --- 19:02:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:02:00 DEBUG ----- output ----- 19:02:00 DEBUG { 19:02:00 DEBUG "status": "success", 19:02:00 DEBUG "data": { 19:02:00 DEBUG "resultType": "vector", 19:02:00 DEBUG "result": [ 19:02:00 DEBUG { 19:02:00 DEBUG "metric": { 19:02:00 DEBUG "pod": "ds-cts-0" 19:02:00 DEBUG }, 19:02:00 DEBUG "value": [ 19:02:00 DEBUG 19:02:00 DEBUG 19:02:00 DEBUG 19:02:00 DEBUG [...] 19:02:00 DEBUG 19:02:00 DEBUG 19:02:00 DEBUG 19:02:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:02:00 DEBUG }, 19:02:00 DEBUG "value": [ 19:02:00 DEBUG 1734462120, 19:02:00 DEBUG "0.0022154226825064757" 19:02:00 DEBUG ] 19:02:00 DEBUG } 19:02:00 DEBUG ] 19:02:00 DEBUG } 19:02:00 DEBUG } 19:02:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462120" 19:02:00 INFO [http_cmd]: http status code OK 19:02:00 DEBUG --- status code --- 19:02:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:02:00 DEBUG --- http response --- 19:02:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:02:00 DEBUG ----- output ----- 19:02:00 DEBUG { 19:02:00 DEBUG "status": "success", 19:02:00 DEBUG "data": { 19:02:00 DEBUG "resultType": "vector", 19:02:00 DEBUG "result": [ 19:02:00 DEBUG { 19:02:00 DEBUG "metric": { 19:02:00 DEBUG "instance": "10.142.0.11:9100" 19:02:00 DEBUG }, 19:02:00 DEBUG "value": [ 19:02:00 DEBUG 19:02:00 DEBUG 19:02:00 DEBUG 19:02:00 DEBUG [...] 19:02:00 DEBUG 19:02:00 DEBUG 19:02:00 DEBUG 19:02:00 DEBUG "instance": "10.142.0.51:9100" 19:02:00 DEBUG }, 19:02:00 DEBUG "value": [ 19:02:00 DEBUG 1734462120, 19:02:00 DEBUG "90153.16666666669" 19:02:00 DEBUG ] 19:02:00 DEBUG } 19:02:00 DEBUG ] 19:02:00 DEBUG } 19:02:00 DEBUG } 19:02:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462120" 19:02:00 INFO [http_cmd]: http status code OK 19:02:00 DEBUG --- status code --- 19:02:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:02:00 DEBUG --- http response --- 19:02:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:02:00 DEBUG ----- output ----- 19:02:00 DEBUG { 19:02:00 DEBUG "status": "success", 19:02:00 DEBUG "data": { 19:02:00 DEBUG "resultType": "vector", 19:02:00 DEBUG "result": [ 19:02:00 DEBUG { 19:02:00 DEBUG "metric": { 19:02:00 DEBUG "instance": "10.142.0.11:9100" 19:02:00 DEBUG }, 19:02:00 DEBUG "value": [ 19:02:00 DEBUG 19:02:00 DEBUG 19:02:00 DEBUG 19:02:00 DEBUG [...] 19:02:00 DEBUG 19:02:00 DEBUG 19:02:00 DEBUG 19:02:00 DEBUG "instance": "10.142.0.51:9100" 19:02:00 DEBUG }, 19:02:00 DEBUG "value": [ 19:02:00 DEBUG 1734462120, 19:02:00 DEBUG "0" 19:02:00 DEBUG ] 19:02:00 DEBUG } 19:02:00 DEBUG ] 19:02:00 DEBUG } 19:02:00 DEBUG } 19:02:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462120" 19:02:00 INFO [http_cmd]: http status code OK 19:02:00 DEBUG --- status code --- 19:02:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:02:00 DEBUG --- http response --- 19:02:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:02:00 DEBUG ----- output ----- 19:02:00 DEBUG { 19:02:00 DEBUG "status": "success", 19:02:00 DEBUG "data": { 19:02:00 DEBUG "resultType": "vector", 19:02:00 DEBUG "result": [ 19:02:00 DEBUG { 19:02:00 DEBUG "metric": { 19:02:00 DEBUG "instance": "10.142.0.11:9100" 19:02:00 DEBUG }, 19:02:00 DEBUG "value": [ 19:02:00 DEBUG 19:02:00 DEBUG 19:02:00 DEBUG 19:02:00 DEBUG [...] 19:02:00 DEBUG 19:02:00 DEBUG 19:02:00 DEBUG 19:02:00 DEBUG "instance": "10.142.0.51:9100" 19:02:00 DEBUG }, 19:02:00 DEBUG "value": [ 19:02:00 DEBUG 1734462120, 19:02:00 DEBUG "0.0004666666666655792" 19:02:00 DEBUG ] 19:02:00 DEBUG } 19:02:00 DEBUG ] 19:02:00 DEBUG } 19:02:00 DEBUG } 19:02:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462120" 19:02:00 INFO [http_cmd]: http status code OK 19:02:00 DEBUG --- status code --- 19:02:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:02:00 DEBUG --- http response --- 19:02:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:02:00 DEBUG ----- output ----- 19:02:00 DEBUG { 19:02:00 DEBUG "status": "success", 19:02:00 DEBUG "data": { 19:02:00 DEBUG "resultType": "vector", 19:02:00 DEBUG "result": [ 19:02:00 DEBUG { 19:02:00 DEBUG "metric": { 19:02:00 DEBUG "pod": "ds-cts-0" 19:02:00 DEBUG }, 19:02:00 DEBUG "value": [ 19:02:00 DEBUG 19:02:00 DEBUG 19:02:00 DEBUG 19:02:00 DEBUG [...] 19:02:00 DEBUG 19:02:00 DEBUG 19:02:00 DEBUG 19:02:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:02:00 DEBUG }, 19:02:00 DEBUG "value": [ 19:02:00 DEBUG 1734462120, 19:02:00 DEBUG "0" 19:02:00 DEBUG ] 19:02:00 DEBUG } 19:02:00 DEBUG ] 19:02:00 DEBUG } 19:02:00 DEBUG } 19:02:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462120" 19:02:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734462120" 19:02:00 INFO [http_cmd]: http status code OK 19:02:00 DEBUG --- status code --- 19:02:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:02:00 DEBUG --- http response --- 19:02:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:02:00 DEBUG ----- output ----- 19:02:00 DEBUG { 19:02:00 DEBUG "status": "success", 19:02:00 DEBUG "data": { 19:02:00 DEBUG "resultType": "vector", 19:02:00 DEBUG "result": [ 19:02:00 DEBUG { 19:02:00 DEBUG "metric": { 19:02:00 DEBUG "pod": "ds-cts-0" 19:02:00 DEBUG }, 19:02:00 DEBUG "value": [ 19:02:00 DEBUG 19:02:00 DEBUG 19:02:00 DEBUG 19:02:00 DEBUG [...] 19:02:00 DEBUG 19:02:00 DEBUG 19:02:00 DEBUG 19:02:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:02:00 DEBUG }, 19:02:00 DEBUG "value": [ 19:02:00 DEBUG 1734462120, 19:02:00 DEBUG "1709.4448676088884" 19:02:00 DEBUG ] 19:02:00 DEBUG } 19:02:00 DEBUG ] 19:02:00 DEBUG } 19:02:00 DEBUG } 19:02:00 INFO [http_cmd]: http status code OK 19:02:00 DEBUG --- status code --- 19:02:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:02:00 DEBUG --- http response --- 19:02:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:02:00 DEBUG ----- output ----- 19:02:00 DEBUG { 19:02:00 DEBUG "status": "success", 19:02:00 DEBUG "data": { 19:02:00 DEBUG "resultType": "vector", 19:02:00 DEBUG "result": [ 19:02:00 DEBUG { 19:02:00 DEBUG "metric": { 19:02:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 19:02:00 DEBUG }, 19:02:00 DEBUG "value": [ 19:02:00 DEBUG 19:02:00 DEBUG 19:02:00 DEBUG 19:02:00 DEBUG [...] 19:02:00 DEBUG 19:02:00 DEBUG 19:02:00 DEBUG 19:02:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 19:02:00 DEBUG }, 19:02:00 DEBUG "value": [ 19:02:00 DEBUG 1734462120, 19:02:00 DEBUG "2574577664" 19:02:00 DEBUG ] 19:02:00 DEBUG } 19:02:00 DEBUG ] 19:02:00 DEBUG } 19:02:00 DEBUG } 19:02:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462120" 19:02:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462120" 19:02:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734462120" 19:02:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462120" 19:02:00 INFO [http_cmd]: http status code OK 19:02:00 DEBUG --- status code --- 19:02:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:02:00 DEBUG --- http response --- 19:02:00 INFO [http_cmd]: http status code OK 19:02:00 DEBUG --- status code --- 19:02:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:02:00 DEBUG --- http response --- 19:02:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:02:00 DEBUG ----- output ----- 19:02:00 DEBUG { 19:02:00 DEBUG "status": "success", 19:02:00 DEBUG "data": { 19:02:00 DEBUG "resultType": "vector", 19:02:00 DEBUG "result": [ 19:02:00 DEBUG { 19:02:00 DEBUG "metric": { 19:02:00 DEBUG "instance": "10.142.0.11:9100" 19:02:00 DEBUG }, 19:02:00 DEBUG "value": [ 19:02:00 DEBUG 19:02:00 DEBUG 19:02:00 DEBUG 19:02:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:02:00 INFO [http_cmd]: http status code OK 19:02:00 DEBUG [...] 19:02:00 INFO [http_cmd]: http status code OK 19:02:00 DEBUG ----- output ----- 19:02:00 DEBUG --- status code --- 19:02:00 DEBUG 19:02:00 DEBUG --- status code --- 19:02:00 DEBUG { 19:02:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:02:00 DEBUG 19:02:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:02:00 DEBUG "status": "success", 19:02:00 DEBUG --- http response --- 19:02:00 DEBUG 19:02:00 DEBUG --- http response --- 19:02:00 DEBUG "data": { 19:02:00 DEBUG "resultType": "vector", 19:02:00 DEBUG "result": [ 19:02:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:02:00 DEBUG "instance": "10.142.0.51:9100" 19:02:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:02:00 DEBUG { 19:02:00 DEBUG ----- output ----- 19:02:00 DEBUG }, 19:02:00 DEBUG ----- output ----- 19:02:00 DEBUG "metric": { 19:02:00 DEBUG { 19:02:00 DEBUG "value": [ 19:02:00 DEBUG { 19:02:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 19:02:00 DEBUG "status": "success", 19:02:00 DEBUG 1734462120, 19:02:00 DEBUG "status": "success", 19:02:00 DEBUG }, 19:02:00 DEBUG "data": { 19:02:00 DEBUG "28262.4" 19:02:00 DEBUG "data": { 19:02:00 DEBUG "value": [ 19:02:00 DEBUG "resultType": "vector", 19:02:00 DEBUG ] 19:02:00 DEBUG "resultType": "vector", 19:02:00 DEBUG 19:02:00 DEBUG "result": [ 19:02:00 DEBUG } 19:02:00 DEBUG "result": [ 19:02:00 DEBUG 19:02:00 DEBUG { 19:02:00 DEBUG ] 19:02:00 DEBUG { 19:02:00 DEBUG 19:02:00 DEBUG "metric": { 19:02:00 DEBUG } 19:02:00 DEBUG "metric": { 19:02:00 DEBUG [...] 19:02:00 DEBUG "pod": "ds-cts-0" 19:02:00 DEBUG } 19:02:00 DEBUG "instance": "10.142.0.11:9100" 19:02:00 DEBUG 19:02:00 DEBUG }, 19:02:00 DEBUG }, 19:02:00 DEBUG 19:02:00 DEBUG "value": [ 19:02:00 DEBUG "value": [ 19:02:00 DEBUG 19:02:00 DEBUG 19:02:00 DEBUG 19:02:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 19:02:00 DEBUG 19:02:00 DEBUG 19:02:00 DEBUG }, 19:02:00 DEBUG 19:02:00 DEBUG 19:02:00 DEBUG "value": [ 19:02:00 DEBUG [...] 19:02:00 DEBUG [...] 19:02:00 DEBUG 1734462120, 19:02:00 DEBUG 19:02:00 DEBUG 19:02:00 DEBUG "1.771545473451819" 19:02:00 DEBUG 19:02:00 DEBUG 19:02:00 DEBUG ] 19:02:00 DEBUG 19:02:00 DEBUG 19:02:00 DEBUG } 19:02:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:02:00 DEBUG "instance": "10.142.0.51:9100" 19:02:00 DEBUG ] 19:02:00 DEBUG }, 19:02:00 DEBUG }, 19:02:00 DEBUG } 19:02:00 DEBUG "value": [ 19:02:00 DEBUG "value": [ 19:02:00 DEBUG } 19:02:00 DEBUG 1734462120, 19:02:00 DEBUG 1734462120, 19:02:00 DEBUG "695685120" 19:02:00 DEBUG "76454.93333333335" 19:02:00 DEBUG ] 19:02:00 DEBUG ] 19:02:00 DEBUG } 19:02:00 DEBUG } 19:02:00 DEBUG ] 19:02:00 DEBUG ] 19:02:00 DEBUG } 19:02:00 DEBUG } 19:02:00 DEBUG } 19:02:00 DEBUG } 19:02:02 INFO 19:02:02 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 19:02:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:02:02 INFO [loop_until]: OK (rc = 0) 19:02:02 DEBUG --- stdout --- 19:02:02 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 544Mi ds-idrepo-0 7m 738Mi rcs-5d696ff5b-7ttvr 4m 297Mi rcs-5d696ff5b-wwlbr 2m 331Mi 19:02:02 DEBUG --- stderr --- 19:02:02 DEBUG 19:02:03 INFO 19:02:03 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 19:02:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:02:03 INFO [loop_until]: OK (rc = 0) 19:02:03 DEBUG --- stdout --- 19:02:03 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 102m 0% 2270Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1848m 5% 3646Mi 12% 19:02:03 DEBUG --- stderr --- 19:02:03 DEBUG 19:02:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462179" 19:02:59 INFO [http_cmd]: http status code OK 19:02:59 DEBUG --- status code --- 19:02:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:02:59 DEBUG --- http response --- 19:02:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:02:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462179" 19:02:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734462179" 19:02:59 INFO [http_cmd]: http status code OK 19:02:59 DEBUG --- status code --- 19:02:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:02:59 DEBUG --- http response --- 19:02:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:02:59 INFO [http_cmd]: http status code OK 19:02:59 DEBUG --- status code --- 19:02:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:02:59 DEBUG --- http response --- 19:02:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:02:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462179" 19:02:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462179" 19:02:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462179" 19:02:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734462179" 19:02:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462179" 19:02:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462179" 19:02:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462179" 19:02:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462179" 19:02:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462179" 19:02:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462179" 19:02:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462179" 19:02:59 INFO [http_cmd]: http status code OK 19:02:59 DEBUG --- status code --- 19:02:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:02:59 DEBUG --- http response --- 19:02:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:02:59 INFO [http_cmd]: http status code OK 19:02:59 DEBUG --- status code --- 19:02:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:02:59 DEBUG --- http response --- 19:02:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:02:59 INFO [http_cmd]: http status code OK 19:02:59 DEBUG --- status code --- 19:02:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:02:59 DEBUG --- http response --- 19:02:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:02:59 INFO [http_cmd]: http status code OK 19:02:59 DEBUG --- status code --- 19:02:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:02:59 DEBUG --- http response --- 19:02:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:02:59 ERROR [http_cmd]: ERROR 19:02:59 ERROR --- status code --- 19:02:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:02:59 ERROR --- http response --- 19:02:59 INFO [http_cmd]: http status code OK 19:02:59 DEBUG --- status code --- 19:02:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:02:59 DEBUG --- http response --- 19:02:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:02:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:02:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:02:59 INFO [http_cmd]: http status code OK 19:02:59 DEBUG --- status code --- 19:02:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:02:59 DEBUG --- http response --- 19:02:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:02:59 INFO [http_cmd]: http status code OK 19:02:59 DEBUG --- status code --- 19:02:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:02:59 DEBUG --- http response --- 19:02:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:02:59 INFO [http_cmd]: http status code OK 19:02:59 DEBUG --- status code --- 19:02:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:02:59 DEBUG --- http response --- 19:02:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:02:59 INFO [http_cmd]: http status code OK 19:02:59 DEBUG --- status code --- 19:02:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:02:59 DEBUG --- http response --- 19:02:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:02:59 INFO [http_cmd]: http status code OK 19:02:59 DEBUG --- status code --- 19:02:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:02:59 DEBUG --- http response --- 19:02:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:02:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462179" 19:02:59 INFO [http_cmd]: http status code OK 19:02:59 DEBUG --- status code --- 19:02:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:02:59 DEBUG --- http response --- 19:02:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:02:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462179" 19:02:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462179" 19:02:59 INFO [http_cmd]: http status code OK 19:02:59 DEBUG --- status code --- 19:02:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:02:59 DEBUG --- http response --- 19:02:59 INFO [http_cmd]: http status code OK 19:02:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:02:59 DEBUG --- status code --- 19:02:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:02:59 DEBUG --- http response --- 19:02:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:02:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462179" 19:02:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462179" 19:02:59 INFO [http_cmd]: http status code OK 19:02:59 DEBUG --- status code --- 19:02:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:02:59 DEBUG --- http response --- 19:02:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:02:59 INFO [http_cmd]: http status code OK 19:02:59 DEBUG --- status code --- 19:02:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:02:59 DEBUG --- http response --- 19:02:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:03:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462180" 19:03:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734462180" 19:03:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734462180" 19:03:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734462180" 19:03:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462180" 19:03:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462180" 19:03:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462180" 19:03:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462180" 19:03:00 INFO [http_cmd]: http status code OK 19:03:00 DEBUG --- status code --- 19:03:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:03:00 DEBUG --- http response --- 19:03:00 INFO [http_cmd]: http status code OK 19:03:00 DEBUG --- status code --- 19:03:00 INFO [http_cmd]: http status code OK 19:03:00 INFO [http_cmd]: http status code OK 19:03:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:03:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:03:00 DEBUG --- http response --- 19:03:00 DEBUG --- status code --- 19:03:00 DEBUG --- status code --- 19:03:00 INFO [http_cmd]: http status code OK 19:03:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:03:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:03:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:03:00 DEBUG --- status code --- 19:03:00 DEBUG --- http response --- 19:03:00 DEBUG --- http response --- 19:03:00 INFO [http_cmd]: http status code OK 19:03:00 INFO [http_cmd]: http status code OK 19:03:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:03:00 INFO [http_cmd]: http status code OK 19:03:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:03:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:03:00 DEBUG --- status code --- 19:03:00 DEBUG --- status code --- 19:03:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:03:00 DEBUG --- status code --- 19:03:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:03:00 DEBUG --- http response --- 19:03:00 DEBUG --- http response --- 19:03:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:03:00 DEBUG --- http response --- 19:03:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:03:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:03:00 DEBUG --- http response --- 19:03:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:03:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:03:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462180" 19:03:00 INFO [http_cmd]: http status code OK 19:03:00 DEBUG --- status code --- 19:03:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:03:00 DEBUG --- http response --- 19:03:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:03:00 DEBUG ----- output ----- 19:03:00 DEBUG { 19:03:00 DEBUG "status": "success", 19:03:00 DEBUG "data": { 19:03:00 DEBUG "resultType": "vector", 19:03:00 DEBUG "result": [ 19:03:00 DEBUG { 19:03:00 DEBUG "metric": { 19:03:00 DEBUG "pod": "ds-cts-0" 19:03:00 DEBUG }, 19:03:00 DEBUG "value": [ 19:03:00 DEBUG 19:03:00 DEBUG 19:03:00 DEBUG 19:03:00 DEBUG [...] 19:03:00 DEBUG 19:03:00 DEBUG 19:03:00 DEBUG 19:03:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:03:00 DEBUG }, 19:03:00 DEBUG "value": [ 19:03:00 DEBUG 1734462180, 19:03:00 DEBUG "0.0030339893027093697" 19:03:00 DEBUG ] 19:03:00 DEBUG } 19:03:00 DEBUG ] 19:03:00 DEBUG } 19:03:00 DEBUG } 19:03:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462180" 19:03:00 INFO [http_cmd]: http status code OK 19:03:00 DEBUG --- status code --- 19:03:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:03:00 DEBUG --- http response --- 19:03:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:03:00 DEBUG ----- output ----- 19:03:00 DEBUG { 19:03:00 DEBUG "status": "success", 19:03:00 DEBUG "data": { 19:03:00 DEBUG "resultType": "vector", 19:03:00 DEBUG "result": [ 19:03:00 DEBUG { 19:03:00 DEBUG "metric": { 19:03:00 DEBUG "instance": "10.142.0.11:9100" 19:03:00 DEBUG }, 19:03:00 DEBUG "value": [ 19:03:00 DEBUG 19:03:00 DEBUG 19:03:00 DEBUG 19:03:00 DEBUG [...] 19:03:00 DEBUG 19:03:00 DEBUG 19:03:00 DEBUG 19:03:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462180" 19:03:00 DEBUG "instance": "10.142.0.51:9100" 19:03:00 DEBUG }, 19:03:00 DEBUG "value": [ 19:03:00 DEBUG 1734462180, 19:03:00 DEBUG "87758.50000000001" 19:03:00 DEBUG ] 19:03:00 DEBUG } 19:03:00 DEBUG ] 19:03:00 DEBUG } 19:03:00 DEBUG } 19:03:00 INFO [http_cmd]: http status code OK 19:03:00 DEBUG --- status code --- 19:03:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:03:00 DEBUG --- http response --- 19:03:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:03:00 DEBUG ----- output ----- 19:03:00 DEBUG { 19:03:00 DEBUG "status": "success", 19:03:00 DEBUG "data": { 19:03:00 DEBUG "resultType": "vector", 19:03:00 DEBUG "result": [ 19:03:00 DEBUG { 19:03:00 DEBUG "metric": { 19:03:00 DEBUG "instance": "10.142.0.11:9100" 19:03:00 DEBUG }, 19:03:00 DEBUG "value": [ 19:03:00 DEBUG 19:03:00 DEBUG 19:03:00 DEBUG 19:03:00 DEBUG [...] 19:03:00 DEBUG 19:03:00 DEBUG 19:03:00 DEBUG 19:03:00 DEBUG "instance": "10.142.0.51:9100" 19:03:00 DEBUG }, 19:03:00 DEBUG "value": [ 19:03:00 DEBUG 1734462180, 19:03:00 DEBUG "0.0003333333333332671" 19:03:00 DEBUG ] 19:03:00 DEBUG } 19:03:00 DEBUG ] 19:03:00 DEBUG } 19:03:00 DEBUG } 19:03:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462180" 19:03:00 INFO [http_cmd]: http status code OK 19:03:00 DEBUG --- status code --- 19:03:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:03:00 DEBUG --- http response --- 19:03:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:03:00 DEBUG ----- output ----- 19:03:00 DEBUG { 19:03:00 DEBUG "status": "success", 19:03:00 DEBUG "data": { 19:03:00 DEBUG "resultType": "vector", 19:03:00 DEBUG "result": [ 19:03:00 DEBUG { 19:03:00 DEBUG "metric": { 19:03:00 DEBUG "instance": "10.142.0.11:9100" 19:03:00 DEBUG }, 19:03:00 DEBUG "value": [ 19:03:00 DEBUG 19:03:00 DEBUG 19:03:00 DEBUG 19:03:00 DEBUG [...] 19:03:00 DEBUG 19:03:00 DEBUG 19:03:00 DEBUG 19:03:00 DEBUG "instance": "10.142.0.51:9100" 19:03:00 DEBUG }, 19:03:00 DEBUG "value": [ 19:03:00 DEBUG 1734462180, 19:03:00 DEBUG "0.0004333333333330341" 19:03:00 DEBUG ] 19:03:00 DEBUG } 19:03:00 DEBUG ] 19:03:00 DEBUG } 19:03:00 DEBUG } 19:03:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462180" 19:03:00 INFO [http_cmd]: http status code OK 19:03:00 DEBUG --- status code --- 19:03:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:03:00 DEBUG --- http response --- 19:03:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:03:00 DEBUG ----- output ----- 19:03:00 DEBUG { 19:03:00 DEBUG "status": "success", 19:03:00 DEBUG "data": { 19:03:00 DEBUG "resultType": "vector", 19:03:00 DEBUG "result": [ 19:03:00 DEBUG { 19:03:00 DEBUG "metric": { 19:03:00 DEBUG "pod": "ds-cts-0" 19:03:00 DEBUG }, 19:03:00 DEBUG "value": [ 19:03:00 DEBUG 19:03:00 DEBUG 19:03:00 DEBUG 19:03:00 DEBUG [...] 19:03:00 DEBUG 19:03:00 DEBUG 19:03:00 DEBUG 19:03:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:03:00 DEBUG }, 19:03:00 DEBUG "value": [ 19:03:00 DEBUG 1734462180, 19:03:00 DEBUG "0" 19:03:00 DEBUG ] 19:03:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462180" 19:03:00 DEBUG } 19:03:00 DEBUG ] 19:03:00 DEBUG } 19:03:00 DEBUG } 19:03:00 INFO [http_cmd]: http status code OK 19:03:00 DEBUG --- status code --- 19:03:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:03:00 DEBUG --- http response --- 19:03:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734462180" 19:03:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:03:00 DEBUG ----- output ----- 19:03:00 DEBUG { 19:03:00 DEBUG "status": "success", 19:03:00 DEBUG "data": { 19:03:00 DEBUG "resultType": "vector", 19:03:00 DEBUG "result": [ 19:03:00 DEBUG { 19:03:00 DEBUG "metric": { 19:03:00 DEBUG "pod": "ds-cts-0" 19:03:00 DEBUG }, 19:03:00 DEBUG "value": [ 19:03:00 DEBUG 19:03:00 DEBUG 19:03:00 DEBUG 19:03:00 DEBUG [...] 19:03:00 DEBUG 19:03:00 DEBUG 19:03:00 DEBUG 19:03:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:03:00 DEBUG }, 19:03:00 DEBUG "value": [ 19:03:00 DEBUG 1734462180, 19:03:00 DEBUG "1245.253822690237" 19:03:00 DEBUG ] 19:03:00 DEBUG } 19:03:00 DEBUG ] 19:03:00 DEBUG } 19:03:00 DEBUG } 19:03:00 INFO [http_cmd]: http status code OK 19:03:00 DEBUG --- status code --- 19:03:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:03:00 DEBUG --- http response --- 19:03:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:03:00 DEBUG ----- output ----- 19:03:00 DEBUG { 19:03:00 DEBUG "status": "success", 19:03:00 DEBUG "data": { 19:03:00 DEBUG "resultType": "vector", 19:03:00 DEBUG "result": [ 19:03:00 DEBUG { 19:03:00 DEBUG "metric": { 19:03:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 19:03:00 DEBUG }, 19:03:00 DEBUG "value": [ 19:03:00 DEBUG 19:03:00 DEBUG 19:03:00 DEBUG 19:03:00 DEBUG [...] 19:03:00 DEBUG 19:03:00 DEBUG 19:03:00 DEBUG 19:03:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 19:03:00 DEBUG }, 19:03:00 DEBUG "value": [ 19:03:00 DEBUG 1734462180, 19:03:00 DEBUG "2569912320" 19:03:00 DEBUG ] 19:03:00 DEBUG } 19:03:00 DEBUG ] 19:03:00 DEBUG } 19:03:00 DEBUG } 19:03:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462180" 19:03:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734462180" 19:03:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462180" 19:03:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462180" 19:03:00 INFO [http_cmd]: http status code OK 19:03:00 DEBUG --- status code --- 19:03:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:03:00 DEBUG --- http response --- 19:03:00 INFO [http_cmd]: http status code OK 19:03:00 DEBUG --- status code --- 19:03:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:03:00 DEBUG --- http response --- 19:03:00 INFO [http_cmd]: http status code OK 19:03:00 DEBUG --- status code --- 19:03:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:03:00 DEBUG --- http response --- 19:03:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:03:00 DEBUG ----- output ----- 19:03:00 DEBUG { 19:03:00 DEBUG "status": "success", 19:03:00 DEBUG "data": { 19:03:00 DEBUG "resultType": "vector", 19:03:00 DEBUG "result": [ 19:03:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:03:00 INFO [http_cmd]: http status code OK 19:03:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:03:00 DEBUG { 19:03:00 DEBUG ----- output ----- 19:03:00 DEBUG --- status code --- 19:03:00 DEBUG ----- output ----- 19:03:00 DEBUG "metric": { 19:03:00 DEBUG { 19:03:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:03:00 DEBUG { 19:03:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 19:03:00 DEBUG "status": "success", 19:03:00 DEBUG --- http response --- 19:03:00 DEBUG "status": "success", 19:03:00 DEBUG }, 19:03:00 DEBUG "data": { 19:03:00 DEBUG "data": { 19:03:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:03:00 DEBUG "value": [ 19:03:00 DEBUG "resultType": "vector", 19:03:00 DEBUG "resultType": "vector", 19:03:00 DEBUG ----- output ----- 19:03:00 DEBUG 19:03:00 DEBUG 19:03:00 DEBUG "result": [ 19:03:00 DEBUG { 19:03:00 DEBUG "result": [ 19:03:00 DEBUG 19:03:00 DEBUG { 19:03:00 DEBUG "status": "success", 19:03:00 DEBUG { 19:03:00 DEBUG [...] 19:03:00 DEBUG "metric": { 19:03:00 DEBUG "data": { 19:03:00 DEBUG "metric": { 19:03:00 DEBUG 19:03:00 DEBUG "instance": "10.142.0.11:9100" 19:03:00 DEBUG "resultType": "vector", 19:03:00 DEBUG "pod": "ds-cts-0" 19:03:00 DEBUG 19:03:00 DEBUG }, 19:03:00 DEBUG "result": [ 19:03:00 DEBUG }, 19:03:00 DEBUG 19:03:00 DEBUG "value": [ 19:03:00 DEBUG { 19:03:00 DEBUG "value": [ 19:03:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 19:03:00 DEBUG 19:03:00 DEBUG "metric": { 19:03:00 DEBUG 19:03:00 DEBUG }, 19:03:00 DEBUG 19:03:00 DEBUG "instance": "10.142.0.11:9100" 19:03:00 DEBUG 19:03:00 DEBUG "value": [ 19:03:00 DEBUG 19:03:00 DEBUG }, 19:03:00 DEBUG 19:03:00 DEBUG 1734462180, 19:03:00 DEBUG [...] 19:03:00 DEBUG "value": [ 19:03:00 DEBUG [...] 19:03:00 DEBUG "1.7680299044060879" 19:03:00 DEBUG 19:03:00 DEBUG 19:03:00 DEBUG 19:03:00 DEBUG ] 19:03:00 DEBUG 19:03:00 DEBUG 19:03:00 DEBUG 19:03:00 DEBUG } 19:03:00 DEBUG 19:03:00 DEBUG 19:03:00 DEBUG 19:03:00 DEBUG ] 19:03:00 DEBUG "instance": "10.142.0.51:9100" 19:03:00 DEBUG [...] 19:03:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:03:00 DEBUG } 19:03:00 DEBUG }, 19:03:00 DEBUG 19:03:00 DEBUG }, 19:03:00 DEBUG } 19:03:00 DEBUG "value": [ 19:03:00 DEBUG 19:03:00 DEBUG "value": [ 19:03:00 DEBUG 1734462180, 19:03:00 DEBUG 19:03:00 DEBUG 1734462180, 19:03:00 DEBUG "71989.03333333334" 19:03:00 DEBUG "instance": "10.142.0.51:9100" 19:03:00 DEBUG "695689216" 19:03:00 DEBUG ] 19:03:00 DEBUG }, 19:03:00 DEBUG ] 19:03:00 DEBUG } 19:03:00 DEBUG "value": [ 19:03:00 DEBUG } 19:03:00 DEBUG ] 19:03:00 DEBUG 1734462180, 19:03:00 DEBUG ] 19:03:00 DEBUG } 19:03:00 DEBUG "25941.33333333334" 19:03:00 DEBUG } 19:03:00 DEBUG } 19:03:00 DEBUG ] 19:03:00 DEBUG } 19:03:00 DEBUG } 19:03:00 DEBUG ] 19:03:00 DEBUG } 19:03:00 DEBUG } 19:03:02 INFO 19:03:02 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 19:03:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:03:03 INFO [loop_until]: OK (rc = 0) 19:03:03 DEBUG --- stdout --- 19:03:03 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 544Mi ds-idrepo-0 9m 738Mi rcs-5d696ff5b-7ttvr 2m 297Mi rcs-5d696ff5b-wwlbr 2m 331Mi 19:03:03 DEBUG --- stderr --- 19:03:03 DEBUG 19:03:03 INFO 19:03:03 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 19:03:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:03:03 INFO [loop_until]: OK (rc = 0) 19:03:03 DEBUG --- stdout --- 19:03:03 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 99m 0% 2273Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1848m 5% 3649Mi 12% 19:03:03 DEBUG --- stderr --- 19:03:03 DEBUG 19:03:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462239" 19:03:59 INFO [http_cmd]: http status code OK 19:03:59 DEBUG --- status code --- 19:03:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:03:59 DEBUG --- http response --- 19:03:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:03:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462239" 19:03:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734462239" 19:03:59 INFO [http_cmd]: http status code OK 19:03:59 INFO [http_cmd]: http status code OK 19:03:59 DEBUG --- status code --- 19:03:59 DEBUG --- status code --- 19:03:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:03:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:03:59 DEBUG --- http response --- 19:03:59 DEBUG --- http response --- 19:03:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:03:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:03:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462239" 19:03:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462239" 19:03:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734462239" 19:03:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462239" 19:03:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462239" 19:03:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462239" 19:03:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462239" 19:03:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462239" 19:03:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462239" 19:03:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462239" 19:03:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462239" 19:03:59 ERROR [http_cmd]: ERROR 19:03:59 ERROR --- status code --- 19:03:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:03:59 ERROR --- http response --- 19:03:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:03:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:03:59 INFO [http_cmd]: http status code OK 19:03:59 DEBUG --- status code --- 19:03:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:03:59 DEBUG --- http response --- 19:03:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:03:59 INFO [http_cmd]: http status code OK 19:03:59 DEBUG --- status code --- 19:03:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:03:59 DEBUG --- http response --- 19:03:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:03:59 INFO [http_cmd]: http status code OK 19:03:59 DEBUG --- status code --- 19:03:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:03:59 INFO [http_cmd]: http status code OK 19:03:59 DEBUG --- http response --- 19:03:59 INFO [http_cmd]: http status code OK 19:03:59 INFO [http_cmd]: http status code OK 19:03:59 DEBUG --- status code --- 19:03:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:03:59 DEBUG --- status code --- 19:03:59 DEBUG --- status code --- 19:03:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:03:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:03:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:03:59 DEBUG --- http response --- 19:03:59 DEBUG --- http response --- 19:03:59 DEBUG --- http response --- 19:03:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:03:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:03:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:03:59 INFO [http_cmd]: http status code OK 19:03:59 DEBUG --- status code --- 19:03:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:03:59 DEBUG --- http response --- 19:03:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:03:59 INFO [http_cmd]: http status code OK 19:03:59 DEBUG --- status code --- 19:03:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:03:59 DEBUG --- http response --- 19:03:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:03:59 INFO [http_cmd]: http status code OK 19:03:59 INFO [http_cmd]: http status code OK 19:03:59 DEBUG --- status code --- 19:03:59 DEBUG --- status code --- 19:03:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:03:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:03:59 DEBUG --- http response --- 19:03:59 DEBUG --- http response --- 19:03:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:03:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:03:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462239" 19:03:59 INFO [http_cmd]: http status code OK 19:03:59 DEBUG --- status code --- 19:03:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:03:59 DEBUG --- http response --- 19:03:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:03:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462239" 19:03:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462239" 19:03:59 INFO [http_cmd]: http status code OK 19:03:59 INFO [http_cmd]: http status code OK 19:03:59 DEBUG --- status code --- 19:03:59 DEBUG --- status code --- 19:03:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:03:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:03:59 DEBUG --- http response --- 19:03:59 DEBUG --- http response --- 19:03:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:03:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:03:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462239" 19:03:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462239" 19:03:59 INFO [http_cmd]: http status code OK 19:03:59 DEBUG --- status code --- 19:03:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:03:59 DEBUG --- http response --- 19:03:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:03:59 INFO [http_cmd]: http status code OK 19:03:59 DEBUG --- status code --- 19:03:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:03:59 DEBUG --- http response --- 19:04:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:04:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734462240" 19:04:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462240" 19:04:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462240" 19:04:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734462240" 19:04:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734462240" 19:04:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462240" 19:04:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462240" 19:04:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462240" 19:04:00 INFO [http_cmd]: http status code OK 19:04:00 DEBUG --- status code --- 19:04:00 INFO [http_cmd]: http status code OK 19:04:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:04:00 DEBUG --- status code --- 19:04:00 DEBUG --- http response --- 19:04:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:04:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:04:00 DEBUG --- http response --- 19:04:00 INFO [http_cmd]: http status code OK 19:04:00 DEBUG --- status code --- 19:04:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:04:00 DEBUG --- http response --- 19:04:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:04:00 INFO [http_cmd]: http status code OK 19:04:00 DEBUG --- status code --- 19:04:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:04:00 DEBUG --- http response --- 19:04:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:04:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:04:00 INFO [http_cmd]: http status code OK 19:04:00 DEBUG --- status code --- 19:04:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:04:00 DEBUG --- http response --- 19:04:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:04:00 INFO [http_cmd]: http status code OK 19:04:00 DEBUG --- status code --- 19:04:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:04:00 DEBUG --- http response --- 19:04:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:04:00 INFO [http_cmd]: http status code OK 19:04:00 DEBUG --- status code --- 19:04:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:04:00 DEBUG --- http response --- 19:04:00 INFO [http_cmd]: http status code OK 19:04:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:04:00 DEBUG --- status code --- 19:04:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:04:00 DEBUG --- http response --- 19:04:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:04:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462240" 19:04:00 INFO [http_cmd]: http status code OK 19:04:00 DEBUG --- status code --- 19:04:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:04:00 DEBUG --- http response --- 19:04:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:04:00 DEBUG ----- output ----- 19:04:00 DEBUG { 19:04:00 DEBUG "status": "success", 19:04:00 DEBUG "data": { 19:04:00 DEBUG "resultType": "vector", 19:04:00 DEBUG "result": [ 19:04:00 DEBUG { 19:04:00 DEBUG "metric": { 19:04:00 DEBUG "pod": "ds-cts-0" 19:04:00 DEBUG }, 19:04:00 DEBUG "value": [ 19:04:00 DEBUG 19:04:00 DEBUG 19:04:00 DEBUG 19:04:00 DEBUG [...] 19:04:00 DEBUG 19:04:00 DEBUG 19:04:00 DEBUG 19:04:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:04:00 DEBUG }, 19:04:00 DEBUG "value": [ 19:04:00 DEBUG 1734462240, 19:04:00 DEBUG "0.006078768381872218" 19:04:00 DEBUG ] 19:04:00 DEBUG } 19:04:00 DEBUG ] 19:04:00 DEBUG } 19:04:00 DEBUG } 19:04:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462240" 19:04:00 INFO [http_cmd]: http status code OK 19:04:00 DEBUG --- status code --- 19:04:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:04:00 DEBUG --- http response --- 19:04:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:04:00 DEBUG ----- output ----- 19:04:00 DEBUG { 19:04:00 DEBUG "status": "success", 19:04:00 DEBUG "data": { 19:04:00 DEBUG "resultType": "vector", 19:04:00 DEBUG "result": [ 19:04:00 DEBUG { 19:04:00 DEBUG "metric": { 19:04:00 DEBUG "instance": "10.142.0.11:9100" 19:04:00 DEBUG }, 19:04:00 DEBUG "value": [ 19:04:00 DEBUG 19:04:00 DEBUG 19:04:00 DEBUG 19:04:00 DEBUG [...] 19:04:00 DEBUG 19:04:00 DEBUG 19:04:00 DEBUG 19:04:00 DEBUG "instance": "10.142.0.51:9100" 19:04:00 DEBUG }, 19:04:00 DEBUG "value": [ 19:04:00 DEBUG 1734462240, 19:04:00 DEBUG "90164.70000000003" 19:04:00 DEBUG ] 19:04:00 DEBUG } 19:04:00 DEBUG ] 19:04:00 DEBUG } 19:04:00 DEBUG } 19:04:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462240" 19:04:00 INFO [http_cmd]: http status code OK 19:04:00 DEBUG --- status code --- 19:04:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:04:00 DEBUG --- http response --- 19:04:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:04:00 DEBUG ----- output ----- 19:04:00 DEBUG { 19:04:00 DEBUG "status": "success", 19:04:00 DEBUG "data": { 19:04:00 DEBUG "resultType": "vector", 19:04:00 DEBUG "result": [ 19:04:00 DEBUG { 19:04:00 DEBUG "metric": { 19:04:00 DEBUG "instance": "10.142.0.11:9100" 19:04:00 DEBUG }, 19:04:00 DEBUG "value": [ 19:04:00 DEBUG 19:04:00 DEBUG 19:04:00 DEBUG 19:04:00 DEBUG [...] 19:04:00 DEBUG 19:04:00 DEBUG 19:04:00 DEBUG 19:04:00 DEBUG "instance": "10.142.0.51:9100" 19:04:00 DEBUG }, 19:04:00 DEBUG "value": [ 19:04:00 DEBUG 1734462240, 19:04:00 DEBUG "0.0003333333333333855" 19:04:00 DEBUG ] 19:04:00 DEBUG } 19:04:00 DEBUG ] 19:04:00 DEBUG } 19:04:00 DEBUG } 19:04:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462240" 19:04:00 INFO [http_cmd]: http status code OK 19:04:00 DEBUG --- status code --- 19:04:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:04:00 DEBUG --- http response --- 19:04:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:04:00 DEBUG ----- output ----- 19:04:00 DEBUG { 19:04:00 DEBUG "status": "success", 19:04:00 DEBUG "data": { 19:04:00 DEBUG "resultType": "vector", 19:04:00 DEBUG "result": [ 19:04:00 DEBUG { 19:04:00 DEBUG "metric": { 19:04:00 DEBUG "instance": "10.142.0.11:9100" 19:04:00 DEBUG }, 19:04:00 DEBUG "value": [ 19:04:00 DEBUG 19:04:00 DEBUG 19:04:00 DEBUG 19:04:00 DEBUG [...] 19:04:00 DEBUG 19:04:00 DEBUG 19:04:00 DEBUG 19:04:00 DEBUG "instance": "10.142.0.51:9100" 19:04:00 DEBUG }, 19:04:00 DEBUG "value": [ 19:04:00 DEBUG 1734462240, 19:04:00 DEBUG "0.0006333333333358839" 19:04:00 DEBUG ] 19:04:00 DEBUG } 19:04:00 DEBUG ] 19:04:00 DEBUG } 19:04:00 DEBUG } 19:04:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462240" 19:04:00 INFO [http_cmd]: http status code OK 19:04:00 DEBUG --- status code --- 19:04:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:04:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462240" 19:04:00 DEBUG --- http response --- 19:04:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:04:00 DEBUG ----- output ----- 19:04:00 DEBUG { 19:04:00 DEBUG "status": "success", 19:04:00 DEBUG "data": { 19:04:00 DEBUG "resultType": "vector", 19:04:00 DEBUG "result": [ 19:04:00 DEBUG { 19:04:00 DEBUG "metric": { 19:04:00 DEBUG "pod": "ds-cts-0" 19:04:00 DEBUG }, 19:04:00 DEBUG "value": [ 19:04:00 DEBUG 19:04:00 DEBUG 19:04:00 DEBUG 19:04:00 DEBUG [...] 19:04:00 DEBUG 19:04:00 DEBUG 19:04:00 DEBUG 19:04:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:04:00 DEBUG }, 19:04:00 DEBUG "value": [ 19:04:00 DEBUG 1734462240, 19:04:00 DEBUG "0" 19:04:00 DEBUG ] 19:04:00 DEBUG } 19:04:00 DEBUG ] 19:04:00 DEBUG } 19:04:00 DEBUG } 19:04:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734462240" 19:04:00 INFO [http_cmd]: http status code OK 19:04:00 DEBUG --- status code --- 19:04:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:04:00 DEBUG --- http response --- 19:04:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:04:00 DEBUG ----- output ----- 19:04:00 DEBUG { 19:04:00 DEBUG "status": "success", 19:04:00 DEBUG "data": { 19:04:00 DEBUG "resultType": "vector", 19:04:00 DEBUG "result": [ 19:04:00 DEBUG { 19:04:00 DEBUG "metric": { 19:04:00 DEBUG "pod": "ds-cts-0" 19:04:00 DEBUG }, 19:04:00 DEBUG "value": [ 19:04:00 DEBUG 19:04:00 DEBUG 19:04:00 DEBUG 19:04:00 DEBUG [...] 19:04:00 DEBUG 19:04:00 DEBUG 19:04:00 DEBUG 19:04:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:04:00 DEBUG }, 19:04:00 DEBUG "value": [ 19:04:00 DEBUG 1734462240, 19:04:00 DEBUG "1238.16053481353" 19:04:00 DEBUG ] 19:04:00 DEBUG } 19:04:00 DEBUG ] 19:04:00 DEBUG } 19:04:00 DEBUG } 19:04:00 INFO [http_cmd]: http status code OK 19:04:00 DEBUG --- status code --- 19:04:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:04:00 DEBUG --- http response --- 19:04:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:04:00 DEBUG ----- output ----- 19:04:00 DEBUG { 19:04:00 DEBUG "status": "success", 19:04:00 DEBUG "data": { 19:04:00 DEBUG "resultType": "vector", 19:04:00 DEBUG "result": [ 19:04:00 DEBUG { 19:04:00 DEBUG "metric": { 19:04:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 19:04:00 DEBUG }, 19:04:00 DEBUG "value": [ 19:04:00 DEBUG 19:04:00 DEBUG 19:04:00 DEBUG 19:04:00 DEBUG [...] 19:04:00 DEBUG 19:04:00 DEBUG 19:04:00 DEBUG 19:04:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 19:04:00 DEBUG }, 19:04:00 DEBUG "value": [ 19:04:00 DEBUG 1734462240, 19:04:00 DEBUG "2576076800" 19:04:00 DEBUG ] 19:04:00 DEBUG } 19:04:00 DEBUG ] 19:04:00 DEBUG } 19:04:00 DEBUG } 19:04:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734462240" 19:04:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462240" 19:04:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462240" 19:04:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462240" 19:04:00 INFO [http_cmd]: http status code OK 19:04:00 INFO [http_cmd]: http status code OK 19:04:00 INFO [http_cmd]: http status code OK 19:04:00 DEBUG --- status code --- 19:04:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:04:00 DEBUG --- status code --- 19:04:00 DEBUG --- status code --- 19:04:00 DEBUG --- http response --- 19:04:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:04:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:04:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:04:00 DEBUG --- http response --- 19:04:00 DEBUG --- http response --- 19:04:00 DEBUG ----- output ----- 19:04:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:04:00 DEBUG ----- output ----- 19:04:00 DEBUG { 19:04:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:04:00 DEBUG { 19:04:00 DEBUG "status": "success", 19:04:00 DEBUG ----- output ----- 19:04:00 DEBUG "status": "success", 19:04:00 DEBUG "data": { 19:04:00 DEBUG { 19:04:00 DEBUG "data": { 19:04:00 DEBUG "resultType": "vector", 19:04:00 DEBUG "status": "success", 19:04:00 DEBUG "resultType": "vector", 19:04:00 DEBUG "result": [ 19:04:00 DEBUG "data": { 19:04:00 DEBUG "result": [ 19:04:00 DEBUG { 19:04:00 DEBUG "metric": { 19:04:00 DEBUG { 19:04:00 DEBUG "resultType": "vector", 19:04:00 DEBUG "instance": "10.142.0.11:9100" 19:04:00 DEBUG "metric": { 19:04:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 19:04:00 DEBUG "result": [ 19:04:00 DEBUG }, 19:04:00 DEBUG }, 19:04:00 DEBUG { 19:04:00 DEBUG "value": [ 19:04:00 DEBUG "value": [ 19:04:00 DEBUG "metric": { 19:04:00 DEBUG 19:04:00 DEBUG 19:04:00 DEBUG "pod": "ds-cts-0" 19:04:00 DEBUG 19:04:00 DEBUG 19:04:00 DEBUG }, 19:04:00 DEBUG 19:04:00 DEBUG 19:04:00 INFO [http_cmd]: http status code OK 19:04:00 DEBUG "value": [ 19:04:00 DEBUG [...] 19:04:00 DEBUG [...] 19:04:00 DEBUG --- status code --- 19:04:00 DEBUG 19:04:00 DEBUG 19:04:00 DEBUG 19:04:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:04:00 DEBUG --- http response --- 19:04:00 DEBUG 19:04:00 DEBUG 19:04:00 DEBUG 19:04:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:04:00 DEBUG 19:04:00 DEBUG 19:04:00 DEBUG 19:04:00 DEBUG ----- output ----- 19:04:00 DEBUG "instance": "10.142.0.51:9100" 19:04:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 19:04:00 DEBUG [...] 19:04:00 DEBUG { 19:04:00 DEBUG }, 19:04:00 DEBUG }, 19:04:00 DEBUG 19:04:00 DEBUG "status": "success", 19:04:00 DEBUG "value": [ 19:04:00 DEBUG "value": [ 19:04:00 DEBUG 19:04:00 DEBUG "data": { 19:04:00 DEBUG 1734462240, 19:04:00 DEBUG 1734462240, 19:04:00 DEBUG "1.769678052439476" 19:04:00 DEBUG 19:04:00 DEBUG "76299.23333333335" 19:04:00 DEBUG "resultType": "vector", 19:04:00 DEBUG ] 19:04:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:04:00 DEBUG ] 19:04:00 DEBUG "result": [ 19:04:00 DEBUG } 19:04:00 DEBUG }, 19:04:00 DEBUG } 19:04:00 DEBUG { 19:04:00 DEBUG ] 19:04:00 DEBUG "value": [ 19:04:00 DEBUG ] 19:04:00 DEBUG "metric": { 19:04:00 DEBUG } 19:04:00 DEBUG 1734462240, 19:04:00 DEBUG } 19:04:00 DEBUG "instance": "10.142.0.11:9100" 19:04:00 DEBUG } 19:04:00 DEBUG "696016896" 19:04:00 DEBUG } 19:04:00 DEBUG }, 19:04:00 DEBUG ] 19:04:00 DEBUG "value": [ 19:04:00 DEBUG } 19:04:00 DEBUG 19:04:00 DEBUG ] 19:04:00 DEBUG 19:04:00 DEBUG } 19:04:00 DEBUG 19:04:00 DEBUG } 19:04:00 DEBUG [...] 19:04:00 DEBUG 19:04:00 DEBUG 19:04:00 DEBUG 19:04:00 DEBUG "instance": "10.142.0.51:9100" 19:04:00 DEBUG }, 19:04:00 DEBUG "value": [ 19:04:00 DEBUG 1734462240, 19:04:00 DEBUG "25258.66666666667" 19:04:00 DEBUG ] 19:04:00 DEBUG } 19:04:00 DEBUG ] 19:04:00 DEBUG } 19:04:00 DEBUG } 19:04:03 INFO 19:04:03 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 19:04:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:04:03 INFO [loop_until]: OK (rc = 0) 19:04:03 DEBUG --- stdout --- 19:04:03 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 7m 544Mi ds-idrepo-0 10m 740Mi rcs-5d696ff5b-7ttvr 3m 297Mi rcs-5d696ff5b-wwlbr 3m 331Mi 19:04:03 DEBUG --- stderr --- 19:04:03 DEBUG 19:04:03 INFO 19:04:03 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 19:04:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:04:03 INFO [loop_until]: OK (rc = 0) 19:04:03 DEBUG --- stdout --- 19:04:03 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 98m 0% 2266Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1833m 5% 3646Mi 12% 19:04:03 DEBUG --- stderr --- 19:04:03 DEBUG 19:04:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462299" 19:04:59 INFO [http_cmd]: http status code OK 19:04:59 DEBUG --- status code --- 19:04:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:04:59 DEBUG --- http response --- 19:04:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:04:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462299" 19:04:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734462299" 19:04:59 INFO [http_cmd]: http status code OK 19:04:59 DEBUG --- status code --- 19:04:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:04:59 INFO [http_cmd]: http status code OK 19:04:59 DEBUG --- http response --- 19:04:59 DEBUG --- status code --- 19:04:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:04:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:04:59 DEBUG --- http response --- 19:04:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:04:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734462299" 19:04:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462299" 19:04:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462299" 19:04:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462299" 19:04:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462299" 19:04:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462299" 19:04:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462299" 19:04:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462299" 19:04:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462299" 19:04:59 INFO [http_cmd]: http status code OK 19:04:59 DEBUG --- status code --- 19:04:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:04:59 DEBUG --- http response --- 19:04:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:04:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462299" 19:04:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462299" 19:04:59 ERROR [http_cmd]: ERROR 19:04:59 ERROR --- status code --- 19:04:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:04:59 ERROR --- http response --- 19:04:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:04:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:04:59 INFO [http_cmd]: http status code OK 19:04:59 DEBUG --- status code --- 19:04:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:04:59 DEBUG --- http response --- 19:04:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:04:59 INFO [http_cmd]: http status code OK 19:04:59 DEBUG --- status code --- 19:04:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:04:59 DEBUG --- http response --- 19:04:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:04:59 INFO [http_cmd]: http status code OK 19:04:59 DEBUG --- status code --- 19:04:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:04:59 DEBUG --- http response --- 19:04:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:04:59 INFO [http_cmd]: http status code OK 19:04:59 DEBUG --- status code --- 19:04:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:04:59 DEBUG --- http response --- 19:04:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:04:59 INFO [http_cmd]: http status code OK 19:04:59 DEBUG --- status code --- 19:04:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:04:59 DEBUG --- http response --- 19:04:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:04:59 INFO [http_cmd]: http status code OK 19:04:59 INFO [http_cmd]: http status code OK 19:04:59 DEBUG --- status code --- 19:04:59 INFO [http_cmd]: http status code OK 19:04:59 DEBUG --- status code --- 19:04:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:04:59 DEBUG --- http response --- 19:04:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:04:59 DEBUG --- http response --- 19:04:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:04:59 DEBUG --- status code --- 19:04:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:04:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:04:59 DEBUG --- http response --- 19:04:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:04:59 INFO [http_cmd]: http status code OK 19:04:59 DEBUG --- status code --- 19:04:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:04:59 DEBUG --- http response --- 19:04:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:04:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462299" 19:04:59 INFO [http_cmd]: http status code OK 19:04:59 DEBUG --- status code --- 19:04:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:04:59 DEBUG --- http response --- 19:04:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:04:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462299" 19:04:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462299" 19:04:59 INFO [http_cmd]: http status code OK 19:04:59 DEBUG --- status code --- 19:04:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:04:59 DEBUG --- http response --- 19:04:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:04:59 INFO [http_cmd]: http status code OK 19:04:59 DEBUG --- status code --- 19:04:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:04:59 DEBUG --- http response --- 19:04:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:04:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462299" 19:05:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462300" 19:05:00 INFO [http_cmd]: http status code OK 19:05:00 DEBUG --- status code --- 19:05:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:05:00 DEBUG --- http response --- 19:05:00 INFO [http_cmd]: http status code OK 19:05:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:05:00 DEBUG --- status code --- 19:05:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:05:00 DEBUG --- http response --- 19:05:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:05:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462300" 19:05:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734462300" 19:05:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462300" 19:05:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734462300" 19:05:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462300" 19:05:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462300" 19:05:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462300" 19:05:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734462300" 19:05:00 INFO [http_cmd]: http status code OK 19:05:00 DEBUG --- status code --- 19:05:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:05:00 DEBUG --- http response --- 19:05:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:05:00 INFO [http_cmd]: http status code OK 19:05:00 DEBUG --- status code --- 19:05:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:05:00 DEBUG --- http response --- 19:05:00 INFO [http_cmd]: http status code OK 19:05:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:05:00 INFO [http_cmd]: http status code OK 19:05:00 DEBUG --- status code --- 19:05:00 DEBUG --- status code --- 19:05:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:05:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:05:00 DEBUG --- http response --- 19:05:00 INFO [http_cmd]: http status code OK 19:05:00 INFO [http_cmd]: http status code OK 19:05:00 DEBUG --- http response --- 19:05:00 INFO [http_cmd]: http status code OK 19:05:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:05:00 DEBUG --- status code --- 19:05:00 DEBUG --- status code --- 19:05:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:05:00 DEBUG --- status code --- 19:05:00 INFO [http_cmd]: http status code OK 19:05:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:05:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:05:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:05:00 DEBUG --- status code --- 19:05:00 DEBUG --- http response --- 19:05:00 DEBUG --- http response --- 19:05:00 DEBUG --- http response --- 19:05:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:05:00 DEBUG --- http response --- 19:05:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:05:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:05:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:05:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:05:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462300" 19:05:00 INFO [http_cmd]: http status code OK 19:05:00 DEBUG --- status code --- 19:05:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:05:00 DEBUG --- http response --- 19:05:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:05:00 DEBUG ----- output ----- 19:05:00 DEBUG { 19:05:00 DEBUG "status": "success", 19:05:00 DEBUG "data": { 19:05:00 DEBUG "resultType": "vector", 19:05:00 DEBUG "result": [ 19:05:00 DEBUG { 19:05:00 DEBUG "metric": { 19:05:00 DEBUG "pod": "ds-cts-0" 19:05:00 DEBUG }, 19:05:00 DEBUG "value": [ 19:05:00 DEBUG 19:05:00 DEBUG 19:05:00 DEBUG 19:05:00 DEBUG [...] 19:05:00 DEBUG 19:05:00 DEBUG 19:05:00 DEBUG 19:05:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:05:00 DEBUG }, 19:05:00 DEBUG "value": [ 19:05:00 DEBUG 1734462300, 19:05:00 DEBUG "0.00297682123213072" 19:05:00 DEBUG ] 19:05:00 DEBUG } 19:05:00 DEBUG ] 19:05:00 DEBUG } 19:05:00 DEBUG } 19:05:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462300" 19:05:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462300" 19:05:00 INFO [http_cmd]: http status code OK 19:05:00 DEBUG --- status code --- 19:05:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:05:00 DEBUG --- http response --- 19:05:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:05:00 DEBUG ----- output ----- 19:05:00 DEBUG { 19:05:00 DEBUG "status": "success", 19:05:00 DEBUG "data": { 19:05:00 DEBUG "resultType": "vector", 19:05:00 DEBUG "result": [ 19:05:00 DEBUG { 19:05:00 DEBUG "metric": { 19:05:00 DEBUG "instance": "10.142.0.11:9100" 19:05:00 DEBUG }, 19:05:00 DEBUG "value": [ 19:05:00 DEBUG 19:05:00 DEBUG 19:05:00 DEBUG 19:05:00 DEBUG [...] 19:05:00 DEBUG 19:05:00 DEBUG 19:05:00 DEBUG 19:05:00 DEBUG "instance": "10.142.0.51:9100" 19:05:00 DEBUG }, 19:05:00 DEBUG "value": [ 19:05:00 DEBUG 1734462300, 19:05:00 DEBUG "87932.43333333335" 19:05:00 DEBUG ] 19:05:00 DEBUG } 19:05:00 DEBUG ] 19:05:00 DEBUG } 19:05:00 DEBUG } 19:05:00 INFO [http_cmd]: http status code OK 19:05:00 DEBUG --- status code --- 19:05:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:05:00 DEBUG --- http response --- 19:05:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:05:00 DEBUG ----- output ----- 19:05:00 DEBUG { 19:05:00 DEBUG "status": "success", 19:05:00 DEBUG "data": { 19:05:00 DEBUG "resultType": "vector", 19:05:00 DEBUG "result": [ 19:05:00 DEBUG { 19:05:00 DEBUG "metric": { 19:05:00 DEBUG "instance": "10.142.0.11:9100" 19:05:00 DEBUG }, 19:05:00 DEBUG "value": [ 19:05:00 DEBUG 19:05:00 DEBUG 19:05:00 DEBUG 19:05:00 DEBUG [...] 19:05:00 DEBUG 19:05:00 DEBUG 19:05:00 DEBUG 19:05:00 DEBUG "instance": "10.142.0.51:9100" 19:05:00 DEBUG }, 19:05:00 DEBUG "value": [ 19:05:00 DEBUG 1734462300, 19:05:00 DEBUG "0.0003333333333333855" 19:05:00 DEBUG ] 19:05:00 DEBUG } 19:05:00 DEBUG ] 19:05:00 DEBUG } 19:05:00 DEBUG } 19:05:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462300" 19:05:00 INFO [http_cmd]: http status code OK 19:05:00 DEBUG --- status code --- 19:05:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:05:00 DEBUG --- http response --- 19:05:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:05:00 DEBUG ----- output ----- 19:05:00 DEBUG { 19:05:00 DEBUG "status": "success", 19:05:00 DEBUG "data": { 19:05:00 DEBUG "resultType": "vector", 19:05:00 DEBUG "result": [ 19:05:00 DEBUG { 19:05:00 DEBUG "metric": { 19:05:00 DEBUG "instance": "10.142.0.11:9100" 19:05:00 DEBUG }, 19:05:00 DEBUG "value": [ 19:05:00 DEBUG 19:05:00 DEBUG 19:05:00 DEBUG 19:05:00 DEBUG [...] 19:05:00 DEBUG 19:05:00 DEBUG 19:05:00 DEBUG 19:05:00 DEBUG "instance": "10.142.0.51:9100" 19:05:00 DEBUG }, 19:05:00 DEBUG "value": [ 19:05:00 DEBUG 1734462300, 19:05:00 DEBUG "0.0004666666666646317" 19:05:00 DEBUG ] 19:05:00 DEBUG } 19:05:00 DEBUG ] 19:05:00 DEBUG } 19:05:00 DEBUG } 19:05:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462300" 19:05:00 INFO [http_cmd]: http status code OK 19:05:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462300" 19:05:00 DEBUG --- status code --- 19:05:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:05:00 DEBUG --- http response --- 19:05:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:05:00 DEBUG ----- output ----- 19:05:00 DEBUG { 19:05:00 DEBUG "status": "success", 19:05:00 DEBUG "data": { 19:05:00 DEBUG "resultType": "vector", 19:05:00 DEBUG "result": [ 19:05:00 DEBUG { 19:05:00 DEBUG "metric": { 19:05:00 DEBUG "pod": "ds-cts-0" 19:05:00 DEBUG }, 19:05:00 DEBUG "value": [ 19:05:00 DEBUG 19:05:00 DEBUG 19:05:00 DEBUG 19:05:00 DEBUG [...] 19:05:00 DEBUG 19:05:00 DEBUG 19:05:00 DEBUG 19:05:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:05:00 DEBUG }, 19:05:00 DEBUG "value": [ 19:05:00 DEBUG 1734462300, 19:05:00 DEBUG "0" 19:05:00 DEBUG ] 19:05:00 DEBUG } 19:05:00 DEBUG ] 19:05:00 DEBUG } 19:05:00 DEBUG } 19:05:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734462300" 19:05:00 INFO [http_cmd]: http status code OK 19:05:00 DEBUG --- status code --- 19:05:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:05:00 DEBUG --- http response --- 19:05:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:05:00 DEBUG ----- output ----- 19:05:00 DEBUG { 19:05:00 DEBUG "status": "success", 19:05:00 DEBUG "data": { 19:05:00 DEBUG "resultType": "vector", 19:05:00 DEBUG "result": [ 19:05:00 DEBUG { 19:05:00 DEBUG "metric": { 19:05:00 DEBUG "pod": "ds-cts-0" 19:05:00 DEBUG }, 19:05:00 DEBUG "value": [ 19:05:00 DEBUG 19:05:00 DEBUG 19:05:00 DEBUG 19:05:00 DEBUG [...] 19:05:00 DEBUG 19:05:00 DEBUG 19:05:00 DEBUG 19:05:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:05:00 DEBUG }, 19:05:00 DEBUG "value": [ 19:05:00 DEBUG 1734462300, 19:05:00 DEBUG "1460.981674659186" 19:05:00 DEBUG ] 19:05:00 DEBUG } 19:05:00 DEBUG ] 19:05:00 DEBUG } 19:05:00 DEBUG } 19:05:00 INFO [http_cmd]: http status code OK 19:05:00 DEBUG --- status code --- 19:05:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:05:00 DEBUG --- http response --- 19:05:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:05:00 DEBUG ----- output ----- 19:05:00 DEBUG { 19:05:00 DEBUG "status": "success", 19:05:00 DEBUG "data": { 19:05:00 DEBUG "resultType": "vector", 19:05:00 DEBUG "result": [ 19:05:00 DEBUG { 19:05:00 DEBUG "metric": { 19:05:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 19:05:00 DEBUG }, 19:05:00 DEBUG "value": [ 19:05:00 DEBUG 19:05:00 DEBUG 19:05:00 DEBUG 19:05:00 DEBUG [...] 19:05:00 DEBUG 19:05:00 DEBUG 19:05:00 DEBUG 19:05:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 19:05:00 DEBUG }, 19:05:00 DEBUG "value": [ 19:05:00 DEBUG 1734462300, 19:05:00 DEBUG "2583060480" 19:05:00 DEBUG ] 19:05:00 DEBUG } 19:05:00 DEBUG ] 19:05:00 DEBUG } 19:05:00 DEBUG } 19:05:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734462300" 19:05:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462300" 19:05:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462300" 19:05:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462300" 19:05:00 INFO [http_cmd]: http status code OK 19:05:00 INFO [http_cmd]: http status code OK 19:05:00 DEBUG --- status code --- 19:05:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:05:00 DEBUG --- http response --- 19:05:00 DEBUG --- status code --- 19:05:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:05:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:05:00 DEBUG ----- output ----- 19:05:00 INFO [http_cmd]: http status code OK 19:05:00 DEBUG --- http response --- 19:05:00 DEBUG { 19:05:00 DEBUG --- status code --- 19:05:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:05:00 DEBUG "status": "success", 19:05:00 DEBUG "data": { 19:05:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:05:00 DEBUG ----- output ----- 19:05:00 INFO [http_cmd]: http status code OK 19:05:00 DEBUG "resultType": "vector", 19:05:00 DEBUG --- http response --- 19:05:00 DEBUG { 19:05:00 DEBUG "status": "success", 19:05:00 DEBUG "result": [ 19:05:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:05:00 DEBUG --- status code --- 19:05:00 DEBUG "data": { 19:05:00 DEBUG { 19:05:00 DEBUG "metric": { 19:05:00 DEBUG "instance": "10.142.0.11:9100" 19:05:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:05:00 DEBUG ----- output ----- 19:05:00 DEBUG "resultType": "vector", 19:05:00 DEBUG }, 19:05:00 DEBUG "value": [ 19:05:00 DEBUG { 19:05:00 DEBUG "result": [ 19:05:00 DEBUG { 19:05:00 DEBUG 19:05:00 DEBUG "status": "success", 19:05:00 DEBUG "data": { 19:05:00 DEBUG "metric": { 19:05:00 DEBUG 19:05:00 DEBUG 19:05:00 DEBUG "resultType": "vector", 19:05:00 DEBUG "instance": "10.142.0.11:9100" 19:05:00 DEBUG }, 19:05:00 DEBUG [...] 19:05:00 DEBUG "result": [ 19:05:00 DEBUG { 19:05:00 DEBUG "value": [ 19:05:00 DEBUG 19:05:00 DEBUG 19:05:00 DEBUG "metric": { 19:05:00 DEBUG 19:05:00 DEBUG 19:05:00 DEBUG 19:05:00 DEBUG "pod": "ds-cts-0" 19:05:00 DEBUG --- http response --- 19:05:00 DEBUG 19:05:00 DEBUG "instance": "10.142.0.51:9100" 19:05:00 DEBUG }, 19:05:00 DEBUG [...] 19:05:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:05:00 DEBUG }, 19:05:00 DEBUG "value": [ 19:05:00 DEBUG 19:05:00 DEBUG 19:05:00 DEBUG ----- output ----- 19:05:00 DEBUG 19:05:00 DEBUG "value": [ 19:05:00 DEBUG 19:05:00 DEBUG "instance": "10.142.0.51:9100" 19:05:00 DEBUG 19:05:00 DEBUG 1734462300, 19:05:00 DEBUG { 19:05:00 DEBUG }, 19:05:00 DEBUG 19:05:00 DEBUG "71789.26666666669" 19:05:00 DEBUG "status": "success", 19:05:00 DEBUG "value": [ 19:05:00 DEBUG [...] 19:05:00 DEBUG ] 19:05:00 DEBUG "data": { 19:05:00 DEBUG 1734462300, 19:05:00 DEBUG 19:05:00 DEBUG } 19:05:00 DEBUG ] 19:05:00 DEBUG "14062.933333333334" 19:05:00 DEBUG 19:05:00 DEBUG 19:05:00 DEBUG } 19:05:00 DEBUG ] 19:05:00 DEBUG "resultType": "vector", 19:05:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:05:00 DEBUG } 19:05:00 DEBUG } 19:05:00 DEBUG "result": [ 19:05:00 DEBUG }, 19:05:00 DEBUG ] 19:05:00 DEBUG } 19:05:00 DEBUG { 19:05:00 DEBUG "value": [ 19:05:00 DEBUG } 19:05:00 DEBUG "metric": { 19:05:00 DEBUG 1734462300, 19:05:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 19:05:00 DEBUG "695758848" 19:05:00 DEBUG }, 19:05:00 DEBUG ] 19:05:00 DEBUG "value": [ 19:05:00 DEBUG } 19:05:00 DEBUG 19:05:00 DEBUG ] 19:05:00 DEBUG 19:05:00 DEBUG } 19:05:00 DEBUG 19:05:00 DEBUG } 19:05:00 DEBUG [...] 19:05:00 DEBUG 19:05:00 DEBUG 19:05:00 DEBUG 19:05:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 19:05:00 DEBUG }, 19:05:00 DEBUG "value": [ 19:05:00 DEBUG 1734462300, 19:05:00 DEBUG "1.77259807317766" 19:05:00 DEBUG ] 19:05:00 DEBUG } 19:05:00 DEBUG ] 19:05:00 DEBUG } 19:05:00 DEBUG } 19:05:03 INFO 19:05:03 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 19:05:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:05:03 INFO [loop_until]: OK (rc = 0) 19:05:03 DEBUG --- stdout --- 19:05:03 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 545Mi ds-idrepo-0 12m 739Mi rcs-5d696ff5b-7ttvr 3m 297Mi rcs-5d696ff5b-wwlbr 2m 331Mi 19:05:03 DEBUG --- stderr --- 19:05:03 DEBUG 19:05:03 INFO 19:05:03 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 19:05:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:05:03 INFO [loop_until]: OK (rc = 0) 19:05:03 DEBUG --- stdout --- 19:05:03 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 101m 0% 2277Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1846m 5% 3642Mi 12% 19:05:03 DEBUG --- stderr --- 19:05:03 DEBUG 19:05:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462359" 19:05:59 INFO [http_cmd]: http status code OK 19:05:59 DEBUG --- status code --- 19:05:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:05:59 DEBUG --- http response --- 19:05:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:05:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734462359" 19:05:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462359" 19:05:59 INFO [http_cmd]: http status code OK 19:05:59 DEBUG --- status code --- 19:05:59 INFO [http_cmd]: http status code OK 19:05:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:05:59 DEBUG --- status code --- 19:05:59 DEBUG --- http response --- 19:05:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:05:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:05:59 DEBUG --- http response --- 19:05:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:05:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734462359" 19:05:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462359" 19:05:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462359" 19:05:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462359" 19:05:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462359" 19:05:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462359" 19:05:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462359" 19:05:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462359" 19:05:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462359" 19:05:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462359" 19:05:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462359" 19:05:59 INFO [http_cmd]: http status code OK 19:05:59 DEBUG --- status code --- 19:05:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:05:59 DEBUG --- http response --- 19:05:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:05:59 INFO [http_cmd]: http status code OK 19:05:59 DEBUG --- status code --- 19:05:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:05:59 DEBUG --- http response --- 19:05:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:05:59 ERROR [http_cmd]: ERROR 19:05:59 ERROR --- status code --- 19:05:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:05:59 ERROR --- http response --- 19:05:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:05:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:05:59 INFO [http_cmd]: http status code OK 19:05:59 DEBUG --- status code --- 19:05:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:05:59 DEBUG --- http response --- 19:05:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:05:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462359" 19:05:59 INFO [http_cmd]: http status code OK 19:05:59 DEBUG --- status code --- 19:05:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:05:59 DEBUG --- http response --- 19:05:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:05:59 INFO [http_cmd]: http status code OK 19:05:59 DEBUG --- status code --- 19:05:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:05:59 DEBUG --- http response --- 19:05:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:05:59 INFO [http_cmd]: http status code OK 19:05:59 INFO [http_cmd]: http status code OK 19:05:59 INFO [http_cmd]: http status code OK 19:05:59 DEBUG --- status code --- 19:05:59 DEBUG --- status code --- 19:05:59 INFO [http_cmd]: http status code OK 19:05:59 DEBUG --- status code --- 19:05:59 INFO [http_cmd]: http status code OK 19:05:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:05:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:05:59 DEBUG --- status code --- 19:05:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:05:59 DEBUG --- status code --- 19:05:59 DEBUG --- http response --- 19:05:59 DEBUG --- http response --- 19:05:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:05:59 DEBUG --- http response --- 19:05:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:05:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:05:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:05:59 DEBUG --- http response --- 19:05:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:05:59 DEBUG --- http response --- 19:05:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:05:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:05:59 INFO [http_cmd]: http status code OK 19:05:59 DEBUG --- status code --- 19:05:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:05:59 DEBUG --- http response --- 19:05:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:05:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462359" 19:05:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462359" 19:05:59 INFO [http_cmd]: http status code OK 19:05:59 INFO [http_cmd]: http status code OK 19:05:59 DEBUG --- status code --- 19:05:59 DEBUG --- status code --- 19:05:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:05:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:05:59 DEBUG --- http response --- 19:05:59 DEBUG --- http response --- 19:05:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:05:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:06:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462360" 19:06:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462360" 19:06:00 INFO [http_cmd]: http status code OK 19:06:00 DEBUG --- status code --- 19:06:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:06:00 DEBUG --- http response --- 19:06:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:06:00 INFO [http_cmd]: http status code OK 19:06:00 DEBUG --- status code --- 19:06:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:06:00 DEBUG --- http response --- 19:06:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:06:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462360" 19:06:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734462360" 19:06:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462360" 19:06:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462360" 19:06:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734462360" 19:06:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462360" 19:06:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734462360" 19:06:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462360" 19:06:00 INFO [http_cmd]: http status code OK 19:06:00 DEBUG --- status code --- 19:06:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:06:00 DEBUG --- http response --- 19:06:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:06:00 INFO [http_cmd]: http status code OK 19:06:00 DEBUG --- status code --- 19:06:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:06:00 DEBUG --- http response --- 19:06:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:06:00 INFO [http_cmd]: http status code OK 19:06:00 DEBUG --- status code --- 19:06:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:06:00 DEBUG --- http response --- 19:06:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:06:00 INFO [http_cmd]: http status code OK 19:06:00 DEBUG --- status code --- 19:06:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:06:00 DEBUG --- http response --- 19:06:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:06:00 INFO [http_cmd]: http status code OK 19:06:00 DEBUG --- status code --- 19:06:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:06:00 DEBUG --- http response --- 19:06:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:06:00 INFO [http_cmd]: http status code OK 19:06:00 DEBUG --- status code --- 19:06:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:06:00 DEBUG --- http response --- 19:06:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:06:00 INFO [http_cmd]: http status code OK 19:06:00 DEBUG --- status code --- 19:06:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:06:00 DEBUG --- http response --- 19:06:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:06:00 INFO [http_cmd]: http status code OK 19:06:00 DEBUG --- status code --- 19:06:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:06:00 DEBUG --- http response --- 19:06:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:06:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462360" 19:06:00 INFO [http_cmd]: http status code OK 19:06:00 DEBUG --- status code --- 19:06:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:06:00 DEBUG --- http response --- 19:06:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:06:00 DEBUG ----- output ----- 19:06:00 DEBUG { 19:06:00 DEBUG "status": "success", 19:06:00 DEBUG "data": { 19:06:00 DEBUG "resultType": "vector", 19:06:00 DEBUG "result": [ 19:06:00 DEBUG { 19:06:00 DEBUG "metric": { 19:06:00 DEBUG "pod": "ds-cts-0" 19:06:00 DEBUG }, 19:06:00 DEBUG "value": [ 19:06:00 DEBUG 19:06:00 DEBUG 19:06:00 DEBUG 19:06:00 DEBUG [...] 19:06:00 DEBUG 19:06:00 DEBUG 19:06:00 DEBUG 19:06:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:06:00 DEBUG }, 19:06:00 DEBUG "value": [ 19:06:00 DEBUG 1734462360, 19:06:00 DEBUG "0.008222429938716384" 19:06:00 DEBUG ] 19:06:00 DEBUG } 19:06:00 DEBUG ] 19:06:00 DEBUG } 19:06:00 DEBUG } 19:06:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462360" 19:06:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462360" 19:06:00 INFO [http_cmd]: http status code OK 19:06:00 DEBUG --- status code --- 19:06:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:06:00 DEBUG --- http response --- 19:06:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:06:00 DEBUG ----- output ----- 19:06:00 DEBUG { 19:06:00 DEBUG "status": "success", 19:06:00 DEBUG "data": { 19:06:00 DEBUG "resultType": "vector", 19:06:00 DEBUG "result": [ 19:06:00 DEBUG { 19:06:00 DEBUG "metric": { 19:06:00 DEBUG "instance": "10.142.0.11:9100" 19:06:00 DEBUG }, 19:06:00 DEBUG "value": [ 19:06:00 DEBUG 19:06:00 DEBUG 19:06:00 DEBUG 19:06:00 DEBUG [...] 19:06:00 DEBUG 19:06:00 DEBUG 19:06:00 DEBUG 19:06:00 DEBUG "instance": "10.142.0.51:9100" 19:06:00 DEBUG }, 19:06:00 DEBUG "value": [ 19:06:00 DEBUG 1734462360, 19:06:00 DEBUG "90084.13333333333" 19:06:00 DEBUG ] 19:06:00 DEBUG } 19:06:00 DEBUG ] 19:06:00 DEBUG } 19:06:00 DEBUG } 19:06:00 INFO [http_cmd]: http status code OK 19:06:00 DEBUG --- status code --- 19:06:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:06:00 DEBUG --- http response --- 19:06:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:06:00 DEBUG ----- output ----- 19:06:00 DEBUG { 19:06:00 DEBUG "status": "success", 19:06:00 DEBUG "data": { 19:06:00 DEBUG "resultType": "vector", 19:06:00 DEBUG "result": [ 19:06:00 DEBUG { 19:06:00 DEBUG "metric": { 19:06:00 DEBUG "instance": "10.142.0.11:9100" 19:06:00 DEBUG }, 19:06:00 DEBUG "value": [ 19:06:00 DEBUG 19:06:00 DEBUG 19:06:00 DEBUG 19:06:00 DEBUG [...] 19:06:00 DEBUG 19:06:00 DEBUG 19:06:00 DEBUG 19:06:00 DEBUG "instance": "10.142.0.51:9100" 19:06:00 DEBUG }, 19:06:00 DEBUG "value": [ 19:06:00 DEBUG 1734462360, 19:06:00 DEBUG "0" 19:06:00 DEBUG ] 19:06:00 DEBUG } 19:06:00 DEBUG ] 19:06:00 DEBUG } 19:06:00 DEBUG } 19:06:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462360" 19:06:00 INFO [http_cmd]: http status code OK 19:06:00 DEBUG --- status code --- 19:06:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:06:00 DEBUG --- http response --- 19:06:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:06:00 DEBUG ----- output ----- 19:06:00 DEBUG { 19:06:00 DEBUG "status": "success", 19:06:00 DEBUG "data": { 19:06:00 DEBUG "resultType": "vector", 19:06:00 DEBUG "result": [ 19:06:00 DEBUG { 19:06:00 DEBUG "metric": { 19:06:00 DEBUG "instance": "10.142.0.11:9100" 19:06:00 DEBUG }, 19:06:00 DEBUG "value": [ 19:06:00 DEBUG 19:06:00 DEBUG 19:06:00 DEBUG 19:06:00 DEBUG [...] 19:06:00 DEBUG 19:06:00 DEBUG 19:06:00 DEBUG 19:06:00 DEBUG "instance": "10.142.0.51:9100" 19:06:00 DEBUG }, 19:06:00 DEBUG "value": [ 19:06:00 DEBUG 1734462360, 19:06:00 DEBUG "0.0006999999999990793" 19:06:00 DEBUG ] 19:06:00 DEBUG } 19:06:00 DEBUG ] 19:06:00 DEBUG } 19:06:00 DEBUG } 19:06:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462360" 19:06:00 INFO [http_cmd]: http status code OK 19:06:00 DEBUG --- status code --- 19:06:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:06:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462360" 19:06:00 DEBUG --- http response --- 19:06:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:06:00 DEBUG ----- output ----- 19:06:00 DEBUG { 19:06:00 DEBUG "status": "success", 19:06:00 DEBUG "data": { 19:06:00 DEBUG "resultType": "vector", 19:06:00 DEBUG "result": [ 19:06:00 DEBUG { 19:06:00 DEBUG "metric": { 19:06:00 DEBUG "pod": "ds-cts-0" 19:06:00 DEBUG }, 19:06:00 DEBUG "value": [ 19:06:00 DEBUG 19:06:00 DEBUG 19:06:00 DEBUG 19:06:00 DEBUG [...] 19:06:00 DEBUG 19:06:00 DEBUG 19:06:00 DEBUG 19:06:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:06:00 DEBUG }, 19:06:00 DEBUG "value": [ 19:06:00 DEBUG 1734462360, 19:06:00 DEBUG "0" 19:06:00 DEBUG ] 19:06:00 DEBUG } 19:06:00 DEBUG ] 19:06:00 DEBUG } 19:06:00 DEBUG } 19:06:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734462360" 19:06:00 INFO [http_cmd]: http status code OK 19:06:00 DEBUG --- status code --- 19:06:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:06:00 DEBUG --- http response --- 19:06:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:06:00 DEBUG ----- output ----- 19:06:00 DEBUG { 19:06:00 DEBUG "status": "success", 19:06:00 DEBUG "data": { 19:06:00 DEBUG "resultType": "vector", 19:06:00 DEBUG "result": [ 19:06:00 DEBUG { 19:06:00 DEBUG "metric": { 19:06:00 DEBUG "pod": "ds-cts-0" 19:06:00 DEBUG }, 19:06:00 DEBUG "value": [ 19:06:00 DEBUG 19:06:00 DEBUG 19:06:00 DEBUG 19:06:00 DEBUG [...] 19:06:00 DEBUG 19:06:00 DEBUG 19:06:00 DEBUG 19:06:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:06:00 DEBUG }, 19:06:00 DEBUG "value": [ 19:06:00 DEBUG 1734462360, 19:06:00 DEBUG "1365.1929133220858" 19:06:00 DEBUG ] 19:06:00 DEBUG } 19:06:00 DEBUG ] 19:06:00 DEBUG } 19:06:00 DEBUG } 19:06:00 INFO [http_cmd]: http status code OK 19:06:00 DEBUG --- status code --- 19:06:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:06:00 DEBUG --- http response --- 19:06:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:06:00 DEBUG ----- output ----- 19:06:00 DEBUG { 19:06:00 DEBUG "status": "success", 19:06:00 DEBUG "data": { 19:06:00 DEBUG "resultType": "vector", 19:06:00 DEBUG "result": [ 19:06:00 DEBUG { 19:06:00 DEBUG "metric": { 19:06:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 19:06:00 DEBUG }, 19:06:00 DEBUG "value": [ 19:06:00 DEBUG 19:06:00 DEBUG 19:06:00 DEBUG 19:06:00 DEBUG [...] 19:06:00 DEBUG 19:06:00 DEBUG 19:06:00 DEBUG 19:06:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 19:06:00 DEBUG }, 19:06:00 DEBUG "value": [ 19:06:00 DEBUG 1734462360, 19:06:00 DEBUG "2599432192" 19:06:00 DEBUG ] 19:06:00 DEBUG } 19:06:00 DEBUG ] 19:06:00 DEBUG } 19:06:00 DEBUG } 19:06:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462360" 19:06:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462360" 19:06:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462360" 19:06:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734462360" 19:06:00 INFO [http_cmd]: http status code OK 19:06:00 DEBUG --- status code --- 19:06:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:06:00 DEBUG --- http response --- 19:06:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:06:00 DEBUG ----- output ----- 19:06:00 DEBUG { 19:06:00 DEBUG "status": "success", 19:06:00 DEBUG "data": { 19:06:00 DEBUG "resultType": "vector", 19:06:00 DEBUG "result": [ 19:06:00 DEBUG { 19:06:00 DEBUG "metric": { 19:06:00 DEBUG "instance": "10.142.0.11:9100" 19:06:00 INFO [http_cmd]: http status code OK 19:06:00 INFO [http_cmd]: http status code OK 19:06:00 DEBUG }, 19:06:00 INFO [http_cmd]: http status code OK 19:06:00 DEBUG --- status code --- 19:06:00 DEBUG --- status code --- 19:06:00 DEBUG "value": [ 19:06:00 DEBUG --- status code --- 19:06:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:06:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:06:00 DEBUG 19:06:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:06:00 DEBUG --- http response --- 19:06:00 DEBUG --- http response --- 19:06:00 DEBUG 19:06:00 DEBUG --- http response --- 19:06:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:06:00 DEBUG 19:06:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:06:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:06:00 DEBUG ----- output ----- 19:06:00 DEBUG [...] 19:06:00 DEBUG ----- output ----- 19:06:00 DEBUG ----- output ----- 19:06:00 DEBUG { 19:06:00 DEBUG 19:06:00 DEBUG { 19:06:00 DEBUG { 19:06:00 DEBUG "status": "success", 19:06:00 DEBUG 19:06:00 DEBUG "status": "success", 19:06:00 DEBUG "status": "success", 19:06:00 DEBUG "data": { 19:06:00 DEBUG 19:06:00 DEBUG "data": { 19:06:00 DEBUG "data": { 19:06:00 DEBUG "resultType": "vector", 19:06:00 DEBUG "instance": "10.142.0.51:9100" 19:06:00 DEBUG "resultType": "vector", 19:06:00 DEBUG "resultType": "vector", 19:06:00 DEBUG "result": [ 19:06:00 DEBUG }, 19:06:00 DEBUG "result": [ 19:06:00 DEBUG "result": [ 19:06:00 DEBUG { 19:06:00 DEBUG "value": [ 19:06:00 DEBUG { 19:06:00 DEBUG { 19:06:00 DEBUG "metric": { 19:06:00 DEBUG 1734462360, 19:06:00 DEBUG "metric": { 19:06:00 DEBUG "metric": { 19:06:00 DEBUG "instance": "10.142.0.11:9100" 19:06:00 DEBUG "18158.933333333338" 19:06:00 DEBUG "pod": "ds-cts-0" 19:06:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 19:06:00 DEBUG }, 19:06:00 DEBUG ] 19:06:00 DEBUG }, 19:06:00 DEBUG }, 19:06:00 DEBUG "value": [ 19:06:00 DEBUG } 19:06:00 DEBUG "value": [ 19:06:00 DEBUG "value": [ 19:06:00 DEBUG 19:06:00 DEBUG ] 19:06:00 DEBUG 19:06:00 DEBUG 19:06:00 DEBUG 19:06:00 DEBUG } 19:06:00 DEBUG 19:06:00 DEBUG 19:06:00 DEBUG 19:06:00 DEBUG } 19:06:00 DEBUG 19:06:00 DEBUG 19:06:00 DEBUG [...] 19:06:00 DEBUG [...] 19:06:00 DEBUG [...] 19:06:00 DEBUG 19:06:00 DEBUG 19:06:00 DEBUG 19:06:00 DEBUG 19:06:00 DEBUG 19:06:00 DEBUG 19:06:00 DEBUG 19:06:00 DEBUG 19:06:00 DEBUG 19:06:00 DEBUG "instance": "10.142.0.51:9100" 19:06:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:06:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 19:06:00 DEBUG }, 19:06:00 DEBUG }, 19:06:00 DEBUG }, 19:06:00 DEBUG "value": [ 19:06:00 DEBUG "value": [ 19:06:00 DEBUG "value": [ 19:06:00 DEBUG 1734462360, 19:06:00 DEBUG 1734462360, 19:06:00 DEBUG 1734462360, 19:06:00 DEBUG "73872.23333333334" 19:06:00 DEBUG "695857152" 19:06:00 DEBUG "1.761278466689327" 19:06:00 DEBUG ] 19:06:00 DEBUG ] 19:06:00 DEBUG ] 19:06:00 DEBUG } 19:06:00 DEBUG } 19:06:00 DEBUG } 19:06:00 DEBUG ] 19:06:00 DEBUG ] 19:06:00 DEBUG ] 19:06:00 DEBUG } 19:06:00 DEBUG } 19:06:00 DEBUG } 19:06:00 DEBUG } 19:06:00 DEBUG } 19:06:00 DEBUG } 19:06:03 INFO 19:06:03 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 19:06:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:06:03 INFO [loop_until]: OK (rc = 0) 19:06:03 DEBUG --- stdout --- 19:06:03 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 544Mi ds-idrepo-0 7m 741Mi rcs-5d696ff5b-7ttvr 3m 297Mi rcs-5d696ff5b-wwlbr 7m 331Mi 19:06:03 DEBUG --- stderr --- 19:06:03 DEBUG 19:06:03 INFO 19:06:03 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 19:06:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:06:03 INFO [loop_until]: OK (rc = 0) 19:06:03 DEBUG --- stdout --- 19:06:03 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 105m 0% 2285Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1828m 5% 3649Mi 12% 19:06:03 DEBUG --- stderr --- 19:06:03 DEBUG 19:06:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462419" 19:06:59 INFO [http_cmd]: http status code OK 19:06:59 DEBUG --- status code --- 19:06:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:06:59 DEBUG --- http response --- 19:06:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:06:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734462419" 19:06:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462419" 19:06:59 INFO [http_cmd]: http status code OK 19:06:59 DEBUG --- status code --- 19:06:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:06:59 INFO [http_cmd]: http status code OK 19:06:59 DEBUG --- http response --- 19:06:59 DEBUG --- status code --- 19:06:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:06:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:06:59 DEBUG --- http response --- 19:06:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:06:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734462419" 19:06:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462419" 19:06:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462419" 19:06:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462419" 19:06:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462419" 19:06:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462419" 19:06:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462419" 19:06:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462419" 19:06:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462419" 19:06:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462419" 19:06:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462419" 19:06:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462419" 19:06:59 INFO [http_cmd]: http status code OK 19:06:59 DEBUG --- status code --- 19:06:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:06:59 DEBUG --- http response --- 19:06:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:06:59 INFO [http_cmd]: http status code OK 19:06:59 DEBUG --- status code --- 19:06:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:06:59 DEBUG --- http response --- 19:06:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:06:59 INFO [http_cmd]: http status code OK 19:06:59 INFO [http_cmd]: http status code OK 19:06:59 INFO [http_cmd]: http status code OK 19:06:59 INFO [http_cmd]: http status code OK 19:06:59 INFO [http_cmd]: http status code OK 19:06:59 ERROR [http_cmd]: ERROR 19:06:59 DEBUG --- status code --- 19:06:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:06:59 DEBUG --- status code --- 19:06:59 DEBUG --- status code --- 19:06:59 DEBUG --- status code --- 19:06:59 ERROR --- status code --- 19:06:59 DEBUG --- status code --- 19:06:59 DEBUG --- http response --- 19:06:59 INFO [http_cmd]: http status code OK 19:06:59 INFO [http_cmd]: http status code OK 19:06:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:06:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:06:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:06:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:06:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:06:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:06:59 INFO [http_cmd]: http status code OK 19:06:59 DEBUG --- status code --- 19:06:59 DEBUG --- status code --- 19:06:59 DEBUG --- http response --- 19:06:59 DEBUG --- http response --- 19:06:59 DEBUG --- http response --- 19:06:59 ERROR --- http response --- 19:06:59 DEBUG --- http response --- 19:06:59 DEBUG --- status code --- 19:06:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:06:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:06:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:06:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:06:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:06:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:06:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:06:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:06:59 DEBUG --- http response --- 19:06:59 DEBUG --- http response --- 19:06:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:06:59 INFO [http_cmd]: http status code OK 19:06:59 DEBUG --- http response --- 19:06:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:06:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:06:59 DEBUG --- status code --- 19:06:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:06:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:06:59 DEBUG --- http response --- 19:06:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:06:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462419" 19:06:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462419" 19:07:00 INFO [http_cmd]: http status code OK 19:07:00 DEBUG --- status code --- 19:07:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:07:00 INFO [http_cmd]: http status code OK 19:07:00 DEBUG --- http response --- 19:07:00 DEBUG --- status code --- 19:07:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:07:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:07:00 DEBUG --- http response --- 19:07:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:07:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462420" 19:07:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462420" 19:07:00 INFO [http_cmd]: http status code OK 19:07:00 DEBUG --- status code --- 19:07:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:07:00 INFO [http_cmd]: http status code OK 19:07:00 DEBUG --- http response --- 19:07:00 DEBUG --- status code --- 19:07:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:07:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:07:00 DEBUG --- http response --- 19:07:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:07:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734462420" 19:07:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462420" 19:07:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462420" 19:07:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462420" 19:07:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734462420" 19:07:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462420" 19:07:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462420" 19:07:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734462420" 19:07:00 INFO [http_cmd]: http status code OK 19:07:00 DEBUG --- status code --- 19:07:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:07:00 DEBUG --- http response --- 19:07:00 INFO [http_cmd]: http status code OK 19:07:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:07:00 DEBUG --- status code --- 19:07:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:07:00 DEBUG --- http response --- 19:07:00 INFO [http_cmd]: http status code OK 19:07:00 DEBUG --- status code --- 19:07:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:07:00 DEBUG --- http response --- 19:07:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:07:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:07:00 INFO [http_cmd]: http status code OK 19:07:00 INFO [http_cmd]: http status code OK 19:07:00 DEBUG --- status code --- 19:07:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:07:00 DEBUG --- status code --- 19:07:00 DEBUG --- http response --- 19:07:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:07:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:07:00 DEBUG --- http response --- 19:07:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:07:00 INFO [http_cmd]: http status code OK 19:07:00 DEBUG --- status code --- 19:07:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:07:00 DEBUG --- http response --- 19:07:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:07:00 INFO [http_cmd]: http status code OK 19:07:00 DEBUG --- status code --- 19:07:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:07:00 DEBUG --- http response --- 19:07:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:07:00 INFO [http_cmd]: http status code OK 19:07:00 DEBUG --- status code --- 19:07:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:07:00 DEBUG --- http response --- 19:07:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:07:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462420" 19:07:00 INFO [http_cmd]: http status code OK 19:07:00 DEBUG --- status code --- 19:07:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:07:00 DEBUG --- http response --- 19:07:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:07:00 DEBUG ----- output ----- 19:07:00 DEBUG { 19:07:00 DEBUG "status": "success", 19:07:00 DEBUG "data": { 19:07:00 DEBUG "resultType": "vector", 19:07:00 DEBUG "result": [ 19:07:00 DEBUG { 19:07:00 DEBUG "metric": { 19:07:00 DEBUG "pod": "ds-cts-0" 19:07:00 DEBUG }, 19:07:00 DEBUG "value": [ 19:07:00 DEBUG 19:07:00 DEBUG 19:07:00 DEBUG 19:07:00 DEBUG [...] 19:07:00 DEBUG 19:07:00 DEBUG 19:07:00 DEBUG 19:07:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:07:00 DEBUG }, 19:07:00 DEBUG "value": [ 19:07:00 DEBUG 1734462420, 19:07:00 DEBUG "0.005683463050680694" 19:07:00 DEBUG ] 19:07:00 DEBUG } 19:07:00 DEBUG ] 19:07:00 DEBUG } 19:07:00 DEBUG } 19:07:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462420" 19:07:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462420" 19:07:00 INFO [http_cmd]: http status code OK 19:07:00 DEBUG --- status code --- 19:07:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:07:00 DEBUG --- http response --- 19:07:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:07:00 DEBUG ----- output ----- 19:07:00 DEBUG { 19:07:00 DEBUG "status": "success", 19:07:00 DEBUG "data": { 19:07:00 DEBUG "resultType": "vector", 19:07:00 DEBUG "result": [ 19:07:00 DEBUG { 19:07:00 DEBUG "metric": { 19:07:00 DEBUG "instance": "10.142.0.11:9100" 19:07:00 DEBUG }, 19:07:00 DEBUG "value": [ 19:07:00 DEBUG 19:07:00 DEBUG 19:07:00 DEBUG 19:07:00 DEBUG [...] 19:07:00 DEBUG 19:07:00 DEBUG 19:07:00 DEBUG 19:07:00 DEBUG "instance": "10.142.0.51:9100" 19:07:00 DEBUG }, 19:07:00 DEBUG "value": [ 19:07:00 DEBUG 1734462420, 19:07:00 DEBUG "90249.26666666668" 19:07:00 DEBUG ] 19:07:00 DEBUG } 19:07:00 DEBUG ] 19:07:00 DEBUG } 19:07:00 DEBUG } 19:07:00 INFO [http_cmd]: http status code OK 19:07:00 DEBUG --- status code --- 19:07:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:07:00 DEBUG --- http response --- 19:07:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:07:00 DEBUG ----- output ----- 19:07:00 DEBUG { 19:07:00 DEBUG "status": "success", 19:07:00 DEBUG "data": { 19:07:00 DEBUG "resultType": "vector", 19:07:00 DEBUG "result": [ 19:07:00 DEBUG { 19:07:00 DEBUG "metric": { 19:07:00 DEBUG "instance": "10.142.0.11:9100" 19:07:00 DEBUG }, 19:07:00 DEBUG "value": [ 19:07:00 DEBUG 19:07:00 DEBUG 19:07:00 DEBUG 19:07:00 DEBUG [...] 19:07:00 DEBUG 19:07:00 DEBUG 19:07:00 DEBUG 19:07:00 DEBUG "instance": "10.142.0.51:9100" 19:07:00 DEBUG }, 19:07:00 DEBUG "value": [ 19:07:00 DEBUG 1734462420, 19:07:00 DEBUG "0.0013333333333334237" 19:07:00 DEBUG ] 19:07:00 DEBUG } 19:07:00 DEBUG ] 19:07:00 DEBUG } 19:07:00 DEBUG } 19:07:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462420" 19:07:00 INFO [http_cmd]: http status code OK 19:07:00 DEBUG --- status code --- 19:07:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:07:00 DEBUG --- http response --- 19:07:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:07:00 DEBUG ----- output ----- 19:07:00 DEBUG { 19:07:00 DEBUG "status": "success", 19:07:00 DEBUG "data": { 19:07:00 DEBUG "resultType": "vector", 19:07:00 DEBUG "result": [ 19:07:00 DEBUG { 19:07:00 DEBUG "metric": { 19:07:00 DEBUG "instance": "10.142.0.11:9100" 19:07:00 DEBUG }, 19:07:00 DEBUG "value": [ 19:07:00 DEBUG 19:07:00 DEBUG 19:07:00 DEBUG 19:07:00 DEBUG [...] 19:07:00 DEBUG 19:07:00 DEBUG 19:07:00 DEBUG 19:07:00 DEBUG "instance": "10.142.0.51:9100" 19:07:00 DEBUG }, 19:07:00 DEBUG "value": [ 19:07:00 DEBUG 1734462420, 19:07:00 DEBUG "0.0010999999999995683" 19:07:00 DEBUG ] 19:07:00 DEBUG } 19:07:00 DEBUG ] 19:07:00 DEBUG } 19:07:00 DEBUG } 19:07:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462420" 19:07:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462420" 19:07:00 INFO [http_cmd]: http status code OK 19:07:00 DEBUG --- status code --- 19:07:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:07:00 DEBUG --- http response --- 19:07:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:07:00 DEBUG ----- output ----- 19:07:00 DEBUG { 19:07:00 DEBUG "status": "success", 19:07:00 DEBUG "data": { 19:07:00 DEBUG "resultType": "vector", 19:07:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734462420" 19:07:00 DEBUG "result": [ 19:07:00 DEBUG { 19:07:00 DEBUG "metric": { 19:07:00 DEBUG "pod": "ds-cts-0" 19:07:00 DEBUG }, 19:07:00 DEBUG "value": [ 19:07:00 DEBUG 19:07:00 DEBUG 19:07:00 DEBUG 19:07:00 DEBUG [...] 19:07:00 DEBUG 19:07:00 DEBUG 19:07:00 DEBUG 19:07:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:07:00 DEBUG }, 19:07:00 DEBUG "value": [ 19:07:00 DEBUG 1734462420, 19:07:00 DEBUG "0" 19:07:00 DEBUG ] 19:07:00 DEBUG } 19:07:00 DEBUG ] 19:07:00 DEBUG } 19:07:00 DEBUG } 19:07:00 INFO [http_cmd]: http status code OK 19:07:00 DEBUG --- status code --- 19:07:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:07:00 DEBUG --- http response --- 19:07:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:07:00 DEBUG ----- output ----- 19:07:00 DEBUG { 19:07:00 DEBUG "status": "success", 19:07:00 DEBUG "data": { 19:07:00 DEBUG "resultType": "vector", 19:07:00 DEBUG "result": [ 19:07:00 DEBUG { 19:07:00 DEBUG "metric": { 19:07:00 DEBUG "pod": "ds-cts-0" 19:07:00 DEBUG }, 19:07:00 DEBUG "value": [ 19:07:00 DEBUG 19:07:00 DEBUG 19:07:00 DEBUG 19:07:00 DEBUG [...] 19:07:00 DEBUG 19:07:00 DEBUG 19:07:00 DEBUG 19:07:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:07:00 DEBUG }, 19:07:00 DEBUG "value": [ 19:07:00 DEBUG 1734462420, 19:07:00 DEBUG "1432.8260579933535" 19:07:00 DEBUG ] 19:07:00 DEBUG } 19:07:00 DEBUG ] 19:07:00 DEBUG } 19:07:00 DEBUG } 19:07:00 INFO [http_cmd]: http status code OK 19:07:00 DEBUG --- status code --- 19:07:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:07:00 DEBUG --- http response --- 19:07:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:07:00 DEBUG ----- output ----- 19:07:00 DEBUG { 19:07:00 DEBUG "status": "success", 19:07:00 DEBUG "data": { 19:07:00 DEBUG "resultType": "vector", 19:07:00 DEBUG "result": [ 19:07:00 DEBUG { 19:07:00 DEBUG "metric": { 19:07:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 19:07:00 DEBUG }, 19:07:00 DEBUG "value": [ 19:07:00 DEBUG 19:07:00 DEBUG 19:07:00 DEBUG 19:07:00 DEBUG [...] 19:07:00 DEBUG 19:07:00 DEBUG 19:07:00 DEBUG 19:07:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 19:07:00 DEBUG }, 19:07:00 DEBUG "value": [ 19:07:00 DEBUG 1734462420, 19:07:00 DEBUG "2577367040" 19:07:00 DEBUG ] 19:07:00 DEBUG } 19:07:00 DEBUG ] 19:07:00 DEBUG } 19:07:00 DEBUG } 19:07:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462420" 19:07:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462420" 19:07:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734462420" 19:07:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462420" 19:07:00 INFO [http_cmd]: http status code OK 19:07:00 DEBUG --- status code --- 19:07:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:07:00 DEBUG --- http response --- 19:07:00 INFO [http_cmd]: http status code OK 19:07:00 DEBUG --- status code --- 19:07:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:07:00 DEBUG --- http response --- 19:07:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:07:00 DEBUG ----- output ----- 19:07:00 DEBUG { 19:07:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:07:00 DEBUG "status": "success", 19:07:00 DEBUG "data": { 19:07:00 DEBUG ----- output ----- 19:07:00 INFO [http_cmd]: http status code OK 19:07:00 DEBUG "resultType": "vector", 19:07:00 DEBUG { 19:07:00 DEBUG --- status code --- 19:07:00 DEBUG "result": [ 19:07:00 DEBUG "status": "success", 19:07:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:07:00 DEBUG { 19:07:00 DEBUG "data": { 19:07:00 DEBUG --- http response --- 19:07:00 DEBUG "metric": { 19:07:00 DEBUG "resultType": "vector", 19:07:00 DEBUG "pod": "ds-cts-0" 19:07:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:07:00 DEBUG "result": [ 19:07:00 DEBUG }, 19:07:00 DEBUG ----- output ----- 19:07:00 DEBUG { 19:07:00 DEBUG "value": [ 19:07:00 DEBUG { 19:07:00 DEBUG "metric": { 19:07:00 DEBUG 19:07:00 DEBUG "status": "success", 19:07:00 DEBUG "instance": "10.142.0.11:9100" 19:07:00 DEBUG 19:07:00 DEBUG "data": { 19:07:00 DEBUG }, 19:07:00 DEBUG 19:07:00 DEBUG "resultType": "vector", 19:07:00 DEBUG "value": [ 19:07:00 DEBUG [...] 19:07:00 DEBUG "result": [ 19:07:00 DEBUG 19:07:00 DEBUG 19:07:00 DEBUG { 19:07:00 DEBUG 19:07:00 DEBUG 19:07:00 DEBUG 19:07:00 DEBUG 19:07:00 DEBUG "metric": { 19:07:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:07:00 DEBUG [...] 19:07:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 19:07:00 DEBUG }, 19:07:00 DEBUG 19:07:00 DEBUG 19:07:00 DEBUG "value": [ 19:07:00 DEBUG }, 19:07:00 DEBUG 19:07:00 DEBUG 1734462420, 19:07:00 INFO [http_cmd]: http status code OK 19:07:00 DEBUG "value": [ 19:07:00 DEBUG "instance": "10.142.0.51:9100" 19:07:00 DEBUG "695930880" 19:07:00 DEBUG --- status code --- 19:07:00 DEBUG 19:07:00 DEBUG }, 19:07:00 DEBUG ] 19:07:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:07:00 DEBUG 19:07:00 DEBUG "value": [ 19:07:00 DEBUG } 19:07:00 DEBUG --- http response --- 19:07:00 DEBUG 19:07:00 DEBUG 1734462420, 19:07:00 DEBUG ] 19:07:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:07:00 DEBUG [...] 19:07:00 DEBUG "21572.26666666667" 19:07:00 DEBUG } 19:07:00 DEBUG ----- output ----- 19:07:00 DEBUG 19:07:00 DEBUG ] 19:07:00 DEBUG } 19:07:00 DEBUG { 19:07:00 DEBUG 19:07:00 DEBUG } 19:07:00 DEBUG "status": "success", 19:07:00 DEBUG 19:07:00 DEBUG ] 19:07:00 DEBUG "data": { 19:07:00 DEBUG "resultType": "vector", 19:07:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 19:07:00 DEBUG } 19:07:00 DEBUG "result": [ 19:07:00 DEBUG }, 19:07:00 DEBUG } 19:07:00 DEBUG { 19:07:00 DEBUG "value": [ 19:07:00 DEBUG "metric": { 19:07:00 DEBUG 1734462420, 19:07:00 DEBUG "instance": "10.142.0.11:9100" 19:07:00 DEBUG "1.7534211478892239" 19:07:00 DEBUG }, 19:07:00 DEBUG ] 19:07:00 DEBUG "value": [ 19:07:00 DEBUG } 19:07:00 DEBUG 19:07:00 DEBUG ] 19:07:00 DEBUG 19:07:00 DEBUG } 19:07:00 DEBUG 19:07:00 DEBUG } 19:07:00 DEBUG [...] 19:07:00 DEBUG 19:07:00 DEBUG 19:07:00 DEBUG 19:07:00 DEBUG "instance": "10.142.0.51:9100" 19:07:00 DEBUG }, 19:07:00 DEBUG "value": [ 19:07:00 DEBUG 1734462420, 19:07:00 DEBUG "76694.30000000002" 19:07:00 DEBUG ] 19:07:00 DEBUG } 19:07:00 DEBUG ] 19:07:00 DEBUG } 19:07:00 DEBUG } 19:07:03 INFO 19:07:03 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 19:07:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:07:03 INFO [loop_until]: OK (rc = 0) 19:07:03 DEBUG --- stdout --- 19:07:03 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 544Mi ds-idrepo-0 7m 739Mi rcs-5d696ff5b-7ttvr 5m 297Mi rcs-5d696ff5b-wwlbr 3m 331Mi 19:07:03 DEBUG --- stderr --- 19:07:03 DEBUG 19:07:03 INFO 19:07:03 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 19:07:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:07:03 INFO [loop_until]: OK (rc = 0) 19:07:03 DEBUG --- stdout --- 19:07:03 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 99m 0% 2263Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1833m 5% 3652Mi 12% 19:07:03 DEBUG --- stderr --- 19:07:03 DEBUG 19:07:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462479" 19:07:59 INFO [http_cmd]: http status code OK 19:07:59 DEBUG --- status code --- 19:07:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:07:59 DEBUG --- http response --- 19:07:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:07:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734462479" 19:07:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462479" 19:07:59 INFO [http_cmd]: http status code OK 19:07:59 DEBUG --- status code --- 19:07:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:07:59 INFO [http_cmd]: http status code OK 19:07:59 DEBUG --- http response --- 19:07:59 DEBUG --- status code --- 19:07:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:07:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:07:59 DEBUG --- http response --- 19:07:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:07:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462479" 19:07:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462479" 19:07:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462479" 19:07:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462479" 19:07:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462479" 19:07:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462479" 19:07:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462479" 19:07:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462479" 19:07:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462479" 19:07:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462479" 19:07:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734462479" 19:07:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462479" 19:07:59 INFO [http_cmd]: http status code OK 19:07:59 DEBUG --- status code --- 19:07:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:07:59 DEBUG --- http response --- 19:07:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:07:59 INFO [http_cmd]: http status code OK 19:07:59 DEBUG --- status code --- 19:07:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:07:59 DEBUG --- http response --- 19:07:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:07:59 INFO [http_cmd]: http status code OK 19:07:59 DEBUG --- status code --- 19:07:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:07:59 DEBUG --- http response --- 19:07:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:07:59 INFO [http_cmd]: http status code OK 19:07:59 DEBUG --- status code --- 19:07:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:07:59 DEBUG --- http response --- 19:07:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:07:59 INFO [http_cmd]: http status code OK 19:07:59 DEBUG --- status code --- 19:07:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:07:59 DEBUG --- http response --- 19:07:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:07:59 INFO [http_cmd]: http status code OK 19:07:59 DEBUG --- status code --- 19:07:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:07:59 DEBUG --- http response --- 19:07:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:07:59 INFO [http_cmd]: http status code OK 19:07:59 DEBUG --- status code --- 19:07:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:07:59 INFO [http_cmd]: http status code OK 19:07:59 INFO [http_cmd]: http status code OK 19:07:59 DEBUG --- http response --- 19:07:59 INFO [http_cmd]: http status code OK 19:07:59 DEBUG --- status code --- 19:07:59 DEBUG --- status code --- 19:07:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:07:59 DEBUG --- status code --- 19:07:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:07:59 DEBUG --- http response --- 19:07:59 INFO [http_cmd]: http status code OK 19:07:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:07:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:07:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:07:59 ERROR [http_cmd]: ERROR 19:07:59 DEBUG --- status code --- 19:07:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:07:59 DEBUG --- http response --- 19:07:59 ERROR --- status code --- 19:07:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:07:59 DEBUG --- http response --- 19:07:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:07:59 ERROR --- http response --- 19:07:59 DEBUG --- http response --- 19:07:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:07:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:07:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:07:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:08:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462480" 19:08:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462480" 19:08:00 INFO [http_cmd]: http status code OK 19:08:00 DEBUG --- status code --- 19:08:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:08:00 DEBUG --- http response --- 19:08:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:08:00 INFO [http_cmd]: http status code OK 19:08:00 DEBUG --- status code --- 19:08:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:08:00 DEBUG --- http response --- 19:08:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:08:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462480" 19:08:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462480" 19:08:00 INFO [http_cmd]: http status code OK 19:08:00 INFO [http_cmd]: http status code OK 19:08:00 DEBUG --- status code --- 19:08:00 DEBUG --- status code --- 19:08:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:08:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:08:00 DEBUG --- http response --- 19:08:00 DEBUG --- http response --- 19:08:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:08:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:08:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462480" 19:08:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462480" 19:08:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734462480" 19:08:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734462480" 19:08:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462480" 19:08:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462480" 19:08:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734462480" 19:08:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462480" 19:08:00 INFO [http_cmd]: http status code OK 19:08:00 DEBUG --- status code --- 19:08:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:08:00 DEBUG --- http response --- 19:08:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:08:00 INFO [http_cmd]: http status code OK 19:08:00 DEBUG --- status code --- 19:08:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:08:00 DEBUG --- http response --- 19:08:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:08:00 INFO [http_cmd]: http status code OK 19:08:00 DEBUG --- status code --- 19:08:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:08:00 DEBUG --- http response --- 19:08:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:08:00 INFO [http_cmd]: http status code OK 19:08:00 DEBUG --- status code --- 19:08:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:08:00 DEBUG --- http response --- 19:08:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:08:00 INFO [http_cmd]: http status code OK 19:08:00 INFO [http_cmd]: http status code OK 19:08:00 DEBUG --- status code --- 19:08:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:08:00 DEBUG --- status code --- 19:08:00 INFO [http_cmd]: http status code OK 19:08:00 INFO [http_cmd]: http status code OK 19:08:00 DEBUG --- http response --- 19:08:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:08:00 DEBUG --- status code --- 19:08:00 DEBUG --- status code --- 19:08:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:08:00 DEBUG --- http response --- 19:08:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:08:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:08:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:08:00 DEBUG --- http response --- 19:08:00 DEBUG --- http response --- 19:08:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:08:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:08:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462480" 19:08:00 INFO [http_cmd]: http status code OK 19:08:00 DEBUG --- status code --- 19:08:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:08:00 DEBUG --- http response --- 19:08:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:08:00 DEBUG ----- output ----- 19:08:00 DEBUG { 19:08:00 DEBUG "status": "success", 19:08:00 DEBUG "data": { 19:08:00 DEBUG "resultType": "vector", 19:08:00 DEBUG "result": [ 19:08:00 DEBUG { 19:08:00 DEBUG "metric": { 19:08:00 DEBUG "pod": "ds-cts-0" 19:08:00 DEBUG }, 19:08:00 DEBUG "value": [ 19:08:00 DEBUG 19:08:00 DEBUG 19:08:00 DEBUG 19:08:00 DEBUG [...] 19:08:00 DEBUG 19:08:00 DEBUG 19:08:00 DEBUG 19:08:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:08:00 DEBUG }, 19:08:00 DEBUG "value": [ 19:08:00 DEBUG 1734462480, 19:08:00 DEBUG "0.004606528993281749" 19:08:00 DEBUG ] 19:08:00 DEBUG } 19:08:00 DEBUG ] 19:08:00 DEBUG } 19:08:00 DEBUG } 19:08:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462480" 19:08:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462480" 19:08:00 INFO [http_cmd]: http status code OK 19:08:00 DEBUG --- status code --- 19:08:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:08:00 DEBUG --- http response --- 19:08:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:08:00 DEBUG ----- output ----- 19:08:00 DEBUG { 19:08:00 DEBUG "status": "success", 19:08:00 DEBUG "data": { 19:08:00 DEBUG "resultType": "vector", 19:08:00 DEBUG "result": [ 19:08:00 DEBUG { 19:08:00 DEBUG "metric": { 19:08:00 DEBUG "instance": "10.142.0.11:9100" 19:08:00 DEBUG }, 19:08:00 DEBUG "value": [ 19:08:00 DEBUG 19:08:00 DEBUG 19:08:00 DEBUG 19:08:00 DEBUG [...] 19:08:00 DEBUG 19:08:00 DEBUG 19:08:00 DEBUG 19:08:00 DEBUG "instance": "10.142.0.51:9100" 19:08:00 DEBUG }, 19:08:00 DEBUG "value": [ 19:08:00 DEBUG 1734462480, 19:08:00 DEBUG "87655.53333333335" 19:08:00 DEBUG ] 19:08:00 DEBUG } 19:08:00 DEBUG ] 19:08:00 DEBUG } 19:08:00 DEBUG } 19:08:00 INFO [http_cmd]: http status code OK 19:08:00 DEBUG --- status code --- 19:08:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:08:00 DEBUG --- http response --- 19:08:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:08:00 DEBUG ----- output ----- 19:08:00 DEBUG { 19:08:00 DEBUG "status": "success", 19:08:00 DEBUG "data": { 19:08:00 DEBUG "resultType": "vector", 19:08:00 DEBUG "result": [ 19:08:00 DEBUG { 19:08:00 DEBUG "metric": { 19:08:00 DEBUG "instance": "10.142.0.11:9100" 19:08:00 DEBUG }, 19:08:00 DEBUG "value": [ 19:08:00 DEBUG 19:08:00 DEBUG 19:08:00 DEBUG 19:08:00 DEBUG [...] 19:08:00 DEBUG 19:08:00 DEBUG 19:08:00 DEBUG 19:08:00 DEBUG "instance": "10.142.0.51:9100" 19:08:00 DEBUG }, 19:08:00 DEBUG "value": [ 19:08:00 DEBUG 1734462480, 19:08:00 DEBUG "0" 19:08:00 DEBUG ] 19:08:00 DEBUG } 19:08:00 DEBUG ] 19:08:00 DEBUG } 19:08:00 DEBUG } 19:08:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462480" 19:08:00 INFO [http_cmd]: http status code OK 19:08:00 DEBUG --- status code --- 19:08:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:08:00 DEBUG --- http response --- 19:08:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:08:00 DEBUG ----- output ----- 19:08:00 DEBUG { 19:08:00 DEBUG "status": "success", 19:08:00 DEBUG "data": { 19:08:00 DEBUG "resultType": "vector", 19:08:00 DEBUG "result": [ 19:08:00 DEBUG { 19:08:00 DEBUG "metric": { 19:08:00 DEBUG "instance": "10.142.0.11:9100" 19:08:00 DEBUG }, 19:08:00 DEBUG "value": [ 19:08:00 DEBUG 19:08:00 DEBUG 19:08:00 DEBUG 19:08:00 DEBUG [...] 19:08:00 DEBUG 19:08:00 DEBUG 19:08:00 DEBUG 19:08:00 DEBUG "instance": "10.142.0.51:9100" 19:08:00 DEBUG }, 19:08:00 DEBUG "value": [ 19:08:00 DEBUG 1734462480, 19:08:00 DEBUG "0.0005333333333316167" 19:08:00 DEBUG ] 19:08:00 DEBUG } 19:08:00 DEBUG ] 19:08:00 DEBUG } 19:08:00 DEBUG } 19:08:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462480" 19:08:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462480" 19:08:00 INFO [http_cmd]: http status code OK 19:08:00 DEBUG --- status code --- 19:08:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:08:00 DEBUG --- http response --- 19:08:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734462480" 19:08:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:08:00 DEBUG ----- output ----- 19:08:00 DEBUG { 19:08:00 DEBUG "status": "success", 19:08:00 DEBUG "data": { 19:08:00 DEBUG "resultType": "vector", 19:08:00 DEBUG "result": [ 19:08:00 DEBUG { 19:08:00 DEBUG "metric": { 19:08:00 DEBUG "pod": "ds-cts-0" 19:08:00 DEBUG }, 19:08:00 DEBUG "value": [ 19:08:00 DEBUG 19:08:00 DEBUG 19:08:00 DEBUG 19:08:00 DEBUG [...] 19:08:00 DEBUG 19:08:00 DEBUG 19:08:00 DEBUG 19:08:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:08:00 DEBUG }, 19:08:00 DEBUG "value": [ 19:08:00 DEBUG 1734462480, 19:08:00 DEBUG "0" 19:08:00 DEBUG ] 19:08:00 DEBUG } 19:08:00 DEBUG ] 19:08:00 DEBUG } 19:08:00 DEBUG } 19:08:00 INFO [http_cmd]: http status code OK 19:08:00 DEBUG --- status code --- 19:08:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:08:00 DEBUG --- http response --- 19:08:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:08:00 DEBUG ----- output ----- 19:08:00 DEBUG { 19:08:00 DEBUG "status": "success", 19:08:00 DEBUG "data": { 19:08:00 DEBUG "resultType": "vector", 19:08:00 DEBUG "result": [ 19:08:00 DEBUG { 19:08:00 DEBUG "metric": { 19:08:00 DEBUG "pod": "ds-cts-0" 19:08:00 DEBUG }, 19:08:00 DEBUG "value": [ 19:08:00 DEBUG 19:08:00 DEBUG 19:08:00 DEBUG 19:08:00 DEBUG [...] 19:08:00 DEBUG 19:08:00 DEBUG 19:08:00 DEBUG 19:08:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:08:00 DEBUG }, 19:08:00 DEBUG "value": [ 19:08:00 DEBUG 1734462480, 19:08:00 DEBUG "1125.7635050172735" 19:08:00 DEBUG ] 19:08:00 DEBUG } 19:08:00 DEBUG ] 19:08:00 DEBUG } 19:08:00 DEBUG } 19:08:00 INFO [http_cmd]: http status code OK 19:08:00 DEBUG --- status code --- 19:08:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:08:00 DEBUG --- http response --- 19:08:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:08:00 DEBUG ----- output ----- 19:08:00 DEBUG { 19:08:00 DEBUG "status": "success", 19:08:00 DEBUG "data": { 19:08:00 DEBUG "resultType": "vector", 19:08:00 DEBUG "result": [ 19:08:00 DEBUG { 19:08:00 DEBUG "metric": { 19:08:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 19:08:00 DEBUG }, 19:08:00 DEBUG "value": [ 19:08:00 DEBUG 19:08:00 DEBUG 19:08:00 DEBUG 19:08:00 DEBUG [...] 19:08:00 DEBUG 19:08:00 DEBUG 19:08:00 DEBUG 19:08:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 19:08:00 DEBUG }, 19:08:00 DEBUG "value": [ 19:08:00 DEBUG 1734462480, 19:08:00 DEBUG "2566070272" 19:08:00 DEBUG ] 19:08:00 DEBUG } 19:08:00 DEBUG ] 19:08:00 DEBUG } 19:08:00 DEBUG } 19:08:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462480" 19:08:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462480" 19:08:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734462480" 19:08:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462480" 19:08:00 INFO [http_cmd]: http status code OK 19:08:00 DEBUG --- status code --- 19:08:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:08:00 DEBUG --- http response --- 19:08:00 INFO [http_cmd]: http status code OK 19:08:00 DEBUG --- status code --- 19:08:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:08:00 DEBUG --- http response --- 19:08:00 INFO [http_cmd]: http status code OK 19:08:00 DEBUG --- status code --- 19:08:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:08:00 DEBUG --- http response --- 19:08:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:08:00 DEBUG ----- output ----- 19:08:00 DEBUG { 19:08:00 DEBUG "status": "success", 19:08:00 DEBUG "data": { 19:08:00 DEBUG "resultType": "vector", 19:08:00 DEBUG "result": [ 19:08:00 DEBUG { 19:08:00 DEBUG "metric": { 19:08:00 DEBUG "instance": "10.142.0.11:9100" 19:08:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:08:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:08:00 INFO [http_cmd]: http status code OK 19:08:00 DEBUG }, 19:08:00 DEBUG ----- output ----- 19:08:00 DEBUG ----- output ----- 19:08:00 DEBUG --- status code --- 19:08:00 DEBUG "value": [ 19:08:00 DEBUG { 19:08:00 DEBUG { 19:08:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:08:00 DEBUG 19:08:00 DEBUG "status": "success", 19:08:00 DEBUG "status": "success", 19:08:00 DEBUG --- http response --- 19:08:00 DEBUG 19:08:00 DEBUG "data": { 19:08:00 DEBUG "data": { 19:08:00 DEBUG 19:08:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:08:00 DEBUG "resultType": "vector", 19:08:00 DEBUG "resultType": "vector", 19:08:00 DEBUG [...] 19:08:00 DEBUG ----- output ----- 19:08:00 DEBUG "result": [ 19:08:00 DEBUG "result": [ 19:08:00 DEBUG 19:08:00 DEBUG { 19:08:00 DEBUG { 19:08:00 DEBUG { 19:08:00 DEBUG 19:08:00 DEBUG "status": "success", 19:08:00 DEBUG "metric": { 19:08:00 DEBUG "metric": { 19:08:00 DEBUG 19:08:00 DEBUG "data": { 19:08:00 DEBUG "instance": "10.142.0.11:9100" 19:08:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 19:08:00 DEBUG "instance": "10.142.0.51:9100" 19:08:00 DEBUG "resultType": "vector", 19:08:00 DEBUG }, 19:08:00 DEBUG }, 19:08:00 DEBUG }, 19:08:00 DEBUG "value": [ 19:08:00 DEBUG "value": [ 19:08:00 DEBUG "value": [ 19:08:00 DEBUG "result": [ 19:08:00 DEBUG 1734462480, 19:08:00 DEBUG 19:08:00 DEBUG 19:08:00 DEBUG { 19:08:00 DEBUG "71122.40000000004" 19:08:00 DEBUG 19:08:00 DEBUG 19:08:00 DEBUG "metric": { 19:08:00 DEBUG ] 19:08:00 DEBUG 19:08:00 DEBUG 19:08:00 DEBUG "pod": "ds-cts-0" 19:08:00 DEBUG } 19:08:00 DEBUG [...] 19:08:00 DEBUG [...] 19:08:00 DEBUG }, 19:08:00 DEBUG ] 19:08:00 DEBUG 19:08:00 DEBUG 19:08:00 DEBUG "value": [ 19:08:00 DEBUG } 19:08:00 DEBUG 19:08:00 DEBUG 19:08:00 DEBUG 19:08:00 DEBUG } 19:08:00 DEBUG 19:08:00 DEBUG "instance": "10.142.0.51:9100" 19:08:00 DEBUG 19:08:00 DEBUG 19:08:00 DEBUG }, 19:08:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 19:08:00 DEBUG 19:08:00 DEBUG "value": [ 19:08:00 DEBUG }, 19:08:00 DEBUG [...] 19:08:00 DEBUG 1734462480, 19:08:00 DEBUG "value": [ 19:08:00 DEBUG 19:08:00 DEBUG "23893.33333333334" 19:08:00 DEBUG 1734462480, 19:08:00 DEBUG 19:08:00 DEBUG ] 19:08:00 DEBUG "1.7463719784298333" 19:08:00 DEBUG 19:08:00 DEBUG } 19:08:00 DEBUG ] 19:08:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:08:00 DEBUG ] 19:08:00 DEBUG } 19:08:00 DEBUG }, 19:08:00 DEBUG } 19:08:00 DEBUG ] 19:08:00 DEBUG "value": [ 19:08:00 DEBUG } 19:08:00 DEBUG } 19:08:00 DEBUG 1734462480, 19:08:00 DEBUG } 19:08:00 DEBUG "695996416" 19:08:00 DEBUG ] 19:08:00 DEBUG } 19:08:00 DEBUG ] 19:08:00 DEBUG } 19:08:00 DEBUG } 19:08:03 INFO 19:08:03 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 19:08:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:08:03 INFO [loop_until]: OK (rc = 0) 19:08:03 DEBUG --- stdout --- 19:08:03 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 545Mi ds-idrepo-0 7m 739Mi rcs-5d696ff5b-7ttvr 4m 297Mi rcs-5d696ff5b-wwlbr 6m 331Mi 19:08:03 DEBUG --- stderr --- 19:08:03 DEBUG 19:08:03 INFO 19:08:03 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 19:08:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:08:03 INFO [loop_until]: OK (rc = 0) 19:08:03 DEBUG --- stdout --- 19:08:03 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 101m 0% 2269Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1821m 5% 3657Mi 12% 19:08:03 DEBUG --- stderr --- 19:08:03 DEBUG 19:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462539" 19:08:59 INFO [http_cmd]: http status code OK 19:08:59 DEBUG --- status code --- 19:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:08:59 DEBUG --- http response --- 19:08:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462539" 19:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734462539" 19:08:59 INFO [http_cmd]: http status code OK 19:08:59 DEBUG --- status code --- 19:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:08:59 DEBUG --- http response --- 19:08:59 INFO [http_cmd]: http status code OK 19:08:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:08:59 DEBUG --- status code --- 19:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:08:59 DEBUG --- http response --- 19:08:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462539" 19:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462539" 19:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462539" 19:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462539" 19:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462539" 19:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462539" 19:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462539" 19:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462539" 19:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462539" 19:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462539" 19:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462539" 19:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734462539" 19:08:59 INFO [http_cmd]: http status code OK 19:08:59 DEBUG --- status code --- 19:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:08:59 DEBUG --- http response --- 19:08:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:08:59 INFO [http_cmd]: http status code OK 19:08:59 INFO [http_cmd]: http status code OK 19:08:59 INFO [http_cmd]: http status code OK 19:08:59 INFO [http_cmd]: http status code OK 19:08:59 INFO [http_cmd]: http status code OK 19:08:59 INFO [http_cmd]: http status code OK 19:08:59 INFO [http_cmd]: http status code OK 19:08:59 INFO [http_cmd]: http status code OK 19:08:59 DEBUG --- status code --- 19:08:59 DEBUG --- status code --- 19:08:59 DEBUG --- status code --- 19:08:59 DEBUG --- status code --- 19:08:59 INFO [http_cmd]: http status code OK 19:08:59 DEBUG --- status code --- 19:08:59 INFO [http_cmd]: http status code OK 19:08:59 DEBUG --- status code --- 19:08:59 DEBUG --- status code --- 19:08:59 DEBUG --- status code --- 19:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:08:59 DEBUG --- status code --- 19:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:08:59 DEBUG --- status code --- 19:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:08:59 DEBUG --- http response --- 19:08:59 DEBUG --- http response --- 19:08:59 DEBUG --- http response --- 19:08:59 DEBUG --- http response --- 19:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:08:59 DEBUG --- http response --- 19:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:08:59 DEBUG --- http response --- 19:08:59 DEBUG --- http response --- 19:08:59 DEBUG --- http response --- 19:08:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:08:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:08:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:08:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:08:59 DEBUG --- http response --- 19:08:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:08:59 DEBUG --- http response --- 19:08:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:08:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:08:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:08:59 ERROR [http_cmd]: ERROR 19:08:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:08:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:08:59 ERROR --- status code --- 19:08:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:08:59 ERROR --- http response --- 19:08:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:08:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:09:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462540" 19:09:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462540" 19:09:00 INFO [http_cmd]: http status code OK 19:09:00 DEBUG --- status code --- 19:09:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:09:00 INFO [http_cmd]: http status code OK 19:09:00 DEBUG --- http response --- 19:09:00 DEBUG --- status code --- 19:09:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:09:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:09:00 DEBUG --- http response --- 19:09:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:09:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462540" 19:09:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462540" 19:09:00 INFO [http_cmd]: http status code OK 19:09:00 INFO [http_cmd]: http status code OK 19:09:00 DEBUG --- status code --- 19:09:00 DEBUG --- status code --- 19:09:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:09:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:09:00 DEBUG --- http response --- 19:09:00 DEBUG --- http response --- 19:09:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:09:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:09:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462540" 19:09:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462540" 19:09:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734462540" 19:09:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734462540" 19:09:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462540" 19:09:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462540" 19:09:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734462540" 19:09:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462540" 19:09:00 INFO [http_cmd]: http status code OK 19:09:00 DEBUG --- status code --- 19:09:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:09:00 DEBUG --- http response --- 19:09:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:09:00 INFO [http_cmd]: http status code OK 19:09:00 DEBUG --- status code --- 19:09:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:09:00 DEBUG --- http response --- 19:09:00 INFO [http_cmd]: http status code OK 19:09:00 DEBUG --- status code --- 19:09:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:09:00 DEBUG --- http response --- 19:09:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:09:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:09:00 INFO [http_cmd]: http status code OK 19:09:00 DEBUG --- status code --- 19:09:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:09:00 DEBUG --- http response --- 19:09:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:09:00 INFO [http_cmd]: http status code OK 19:09:00 DEBUG --- status code --- 19:09:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:09:00 DEBUG --- http response --- 19:09:00 INFO [http_cmd]: http status code OK 19:09:00 DEBUG --- status code --- 19:09:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:09:00 DEBUG --- http response --- 19:09:00 INFO [http_cmd]: http status code OK 19:09:00 INFO [http_cmd]: http status code OK 19:09:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:09:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:09:00 DEBUG --- status code --- 19:09:00 DEBUG --- status code --- 19:09:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:09:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:09:00 DEBUG --- http response --- 19:09:00 DEBUG --- http response --- 19:09:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:09:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:09:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462540" 19:09:00 INFO [http_cmd]: http status code OK 19:09:00 DEBUG --- status code --- 19:09:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:09:00 DEBUG --- http response --- 19:09:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:09:00 DEBUG ----- output ----- 19:09:00 DEBUG { 19:09:00 DEBUG "status": "success", 19:09:00 DEBUG "data": { 19:09:00 DEBUG "resultType": "vector", 19:09:00 DEBUG "result": [ 19:09:00 DEBUG { 19:09:00 DEBUG "metric": { 19:09:00 DEBUG "pod": "ds-cts-0" 19:09:00 DEBUG }, 19:09:00 DEBUG "value": [ 19:09:00 DEBUG 19:09:00 DEBUG 19:09:00 DEBUG 19:09:00 DEBUG [...] 19:09:00 DEBUG 19:09:00 DEBUG 19:09:00 DEBUG 19:09:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:09:00 DEBUG }, 19:09:00 DEBUG "value": [ 19:09:00 DEBUG 1734462540, 19:09:00 DEBUG "0.002761093158376091" 19:09:00 DEBUG ] 19:09:00 DEBUG } 19:09:00 DEBUG ] 19:09:00 DEBUG } 19:09:00 DEBUG } 19:09:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462540" 19:09:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462540" 19:09:00 INFO [http_cmd]: http status code OK 19:09:00 DEBUG --- status code --- 19:09:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:09:00 DEBUG --- http response --- 19:09:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:09:00 DEBUG ----- output ----- 19:09:00 DEBUG { 19:09:00 DEBUG "status": "success", 19:09:00 DEBUG "data": { 19:09:00 DEBUG "resultType": "vector", 19:09:00 DEBUG "result": [ 19:09:00 DEBUG { 19:09:00 DEBUG "metric": { 19:09:00 DEBUG "instance": "10.142.0.11:9100" 19:09:00 DEBUG }, 19:09:00 DEBUG "value": [ 19:09:00 DEBUG 19:09:00 DEBUG 19:09:00 DEBUG 19:09:00 DEBUG [...] 19:09:00 DEBUG 19:09:00 DEBUG 19:09:00 DEBUG 19:09:00 DEBUG "instance": "10.142.0.51:9100" 19:09:00 DEBUG }, 19:09:00 DEBUG "value": [ 19:09:00 DEBUG 1734462540, 19:09:00 DEBUG "90193.66666666669" 19:09:00 DEBUG ] 19:09:00 DEBUG } 19:09:00 DEBUG ] 19:09:00 DEBUG } 19:09:00 DEBUG } 19:09:00 INFO [http_cmd]: http status code OK 19:09:00 DEBUG --- status code --- 19:09:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:09:00 DEBUG --- http response --- 19:09:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:09:00 DEBUG ----- output ----- 19:09:00 DEBUG { 19:09:00 DEBUG "status": "success", 19:09:00 DEBUG "data": { 19:09:00 DEBUG "resultType": "vector", 19:09:00 DEBUG "result": [ 19:09:00 DEBUG { 19:09:00 DEBUG "metric": { 19:09:00 DEBUG "instance": "10.142.0.11:9100" 19:09:00 DEBUG }, 19:09:00 DEBUG "value": [ 19:09:00 DEBUG 19:09:00 DEBUG 19:09:00 DEBUG 19:09:00 DEBUG [...] 19:09:00 DEBUG 19:09:00 DEBUG 19:09:00 DEBUG 19:09:00 DEBUG "instance": "10.142.0.51:9100" 19:09:00 DEBUG }, 19:09:00 DEBUG "value": [ 19:09:00 DEBUG 1734462540, 19:09:00 DEBUG "0" 19:09:00 DEBUG ] 19:09:00 DEBUG } 19:09:00 DEBUG ] 19:09:00 DEBUG } 19:09:00 DEBUG } 19:09:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462540" 19:09:00 INFO [http_cmd]: http status code OK 19:09:00 DEBUG --- status code --- 19:09:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:09:00 DEBUG --- http response --- 19:09:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:09:00 DEBUG ----- output ----- 19:09:00 DEBUG { 19:09:00 DEBUG "status": "success", 19:09:00 DEBUG "data": { 19:09:00 DEBUG "resultType": "vector", 19:09:00 DEBUG "result": [ 19:09:00 DEBUG { 19:09:00 DEBUG "metric": { 19:09:00 DEBUG "instance": "10.142.0.11:9100" 19:09:00 DEBUG }, 19:09:00 DEBUG "value": [ 19:09:00 DEBUG 19:09:00 DEBUG 19:09:00 DEBUG 19:09:00 DEBUG [...] 19:09:00 DEBUG 19:09:00 DEBUG 19:09:00 DEBUG 19:09:00 DEBUG "instance": "10.142.0.51:9100" 19:09:00 DEBUG }, 19:09:00 DEBUG "value": [ 19:09:00 DEBUG 1734462540, 19:09:00 DEBUG "0.00040000000000048894" 19:09:00 DEBUG ] 19:09:00 DEBUG } 19:09:00 DEBUG ] 19:09:00 DEBUG } 19:09:00 DEBUG } 19:09:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462540" 19:09:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462540" 19:09:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734462540" 19:09:00 INFO [http_cmd]: http status code OK 19:09:00 DEBUG --- status code --- 19:09:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:09:00 DEBUG --- http response --- 19:09:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:09:00 DEBUG ----- output ----- 19:09:00 DEBUG { 19:09:00 DEBUG "status": "success", 19:09:00 DEBUG "data": { 19:09:00 DEBUG "resultType": "vector", 19:09:00 DEBUG "result": [ 19:09:00 DEBUG { 19:09:00 DEBUG "metric": { 19:09:00 DEBUG "pod": "ds-cts-0" 19:09:00 DEBUG }, 19:09:00 DEBUG "value": [ 19:09:00 DEBUG 19:09:00 DEBUG 19:09:00 DEBUG 19:09:00 DEBUG [...] 19:09:00 DEBUG 19:09:00 DEBUG 19:09:00 DEBUG 19:09:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:09:00 DEBUG }, 19:09:00 DEBUG "value": [ 19:09:00 DEBUG 1734462540, 19:09:00 DEBUG "0" 19:09:00 DEBUG ] 19:09:00 DEBUG } 19:09:00 DEBUG ] 19:09:00 DEBUG } 19:09:00 DEBUG } 19:09:00 INFO [http_cmd]: http status code OK 19:09:00 DEBUG --- status code --- 19:09:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:09:00 DEBUG --- http response --- 19:09:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:09:00 DEBUG ----- output ----- 19:09:00 DEBUG { 19:09:00 DEBUG "status": "success", 19:09:00 DEBUG "data": { 19:09:00 DEBUG "resultType": "vector", 19:09:00 DEBUG "result": [ 19:09:00 DEBUG { 19:09:00 DEBUG "metric": { 19:09:00 DEBUG "pod": "ds-cts-0" 19:09:00 DEBUG }, 19:09:00 DEBUG "value": [ 19:09:00 DEBUG 19:09:00 DEBUG 19:09:00 DEBUG 19:09:00 DEBUG [...] 19:09:00 DEBUG 19:09:00 DEBUG 19:09:00 DEBUG 19:09:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:09:00 DEBUG }, 19:09:00 DEBUG "value": [ 19:09:00 DEBUG 1734462540, 19:09:00 DEBUG "921.1261762965139" 19:09:00 DEBUG ] 19:09:00 DEBUG } 19:09:00 DEBUG ] 19:09:00 DEBUG } 19:09:00 DEBUG } 19:09:00 INFO [http_cmd]: http status code OK 19:09:00 DEBUG --- status code --- 19:09:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:09:00 DEBUG --- http response --- 19:09:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:09:00 DEBUG ----- output ----- 19:09:00 DEBUG { 19:09:00 DEBUG "status": "success", 19:09:00 DEBUG "data": { 19:09:00 DEBUG "resultType": "vector", 19:09:00 DEBUG "result": [ 19:09:00 DEBUG { 19:09:00 DEBUG "metric": { 19:09:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 19:09:00 DEBUG }, 19:09:00 DEBUG "value": [ 19:09:00 DEBUG 19:09:00 DEBUG 19:09:00 DEBUG 19:09:00 DEBUG [...] 19:09:00 DEBUG 19:09:00 DEBUG 19:09:00 DEBUG 19:09:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 19:09:00 DEBUG }, 19:09:00 DEBUG "value": [ 19:09:00 DEBUG 1734462540, 19:09:00 DEBUG "2554724352" 19:09:00 DEBUG ] 19:09:00 DEBUG } 19:09:00 DEBUG ] 19:09:00 DEBUG } 19:09:00 DEBUG } 19:09:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462540" 19:09:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462540" 19:09:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734462540" 19:09:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462540" 19:09:00 INFO [http_cmd]: http status code OK 19:09:00 DEBUG --- status code --- 19:09:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:09:00 DEBUG --- http response --- 19:09:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:09:00 DEBUG ----- output ----- 19:09:00 DEBUG { 19:09:00 INFO [http_cmd]: http status code OK 19:09:00 INFO [http_cmd]: http status code OK 19:09:00 DEBUG "status": "success", 19:09:00 DEBUG --- status code --- 19:09:00 DEBUG --- status code --- 19:09:00 DEBUG "data": { 19:09:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:09:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:09:00 DEBUG "resultType": "vector", 19:09:00 DEBUG --- http response --- 19:09:00 DEBUG --- http response --- 19:09:00 DEBUG "result": [ 19:09:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:09:00 DEBUG { 19:09:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:09:00 INFO [http_cmd]: http status code OK 19:09:00 DEBUG ----- output ----- 19:09:00 DEBUG "metric": { 19:09:00 DEBUG ----- output ----- 19:09:00 DEBUG --- status code --- 19:09:00 DEBUG { 19:09:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 19:09:00 DEBUG { 19:09:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:09:00 DEBUG "status": "success", 19:09:00 DEBUG }, 19:09:00 DEBUG "status": "success", 19:09:00 DEBUG --- http response --- 19:09:00 DEBUG "data": { 19:09:00 DEBUG "value": [ 19:09:00 DEBUG "data": { 19:09:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:09:00 DEBUG "resultType": "vector", 19:09:00 DEBUG 19:09:00 DEBUG "resultType": "vector", 19:09:00 DEBUG ----- output ----- 19:09:00 DEBUG "result": [ 19:09:00 DEBUG 19:09:00 DEBUG "result": [ 19:09:00 DEBUG { 19:09:00 DEBUG { 19:09:00 DEBUG 19:09:00 DEBUG { 19:09:00 DEBUG "status": "success", 19:09:00 DEBUG "metric": { 19:09:00 DEBUG [...] 19:09:00 DEBUG "metric": { 19:09:00 DEBUG "data": { 19:09:00 DEBUG "instance": "10.142.0.11:9100" 19:09:00 DEBUG 19:09:00 DEBUG "instance": "10.142.0.11:9100" 19:09:00 DEBUG "resultType": "vector", 19:09:00 DEBUG }, 19:09:00 DEBUG 19:09:00 DEBUG }, 19:09:00 DEBUG "result": [ 19:09:00 DEBUG "value": [ 19:09:00 DEBUG 19:09:00 DEBUG "value": [ 19:09:00 DEBUG { 19:09:00 DEBUG 19:09:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 19:09:00 DEBUG 19:09:00 DEBUG "metric": { 19:09:00 DEBUG 19:09:00 DEBUG }, 19:09:00 DEBUG "value": [ 19:09:00 DEBUG 19:09:00 DEBUG 19:09:00 DEBUG "pod": "ds-cts-0" 19:09:00 DEBUG 1734462540, 19:09:00 DEBUG 19:09:00 DEBUG [...] 19:09:00 DEBUG }, 19:09:00 DEBUG "1.7497236736501132" 19:09:00 DEBUG [...] 19:09:00 DEBUG 19:09:00 DEBUG "value": [ 19:09:00 DEBUG ] 19:09:00 DEBUG 19:09:00 DEBUG 19:09:00 DEBUG 19:09:00 DEBUG } 19:09:00 DEBUG 19:09:00 DEBUG 19:09:00 DEBUG 19:09:00 DEBUG ] 19:09:00 DEBUG 19:09:00 DEBUG "instance": "10.142.0.51:9100" 19:09:00 DEBUG 19:09:00 DEBUG } 19:09:00 DEBUG "instance": "10.142.0.51:9100" 19:09:00 DEBUG }, 19:09:00 DEBUG [...] 19:09:00 DEBUG } 19:09:00 DEBUG }, 19:09:00 DEBUG "value": [ 19:09:00 DEBUG 19:09:00 DEBUG "value": [ 19:09:00 DEBUG 1734462540, 19:09:00 DEBUG 19:09:00 DEBUG 1734462540, 19:09:00 DEBUG "13653.333333333336" 19:09:00 DEBUG 19:09:00 DEBUG "76637.56666666668" 19:09:00 DEBUG ] 19:09:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:09:00 DEBUG ] 19:09:00 DEBUG } 19:09:00 DEBUG }, 19:09:00 DEBUG } 19:09:00 DEBUG ] 19:09:00 DEBUG "value": [ 19:09:00 DEBUG ] 19:09:00 DEBUG } 19:09:00 DEBUG 1734462540, 19:09:00 DEBUG } 19:09:00 DEBUG } 19:09:00 DEBUG "696029184" 19:09:00 DEBUG } 19:09:00 DEBUG ] 19:09:00 DEBUG } 19:09:00 DEBUG ] 19:09:00 DEBUG } 19:09:00 DEBUG } 19:09:03 INFO 19:09:03 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 19:09:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:09:03 INFO [loop_until]: OK (rc = 0) 19:09:03 DEBUG --- stdout --- 19:09:03 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 546Mi ds-idrepo-0 7m 740Mi rcs-5d696ff5b-7ttvr 2m 297Mi rcs-5d696ff5b-wwlbr 2m 331Mi 19:09:03 DEBUG --- stderr --- 19:09:03 DEBUG 19:09:03 INFO 19:09:03 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 19:09:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:09:03 INFO [loop_until]: OK (rc = 0) 19:09:03 DEBUG --- stdout --- 19:09:03 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 102m 0% 2271Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1820m 5% 3663Mi 12% 19:09:03 DEBUG --- stderr --- 19:09:03 DEBUG 19:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462599" 19:09:59 INFO [http_cmd]: http status code OK 19:09:59 DEBUG --- status code --- 19:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:09:59 DEBUG --- http response --- 19:09:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734462599" 19:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462599" 19:09:59 INFO [http_cmd]: http status code OK 19:09:59 INFO [http_cmd]: http status code OK 19:09:59 DEBUG --- status code --- 19:09:59 DEBUG --- status code --- 19:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:09:59 DEBUG --- http response --- 19:09:59 DEBUG --- http response --- 19:09:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:09:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462599" 19:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462599" 19:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462599" 19:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462599" 19:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462599" 19:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462599" 19:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462599" 19:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734462599" 19:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462599" 19:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462599" 19:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462599" 19:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462599" 19:09:59 INFO [http_cmd]: http status code OK 19:09:59 DEBUG --- status code --- 19:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:09:59 DEBUG --- http response --- 19:09:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:09:59 INFO [http_cmd]: http status code OK 19:09:59 DEBUG --- status code --- 19:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:09:59 DEBUG --- http response --- 19:09:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:09:59 INFO [http_cmd]: http status code OK 19:09:59 DEBUG --- status code --- 19:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:09:59 DEBUG --- http response --- 19:09:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:09:59 INFO [http_cmd]: http status code OK 19:09:59 DEBUG --- status code --- 19:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:09:59 DEBUG --- http response --- 19:09:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:09:59 INFO [http_cmd]: http status code OK 19:09:59 DEBUG --- status code --- 19:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:09:59 DEBUG --- http response --- 19:09:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:09:59 INFO [http_cmd]: http status code OK 19:09:59 DEBUG --- status code --- 19:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:09:59 DEBUG --- http response --- 19:09:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:09:59 INFO [http_cmd]: http status code OK 19:09:59 DEBUG --- status code --- 19:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:09:59 DEBUG --- http response --- 19:09:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:09:59 ERROR [http_cmd]: ERROR 19:09:59 ERROR --- status code --- 19:09:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:09:59 ERROR --- http response --- 19:09:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:09:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:09:59 INFO [http_cmd]: http status code OK 19:09:59 DEBUG --- status code --- 19:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:09:59 DEBUG --- http response --- 19:09:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:09:59 INFO [http_cmd]: http status code OK 19:09:59 DEBUG --- status code --- 19:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:09:59 DEBUG --- http response --- 19:09:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:09:59 INFO [http_cmd]: http status code OK 19:09:59 INFO [http_cmd]: http status code OK 19:09:59 DEBUG --- status code --- 19:09:59 DEBUG --- status code --- 19:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:09:59 DEBUG --- http response --- 19:09:59 DEBUG --- http response --- 19:09:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:09:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:10:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462600" 19:10:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462600" 19:10:00 INFO [http_cmd]: http status code OK 19:10:00 DEBUG --- status code --- 19:10:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:10:00 DEBUG --- http response --- 19:10:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:10:00 INFO [http_cmd]: http status code OK 19:10:00 DEBUG --- status code --- 19:10:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:10:00 DEBUG --- http response --- 19:10:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:10:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462600" 19:10:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462600" 19:10:00 INFO [http_cmd]: http status code OK 19:10:00 DEBUG --- status code --- 19:10:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:10:00 DEBUG --- http response --- 19:10:00 INFO [http_cmd]: http status code OK 19:10:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:10:00 DEBUG --- status code --- 19:10:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:10:00 DEBUG --- http response --- 19:10:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:10:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462600" 19:10:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734462600" 19:10:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734462600" 19:10:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462600" 19:10:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462600" 19:10:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462600" 19:10:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734462600" 19:10:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462600" 19:10:00 INFO [http_cmd]: http status code OK 19:10:00 DEBUG --- status code --- 19:10:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:10:00 DEBUG --- http response --- 19:10:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:10:00 INFO [http_cmd]: http status code OK 19:10:00 DEBUG --- status code --- 19:10:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:10:00 DEBUG --- http response --- 19:10:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:10:00 INFO [http_cmd]: http status code OK 19:10:00 DEBUG --- status code --- 19:10:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:10:00 DEBUG --- http response --- 19:10:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:10:00 INFO [http_cmd]: http status code OK 19:10:00 DEBUG --- status code --- 19:10:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:10:00 DEBUG --- http response --- 19:10:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:10:00 INFO [http_cmd]: http status code OK 19:10:00 DEBUG --- status code --- 19:10:00 INFO [http_cmd]: http status code OK 19:10:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:10:00 INFO [http_cmd]: http status code OK 19:10:00 DEBUG --- status code --- 19:10:00 DEBUG --- http response --- 19:10:00 DEBUG --- status code --- 19:10:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:10:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:10:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:10:00 DEBUG --- http response --- 19:10:00 DEBUG --- http response --- 19:10:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:10:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:10:00 INFO [http_cmd]: http status code OK 19:10:00 DEBUG --- status code --- 19:10:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:10:00 DEBUG --- http response --- 19:10:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:10:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462600" 19:10:00 INFO [http_cmd]: http status code OK 19:10:00 DEBUG --- status code --- 19:10:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:10:00 DEBUG --- http response --- 19:10:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:10:00 DEBUG ----- output ----- 19:10:00 DEBUG { 19:10:00 DEBUG "status": "success", 19:10:00 DEBUG "data": { 19:10:00 DEBUG "resultType": "vector", 19:10:00 DEBUG "result": [ 19:10:00 DEBUG { 19:10:00 DEBUG "metric": { 19:10:00 DEBUG "pod": "ds-idrepo-0" 19:10:00 DEBUG }, 19:10:00 DEBUG "value": [ 19:10:00 DEBUG 19:10:00 DEBUG 19:10:00 DEBUG 19:10:00 DEBUG [...] 19:10:00 DEBUG 19:10:00 DEBUG 19:10:00 DEBUG 19:10:00 DEBUG "pod": "ds-cts-0" 19:10:00 DEBUG }, 19:10:00 DEBUG "value": [ 19:10:00 DEBUG 1734462600, 19:10:00 DEBUG "0.005748729819919643" 19:10:00 DEBUG ] 19:10:00 DEBUG } 19:10:00 DEBUG ] 19:10:00 DEBUG } 19:10:00 DEBUG } 19:10:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462600" 19:10:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462600" 19:10:00 INFO [http_cmd]: http status code OK 19:10:00 DEBUG --- status code --- 19:10:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:10:00 DEBUG --- http response --- 19:10:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:10:00 DEBUG ----- output ----- 19:10:00 DEBUG { 19:10:00 DEBUG "status": "success", 19:10:00 DEBUG "data": { 19:10:00 DEBUG "resultType": "vector", 19:10:00 DEBUG "result": [ 19:10:00 DEBUG { 19:10:00 DEBUG "metric": { 19:10:00 DEBUG "instance": "10.142.0.11:9100" 19:10:00 DEBUG }, 19:10:00 DEBUG "value": [ 19:10:00 DEBUG 19:10:00 DEBUG 19:10:00 DEBUG 19:10:00 DEBUG [...] 19:10:00 DEBUG 19:10:00 DEBUG 19:10:00 DEBUG 19:10:00 DEBUG "instance": "10.142.0.51:9100" 19:10:00 DEBUG }, 19:10:00 DEBUG "value": [ 19:10:00 DEBUG 1734462600, 19:10:00 DEBUG "87683.6" 19:10:00 DEBUG ] 19:10:00 DEBUG } 19:10:00 DEBUG ] 19:10:00 DEBUG } 19:10:00 DEBUG } 19:10:00 INFO [http_cmd]: http status code OK 19:10:00 DEBUG --- status code --- 19:10:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:10:00 DEBUG --- http response --- 19:10:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:10:00 DEBUG ----- output ----- 19:10:00 DEBUG { 19:10:00 DEBUG "status": "success", 19:10:00 DEBUG "data": { 19:10:00 DEBUG "resultType": "vector", 19:10:00 DEBUG "result": [ 19:10:00 DEBUG { 19:10:00 DEBUG "metric": { 19:10:00 DEBUG "instance": "10.142.0.11:9100" 19:10:00 DEBUG }, 19:10:00 DEBUG "value": [ 19:10:00 DEBUG 19:10:00 DEBUG 19:10:00 DEBUG 19:10:00 DEBUG [...] 19:10:00 DEBUG 19:10:00 DEBUG 19:10:00 DEBUG 19:10:00 DEBUG "instance": "10.142.0.51:9100" 19:10:00 DEBUG }, 19:10:00 DEBUG "value": [ 19:10:00 DEBUG 1734462600, 19:10:00 DEBUG "0.0006666666666665342" 19:10:00 DEBUG ] 19:10:00 DEBUG } 19:10:00 DEBUG ] 19:10:00 DEBUG } 19:10:00 DEBUG } 19:10:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462600" 19:10:00 INFO [http_cmd]: http status code OK 19:10:00 DEBUG --- status code --- 19:10:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:10:00 DEBUG --- http response --- 19:10:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:10:00 DEBUG ----- output ----- 19:10:00 DEBUG { 19:10:00 DEBUG "status": "success", 19:10:00 DEBUG "data": { 19:10:00 DEBUG "resultType": "vector", 19:10:00 DEBUG "result": [ 19:10:00 DEBUG { 19:10:00 DEBUG "metric": { 19:10:00 DEBUG "instance": "10.142.0.11:9100" 19:10:00 DEBUG }, 19:10:00 DEBUG "value": [ 19:10:00 DEBUG 19:10:00 DEBUG 19:10:00 DEBUG 19:10:00 DEBUG [...] 19:10:00 DEBUG 19:10:00 DEBUG 19:10:00 DEBUG 19:10:00 DEBUG "instance": "10.142.0.51:9100" 19:10:00 DEBUG }, 19:10:00 DEBUG "value": [ 19:10:00 DEBUG 1734462600, 19:10:00 DEBUG "0.0004333333333358762" 19:10:00 DEBUG ] 19:10:00 DEBUG } 19:10:00 DEBUG ] 19:10:00 DEBUG } 19:10:00 DEBUG } 19:10:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462600" 19:10:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462600" 19:10:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734462600" 19:10:00 INFO [http_cmd]: http status code OK 19:10:00 DEBUG --- status code --- 19:10:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:10:00 DEBUG --- http response --- 19:10:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:10:00 DEBUG ----- output ----- 19:10:00 DEBUG { 19:10:00 DEBUG "status": "success", 19:10:00 DEBUG "data": { 19:10:00 DEBUG "resultType": "vector", 19:10:00 DEBUG "result": [ 19:10:00 DEBUG { 19:10:00 DEBUG "metric": { 19:10:00 DEBUG "pod": "ds-idrepo-0" 19:10:00 DEBUG }, 19:10:00 DEBUG "value": [ 19:10:00 DEBUG 19:10:00 DEBUG 19:10:00 DEBUG 19:10:00 DEBUG [...] 19:10:00 DEBUG 19:10:00 DEBUG 19:10:00 DEBUG 19:10:00 DEBUG "pod": "ds-cts-0" 19:10:00 DEBUG }, 19:10:00 DEBUG "value": [ 19:10:00 DEBUG 1734462600, 19:10:00 DEBUG "0" 19:10:00 DEBUG ] 19:10:00 DEBUG } 19:10:00 DEBUG ] 19:10:00 DEBUG } 19:10:00 DEBUG } 19:10:00 INFO [http_cmd]: http status code OK 19:10:00 DEBUG --- status code --- 19:10:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:10:00 DEBUG --- http response --- 19:10:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:10:00 DEBUG ----- output ----- 19:10:00 DEBUG { 19:10:00 DEBUG "status": "success", 19:10:00 DEBUG "data": { 19:10:00 DEBUG "resultType": "vector", 19:10:00 DEBUG "result": [ 19:10:00 DEBUG { 19:10:00 DEBUG "metric": { 19:10:00 DEBUG "pod": "ds-idrepo-0" 19:10:00 DEBUG }, 19:10:00 DEBUG "value": [ 19:10:00 DEBUG 19:10:00 DEBUG 19:10:00 DEBUG 19:10:00 DEBUG [...] 19:10:00 DEBUG 19:10:00 DEBUG 19:10:00 DEBUG 19:10:00 DEBUG "pod": "ds-cts-0" 19:10:00 DEBUG }, 19:10:00 DEBUG "value": [ 19:10:00 DEBUG 1734462600, 19:10:00 DEBUG "492.5593001232601" 19:10:00 DEBUG ] 19:10:00 DEBUG } 19:10:00 DEBUG ] 19:10:00 DEBUG } 19:10:00 DEBUG } 19:10:00 INFO [http_cmd]: http status code OK 19:10:00 DEBUG --- status code --- 19:10:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:10:00 DEBUG --- http response --- 19:10:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:10:00 DEBUG ----- output ----- 19:10:00 DEBUG { 19:10:00 DEBUG "status": "success", 19:10:00 DEBUG "data": { 19:10:00 DEBUG "resultType": "vector", 19:10:00 DEBUG "result": [ 19:10:00 DEBUG { 19:10:00 DEBUG "metric": { 19:10:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 19:10:00 DEBUG }, 19:10:00 DEBUG "value": [ 19:10:00 DEBUG 19:10:00 DEBUG 19:10:00 DEBUG 19:10:00 DEBUG [...] 19:10:00 DEBUG 19:10:00 DEBUG 19:10:00 DEBUG 19:10:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 19:10:00 DEBUG }, 19:10:00 DEBUG "value": [ 19:10:00 DEBUG 1734462600, 19:10:00 DEBUG "2568273920" 19:10:00 DEBUG ] 19:10:00 DEBUG } 19:10:00 DEBUG ] 19:10:00 DEBUG } 19:10:00 DEBUG } 19:10:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734462600" 19:10:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462600" 19:10:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462600" 19:10:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462600" 19:10:00 INFO [http_cmd]: http status code OK 19:10:00 DEBUG --- status code --- 19:10:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:10:00 DEBUG --- http response --- 19:10:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:10:00 DEBUG ----- output ----- 19:10:00 DEBUG { 19:10:00 DEBUG "status": "success", 19:10:00 DEBUG "data": { 19:10:00 DEBUG "resultType": "vector", 19:10:00 DEBUG "result": [ 19:10:00 DEBUG { 19:10:00 DEBUG "metric": { 19:10:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 19:10:00 DEBUG }, 19:10:00 DEBUG "value": [ 19:10:00 DEBUG 19:10:00 DEBUG 19:10:00 DEBUG 19:10:00 DEBUG [...] 19:10:00 DEBUG 19:10:00 DEBUG 19:10:00 DEBUG 19:10:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 19:10:00 DEBUG }, 19:10:00 DEBUG "value": [ 19:10:00 DEBUG 1734462600, 19:10:00 DEBUG "1.7663559419508894" 19:10:00 DEBUG ] 19:10:00 DEBUG } 19:10:00 DEBUG ] 19:10:00 DEBUG } 19:10:00 DEBUG } 19:10:00 INFO [http_cmd]: http status code OK 19:10:00 DEBUG --- status code --- 19:10:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:10:00 DEBUG --- http response --- 19:10:00 INFO [http_cmd]: http status code OK 19:10:00 DEBUG --- status code --- 19:10:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:10:00 INFO [http_cmd]: http status code OK 19:10:00 DEBUG --- http response --- 19:10:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:10:00 DEBUG --- status code --- 19:10:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:10:00 DEBUG ----- output ----- 19:10:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:10:00 DEBUG ----- output ----- 19:10:00 DEBUG { 19:10:00 DEBUG --- http response --- 19:10:00 DEBUG { 19:10:00 DEBUG "status": "success", 19:10:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:10:00 DEBUG "status": "success", 19:10:00 DEBUG "data": { 19:10:00 DEBUG ----- output ----- 19:10:00 DEBUG "data": { 19:10:00 DEBUG "resultType": "vector", 19:10:00 DEBUG { 19:10:00 DEBUG "resultType": "vector", 19:10:00 DEBUG "result": [ 19:10:00 DEBUG "status": "success", 19:10:00 DEBUG "result": [ 19:10:00 DEBUG { 19:10:00 DEBUG "data": { 19:10:00 DEBUG { 19:10:00 DEBUG "metric": { 19:10:00 DEBUG "resultType": "vector", 19:10:00 DEBUG "metric": { 19:10:00 DEBUG "instance": "10.142.0.11:9100" 19:10:00 DEBUG "result": [ 19:10:00 DEBUG "instance": "10.142.0.11:9100" 19:10:00 DEBUG }, 19:10:00 DEBUG { 19:10:00 DEBUG }, 19:10:00 DEBUG "value": [ 19:10:00 DEBUG "metric": { 19:10:00 DEBUG "value": [ 19:10:00 DEBUG 19:10:00 DEBUG "pod": "ds-cts-0" 19:10:00 DEBUG 19:10:00 DEBUG 19:10:00 DEBUG }, 19:10:00 DEBUG 19:10:00 DEBUG 19:10:00 DEBUG "value": [ 19:10:00 DEBUG 19:10:00 DEBUG [...] 19:10:00 DEBUG 19:10:00 DEBUG [...] 19:10:00 DEBUG 19:10:00 DEBUG 19:10:00 DEBUG 19:10:00 DEBUG 19:10:00 DEBUG 19:10:00 DEBUG 19:10:00 DEBUG 19:10:00 DEBUG [...] 19:10:00 DEBUG 19:10:00 DEBUG "instance": "10.142.0.51:9100" 19:10:00 DEBUG 19:10:00 DEBUG "instance": "10.142.0.51:9100" 19:10:00 DEBUG }, 19:10:00 DEBUG 19:10:00 DEBUG }, 19:10:00 DEBUG "value": [ 19:10:00 DEBUG 19:10:00 DEBUG "value": [ 19:10:00 DEBUG 1734462600, 19:10:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:10:00 DEBUG }, 19:10:00 DEBUG "24712.53333333334" 19:10:00 DEBUG 1734462600, 19:10:00 DEBUG "value": [ 19:10:00 DEBUG ] 19:10:00 DEBUG "70618.90000000002" 19:10:00 DEBUG 1734462600, 19:10:00 DEBUG } 19:10:00 DEBUG ] 19:10:00 DEBUG "696045568" 19:10:00 DEBUG ] 19:10:00 DEBUG } 19:10:00 DEBUG ] 19:10:00 DEBUG } 19:10:00 DEBUG ] 19:10:00 DEBUG } 19:10:00 DEBUG } 19:10:00 DEBUG } 19:10:00 DEBUG ] 19:10:00 DEBUG } 19:10:00 DEBUG } 19:10:00 DEBUG } 19:10:03 INFO 19:10:03 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 19:10:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:10:03 INFO [loop_until]: OK (rc = 0) 19:10:03 DEBUG --- stdout --- 19:10:03 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 546Mi ds-idrepo-0 7m 740Mi rcs-5d696ff5b-7ttvr 3m 297Mi rcs-5d696ff5b-wwlbr 2m 331Mi 19:10:03 DEBUG --- stderr --- 19:10:03 DEBUG 19:10:03 INFO 19:10:03 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 19:10:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:10:04 INFO [loop_until]: OK (rc = 0) 19:10:04 DEBUG --- stdout --- 19:10:04 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 104m 0% 2274Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1836m 5% 3657Mi 12% 19:10:04 DEBUG --- stderr --- 19:10:04 DEBUG 19:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462659" 19:10:59 INFO [http_cmd]: http status code OK 19:10:59 DEBUG --- status code --- 19:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:10:59 DEBUG --- http response --- 19:10:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734462659" 19:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462659" 19:10:59 INFO [http_cmd]: http status code OK 19:10:59 DEBUG --- status code --- 19:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:10:59 DEBUG --- http response --- 19:10:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:10:59 INFO [http_cmd]: http status code OK 19:10:59 DEBUG --- status code --- 19:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:10:59 DEBUG --- http response --- 19:10:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462659" 19:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462659" 19:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462659" 19:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462659" 19:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734462659" 19:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462659" 19:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462659" 19:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462659" 19:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462659" 19:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462659" 19:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462659" 19:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462659" 19:10:59 INFO [http_cmd]: http status code OK 19:10:59 DEBUG --- status code --- 19:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:10:59 DEBUG --- http response --- 19:10:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:10:59 INFO [http_cmd]: http status code OK 19:10:59 DEBUG --- status code --- 19:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:10:59 DEBUG --- http response --- 19:10:59 INFO [http_cmd]: http status code OK 19:10:59 INFO [http_cmd]: http status code OK 19:10:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:10:59 INFO [http_cmd]: http status code OK 19:10:59 DEBUG --- status code --- 19:10:59 DEBUG --- status code --- 19:10:59 ERROR [http_cmd]: ERROR 19:10:59 DEBUG --- status code --- 19:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:10:59 ERROR --- status code --- 19:10:59 INFO [http_cmd]: http status code OK 19:10:59 INFO [http_cmd]: http status code OK 19:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:10:59 DEBUG --- http response --- 19:10:59 DEBUG --- http response --- 19:10:59 INFO [http_cmd]: http status code OK 19:10:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:10:59 DEBUG --- status code --- 19:10:59 DEBUG --- status code --- 19:10:59 DEBUG --- http response --- 19:11:00 INFO [http_cmd]: http status code OK 19:11:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:11:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:11:00 DEBUG --- status code --- 19:11:00 INFO [http_cmd]: http status code OK 19:11:00 ERROR --- http response --- 19:11:00 INFO [http_cmd]: http status code OK 19:11:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:11:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:11:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:11:00 DEBUG --- status code --- 19:11:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:11:00 DEBUG --- status code --- 19:11:00 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:11:00 DEBUG --- status code --- 19:11:00 DEBUG --- http response --- 19:11:00 DEBUG --- http response --- 19:11:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:11:00 DEBUG --- http response --- 19:11:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:11:00 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:11:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:11:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:11:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:11:00 DEBUG --- http response --- 19:11:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:11:00 DEBUG --- http response --- 19:11:00 DEBUG --- http response --- 19:11:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:11:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:11:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:11:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462660" 19:11:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462660" 19:11:00 INFO [http_cmd]: http status code OK 19:11:00 DEBUG --- status code --- 19:11:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:11:00 DEBUG --- http response --- 19:11:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:11:00 INFO [http_cmd]: http status code OK 19:11:00 DEBUG --- status code --- 19:11:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:11:00 DEBUG --- http response --- 19:11:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:11:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462660" 19:11:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462660" 19:11:00 INFO [http_cmd]: http status code OK 19:11:00 DEBUG --- status code --- 19:11:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:11:00 DEBUG --- http response --- 19:11:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:11:00 INFO [http_cmd]: http status code OK 19:11:00 DEBUG --- status code --- 19:11:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:11:00 DEBUG --- http response --- 19:11:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:11:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734462660" 19:11:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734462660" 19:11:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462660" 19:11:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462660" 19:11:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462660" 19:11:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734462660" 19:11:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462660" 19:11:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462660" 19:11:00 INFO [http_cmd]: http status code OK 19:11:00 DEBUG --- status code --- 19:11:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:11:00 DEBUG --- http response --- 19:11:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:11:00 INFO [http_cmd]: http status code OK 19:11:00 DEBUG --- status code --- 19:11:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:11:00 DEBUG --- http response --- 19:11:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:11:00 INFO [http_cmd]: http status code OK 19:11:00 DEBUG --- status code --- 19:11:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:11:00 DEBUG --- http response --- 19:11:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:11:00 INFO [http_cmd]: http status code OK 19:11:00 INFO [http_cmd]: http status code OK 19:11:00 DEBUG --- status code --- 19:11:00 DEBUG --- status code --- 19:11:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:11:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:11:00 DEBUG --- http response --- 19:11:00 DEBUG --- http response --- 19:11:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:11:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:11:00 INFO [http_cmd]: http status code OK 19:11:00 DEBUG --- status code --- 19:11:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:11:00 DEBUG --- http response --- 19:11:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:11:00 INFO [http_cmd]: http status code OK 19:11:00 DEBUG --- status code --- 19:11:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:11:00 DEBUG --- http response --- 19:11:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:11:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462660" 19:11:00 INFO [http_cmd]: http status code OK 19:11:00 DEBUG --- status code --- 19:11:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:11:00 DEBUG --- http response --- 19:11:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:11:00 INFO [http_cmd]: http status code OK 19:11:00 DEBUG --- status code --- 19:11:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:11:00 DEBUG --- http response --- 19:11:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:11:00 DEBUG ----- output ----- 19:11:00 DEBUG { 19:11:00 DEBUG "status": "success", 19:11:00 DEBUG "data": { 19:11:00 DEBUG "resultType": "vector", 19:11:00 DEBUG "result": [ 19:11:00 DEBUG { 19:11:00 DEBUG "metric": { 19:11:00 DEBUG "pod": "ds-cts-0" 19:11:00 DEBUG }, 19:11:00 DEBUG "value": [ 19:11:00 DEBUG 19:11:00 DEBUG 19:11:00 DEBUG 19:11:00 DEBUG [...] 19:11:00 DEBUG 19:11:00 DEBUG 19:11:00 DEBUG 19:11:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:11:00 DEBUG }, 19:11:00 DEBUG "value": [ 19:11:00 DEBUG 1734462660, 19:11:00 DEBUG "0.0108070714092901" 19:11:00 DEBUG ] 19:11:00 DEBUG } 19:11:00 DEBUG ] 19:11:00 DEBUG } 19:11:00 DEBUG } 19:11:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462660" 19:11:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462660" 19:11:00 INFO [http_cmd]: http status code OK 19:11:00 DEBUG --- status code --- 19:11:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:11:00 DEBUG --- http response --- 19:11:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:11:00 DEBUG ----- output ----- 19:11:00 DEBUG { 19:11:00 DEBUG "status": "success", 19:11:00 DEBUG "data": { 19:11:00 DEBUG "resultType": "vector", 19:11:00 DEBUG "result": [ 19:11:00 DEBUG { 19:11:00 DEBUG "metric": { 19:11:00 DEBUG "instance": "10.142.0.11:9100" 19:11:00 DEBUG }, 19:11:00 DEBUG "value": [ 19:11:00 DEBUG 19:11:00 DEBUG 19:11:00 DEBUG 19:11:00 DEBUG [...] 19:11:00 DEBUG 19:11:00 DEBUG 19:11:00 DEBUG 19:11:00 DEBUG "instance": "10.142.0.51:9100" 19:11:00 DEBUG }, 19:11:00 DEBUG "value": [ 19:11:00 DEBUG 1734462660, 19:11:00 DEBUG "91416.46666666669" 19:11:00 DEBUG ] 19:11:00 DEBUG } 19:11:00 DEBUG ] 19:11:00 DEBUG } 19:11:00 DEBUG } 19:11:00 INFO [http_cmd]: http status code OK 19:11:00 DEBUG --- status code --- 19:11:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:11:00 DEBUG --- http response --- 19:11:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:11:00 DEBUG ----- output ----- 19:11:00 DEBUG { 19:11:00 DEBUG "status": "success", 19:11:00 DEBUG "data": { 19:11:00 DEBUG "resultType": "vector", 19:11:00 DEBUG "result": [ 19:11:00 DEBUG { 19:11:00 DEBUG "metric": { 19:11:00 DEBUG "instance": "10.142.0.11:9100" 19:11:00 DEBUG }, 19:11:00 DEBUG "value": [ 19:11:00 DEBUG 19:11:00 DEBUG 19:11:00 DEBUG 19:11:00 DEBUG [...] 19:11:00 DEBUG 19:11:00 DEBUG 19:11:00 DEBUG 19:11:00 DEBUG "instance": "10.142.0.51:9100" 19:11:00 DEBUG }, 19:11:00 DEBUG "value": [ 19:11:00 DEBUG 1734462660, 19:11:00 DEBUG "0" 19:11:00 DEBUG ] 19:11:00 DEBUG } 19:11:00 DEBUG ] 19:11:00 DEBUG } 19:11:00 DEBUG } 19:11:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462660" 19:11:00 INFO [http_cmd]: http status code OK 19:11:00 DEBUG --- status code --- 19:11:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:11:00 DEBUG --- http response --- 19:11:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:11:00 DEBUG ----- output ----- 19:11:00 DEBUG { 19:11:00 DEBUG "status": "success", 19:11:00 DEBUG "data": { 19:11:00 DEBUG "resultType": "vector", 19:11:00 DEBUG "result": [ 19:11:00 DEBUG { 19:11:00 DEBUG "metric": { 19:11:00 DEBUG "instance": "10.142.0.11:9100" 19:11:00 DEBUG }, 19:11:00 DEBUG "value": [ 19:11:00 DEBUG 19:11:00 DEBUG 19:11:00 DEBUG 19:11:00 DEBUG [...] 19:11:00 DEBUG 19:11:00 DEBUG 19:11:00 DEBUG 19:11:00 DEBUG "instance": "10.142.0.51:9100" 19:11:00 DEBUG }, 19:11:00 DEBUG "value": [ 19:11:00 DEBUG 1734462660, 19:11:00 DEBUG "0.0003333333333339776" 19:11:00 DEBUG ] 19:11:00 DEBUG } 19:11:00 DEBUG ] 19:11:00 DEBUG } 19:11:00 DEBUG } 19:11:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462660" 19:11:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462660" 19:11:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734462660" 19:11:00 INFO [http_cmd]: http status code OK 19:11:00 DEBUG --- status code --- 19:11:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:11:00 DEBUG --- http response --- 19:11:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:11:00 DEBUG ----- output ----- 19:11:00 DEBUG { 19:11:00 DEBUG "status": "success", 19:11:00 DEBUG "data": { 19:11:00 DEBUG "resultType": "vector", 19:11:00 DEBUG "result": [ 19:11:00 DEBUG { 19:11:00 DEBUG "metric": { 19:11:00 DEBUG "pod": "ds-cts-0" 19:11:00 DEBUG }, 19:11:00 DEBUG "value": [ 19:11:00 DEBUG 19:11:00 DEBUG 19:11:00 DEBUG 19:11:00 DEBUG [...] 19:11:00 DEBUG 19:11:00 DEBUG 19:11:00 DEBUG 19:11:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:11:00 DEBUG }, 19:11:00 DEBUG "value": [ 19:11:00 DEBUG 1734462660, 19:11:00 DEBUG "0" 19:11:00 DEBUG ] 19:11:00 DEBUG } 19:11:00 DEBUG ] 19:11:00 DEBUG } 19:11:00 DEBUG } 19:11:00 INFO [http_cmd]: http status code OK 19:11:00 DEBUG --- status code --- 19:11:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:11:00 DEBUG --- http response --- 19:11:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:11:00 DEBUG ----- output ----- 19:11:00 DEBUG { 19:11:00 DEBUG "status": "success", 19:11:00 DEBUG "data": { 19:11:00 DEBUG "resultType": "vector", 19:11:00 DEBUG "result": [ 19:11:00 DEBUG { 19:11:00 DEBUG "metric": { 19:11:00 DEBUG "pod": "ds-cts-0" 19:11:00 DEBUG }, 19:11:00 DEBUG "value": [ 19:11:00 DEBUG 19:11:00 DEBUG 19:11:00 DEBUG 19:11:00 DEBUG [...] 19:11:00 DEBUG 19:11:00 DEBUG 19:11:00 DEBUG 19:11:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:11:00 DEBUG }, 19:11:00 DEBUG "value": [ 19:11:00 DEBUG 1734462660, 19:11:00 DEBUG "1257.45846081049" 19:11:00 DEBUG ] 19:11:00 DEBUG } 19:11:00 DEBUG ] 19:11:00 DEBUG } 19:11:00 DEBUG } 19:11:00 INFO [http_cmd]: http status code OK 19:11:00 DEBUG --- status code --- 19:11:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:11:00 DEBUG --- http response --- 19:11:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:11:00 DEBUG ----- output ----- 19:11:00 DEBUG { 19:11:00 DEBUG "status": "success", 19:11:00 DEBUG "data": { 19:11:00 DEBUG "resultType": "vector", 19:11:00 DEBUG "result": [ 19:11:00 DEBUG { 19:11:00 DEBUG "metric": { 19:11:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 19:11:00 DEBUG }, 19:11:00 DEBUG "value": [ 19:11:00 DEBUG 19:11:00 DEBUG 19:11:00 DEBUG 19:11:00 DEBUG [...] 19:11:00 DEBUG 19:11:00 DEBUG 19:11:00 DEBUG 19:11:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 19:11:00 DEBUG }, 19:11:00 DEBUG "value": [ 19:11:00 DEBUG 1734462660, 19:11:00 DEBUG "2563846144" 19:11:00 DEBUG ] 19:11:00 DEBUG } 19:11:00 DEBUG ] 19:11:00 DEBUG } 19:11:00 DEBUG } 19:11:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734462660" 19:11:00 INFO [http_cmd]: http status code OK 19:11:00 DEBUG --- status code --- 19:11:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:11:00 DEBUG --- http response --- 19:11:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:11:00 DEBUG ----- output ----- 19:11:00 DEBUG { 19:11:00 DEBUG "status": "success", 19:11:00 DEBUG "data": { 19:11:00 DEBUG "resultType": "vector", 19:11:00 DEBUG "result": [ 19:11:00 DEBUG { 19:11:00 DEBUG "metric": { 19:11:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 19:11:00 DEBUG }, 19:11:00 DEBUG "value": [ 19:11:00 DEBUG 19:11:00 DEBUG 19:11:00 DEBUG 19:11:00 DEBUG [...] 19:11:00 DEBUG 19:11:00 DEBUG 19:11:00 DEBUG 19:11:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 19:11:00 DEBUG }, 19:11:00 DEBUG "value": [ 19:11:00 DEBUG 1734462660, 19:11:00 DEBUG "1.7590036575167345" 19:11:00 DEBUG ] 19:11:00 DEBUG } 19:11:00 DEBUG ] 19:11:00 DEBUG } 19:11:00 DEBUG } 19:11:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462660" 19:11:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462660" 19:11:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462660" 19:11:00 INFO [http_cmd]: http status code OK 19:11:00 DEBUG --- status code --- 19:11:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:11:00 INFO [http_cmd]: http status code OK 19:11:00 DEBUG --- http response --- 19:11:00 DEBUG --- status code --- 19:11:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:11:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:11:00 DEBUG ----- output ----- 19:11:00 DEBUG --- http response --- 19:11:00 DEBUG { 19:11:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:11:00 DEBUG "status": "success", 19:11:00 DEBUG ----- output ----- 19:11:00 DEBUG "data": { 19:11:00 DEBUG { 19:11:00 DEBUG "resultType": "vector", 19:11:00 DEBUG "status": "success", 19:11:00 DEBUG "result": [ 19:11:00 DEBUG "data": { 19:11:00 DEBUG { 19:11:00 DEBUG "resultType": "vector", 19:11:00 DEBUG "metric": { 19:11:00 DEBUG "result": [ 19:11:00 DEBUG "pod": "ds-cts-0" 19:11:00 DEBUG { 19:11:00 DEBUG }, 19:11:00 DEBUG "metric": { 19:11:00 DEBUG "value": [ 19:11:00 DEBUG "instance": "10.142.0.11:9100" 19:11:00 DEBUG 19:11:00 DEBUG }, 19:11:00 DEBUG 19:11:00 DEBUG "value": [ 19:11:00 DEBUG 19:11:00 DEBUG 19:11:00 DEBUG [...] 19:11:00 DEBUG 19:11:00 DEBUG 19:11:00 DEBUG 19:11:00 DEBUG 19:11:00 DEBUG [...] 19:11:00 DEBUG 19:11:00 DEBUG 19:11:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:11:00 DEBUG 19:11:00 DEBUG }, 19:11:00 DEBUG 19:11:00 DEBUG "value": [ 19:11:00 DEBUG "instance": "10.142.0.51:9100" 19:11:00 DEBUG 1734462660, 19:11:00 DEBUG }, 19:11:00 DEBUG "696201216" 19:11:00 DEBUG "value": [ 19:11:00 DEBUG ] 19:11:00 DEBUG } 19:11:00 INFO [http_cmd]: http status code OK 19:11:00 DEBUG 1734462660, 19:11:00 DEBUG ] 19:11:00 DEBUG --- status code --- 19:11:00 DEBUG "26350.933333333338" 19:11:00 DEBUG } 19:11:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:11:00 DEBUG ] 19:11:00 DEBUG } 19:11:00 DEBUG --- http response --- 19:11:00 DEBUG } 19:11:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:11:00 DEBUG ] 19:11:00 DEBUG ----- output ----- 19:11:00 DEBUG } 19:11:00 DEBUG { 19:11:00 DEBUG } 19:11:00 DEBUG "status": "success", 19:11:00 DEBUG "data": { 19:11:00 DEBUG "resultType": "vector", 19:11:00 DEBUG "result": [ 19:11:00 DEBUG { 19:11:00 DEBUG "metric": { 19:11:00 DEBUG "instance": "10.142.0.11:9100" 19:11:00 DEBUG }, 19:11:00 DEBUG "value": [ 19:11:00 DEBUG 19:11:00 DEBUG 19:11:00 DEBUG 19:11:00 DEBUG [...] 19:11:00 DEBUG 19:11:00 DEBUG 19:11:00 DEBUG 19:11:00 DEBUG "instance": "10.142.0.51:9100" 19:11:00 DEBUG }, 19:11:00 DEBUG "value": [ 19:11:00 DEBUG 1734462660, 19:11:00 DEBUG "74965.43333333335" 19:11:00 DEBUG ] 19:11:00 DEBUG } 19:11:00 DEBUG ] 19:11:00 DEBUG } 19:11:00 DEBUG } 19:11:03 INFO 19:11:03 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 19:11:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:11:03 INFO [loop_until]: OK (rc = 0) 19:11:03 DEBUG --- stdout --- 19:11:03 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 545Mi ds-idrepo-0 7m 741Mi rcs-5d696ff5b-7ttvr 3m 297Mi rcs-5d696ff5b-wwlbr 6m 331Mi 19:11:03 DEBUG --- stderr --- 19:11:03 DEBUG 19:11:04 INFO 19:11:04 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 19:11:04 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:11:04 INFO [loop_until]: OK (rc = 0) 19:11:04 DEBUG --- stdout --- 19:11:04 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 99m 0% 2283Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1823m 5% 3653Mi 12% 19:11:04 DEBUG --- stderr --- 19:11:04 DEBUG 19:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462719" 19:11:59 INFO [http_cmd]: http status code OK 19:11:59 DEBUG --- status code --- 19:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:11:59 DEBUG --- http response --- 19:11:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462719" 19:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734462719" 19:11:59 INFO [http_cmd]: http status code OK 19:11:59 DEBUG --- status code --- 19:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:11:59 DEBUG --- http response --- 19:11:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:11:59 INFO [http_cmd]: http status code OK 19:11:59 DEBUG --- status code --- 19:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:11:59 DEBUG --- http response --- 19:11:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462719" 19:12:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462719" 19:12:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462720" 19:12:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462720" 19:12:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462720" 19:12:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734462720" 19:12:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462720" 19:12:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462720" 19:12:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462720" 19:12:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462720" 19:12:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462720" 19:12:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462720" 19:12:00 INFO [http_cmd]: http status code OK 19:12:00 DEBUG --- status code --- 19:12:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:12:00 DEBUG --- http response --- 19:12:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:12:00 INFO [http_cmd]: http status code OK 19:12:00 DEBUG --- status code --- 19:12:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:12:00 DEBUG --- http response --- 19:12:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:12:00 INFO [http_cmd]: http status code OK 19:12:00 DEBUG --- status code --- 19:12:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:12:00 DEBUG --- http response --- 19:12:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:12:00 INFO [http_cmd]: http status code OK 19:12:00 INFO [http_cmd]: http status code OK 19:12:00 INFO [http_cmd]: http status code OK 19:12:00 INFO [http_cmd]: http status code OK 19:12:00 DEBUG --- status code --- 19:12:00 INFO [http_cmd]: http status code OK 19:12:00 INFO [http_cmd]: http status code OK 19:12:00 INFO [http_cmd]: http status code OK 19:12:00 ERROR [http_cmd]: ERROR 19:12:00 INFO [http_cmd]: http status code OK 19:12:00 DEBUG --- status code --- 19:12:00 DEBUG --- status code --- 19:12:00 DEBUG --- status code --- 19:12:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:12:00 DEBUG --- status code --- 19:12:00 DEBUG --- status code --- 19:12:00 DEBUG --- status code --- 19:12:00 ERROR --- status code --- 19:12:00 DEBUG --- status code --- 19:12:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:12:00 DEBUG --- http response --- 19:12:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:12:00 DEBUG --- http response --- 19:12:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:12:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:12:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:12:00 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:12:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:12:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:12:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:12:00 DEBUG --- http response --- 19:12:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:12:00 DEBUG --- http response --- 19:12:00 DEBUG --- http response --- 19:12:00 DEBUG --- http response --- 19:12:00 ERROR --- http response --- 19:12:00 DEBUG --- http response --- 19:12:00 DEBUG --- http response --- 19:12:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:12:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:12:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:12:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:12:00 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:12:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:12:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:12:00 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:12:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462720" 19:12:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462720" 19:12:00 INFO [http_cmd]: http status code OK 19:12:00 DEBUG --- status code --- 19:12:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:12:00 DEBUG --- http response --- 19:12:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:12:00 INFO [http_cmd]: http status code OK 19:12:00 DEBUG --- status code --- 19:12:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:12:00 DEBUG --- http response --- 19:12:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:12:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462720" 19:12:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462720" 19:12:00 INFO [http_cmd]: http status code OK 19:12:00 DEBUG --- status code --- 19:12:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:12:00 DEBUG --- http response --- 19:12:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:12:00 INFO [http_cmd]: http status code OK 19:12:00 DEBUG --- status code --- 19:12:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:12:00 DEBUG --- http response --- 19:12:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:12:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734462720" 19:12:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462720" 19:12:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734462720" 19:12:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462720" 19:12:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734462720" 19:12:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462720" 19:12:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462720" 19:12:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462720" 19:12:00 INFO [http_cmd]: http status code OK 19:12:00 DEBUG --- status code --- 19:12:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:12:00 DEBUG --- http response --- 19:12:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:12:00 INFO [http_cmd]: http status code OK 19:12:00 DEBUG --- status code --- 19:12:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:12:00 DEBUG --- http response --- 19:12:00 INFO [http_cmd]: http status code OK 19:12:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:12:00 INFO [http_cmd]: http status code OK 19:12:00 DEBUG --- status code --- 19:12:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462720" 19:12:00 DEBUG --- status code --- 19:12:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:12:00 INFO [http_cmd]: http status code OK 19:12:00 INFO [http_cmd]: http status code OK 19:12:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:12:00 DEBUG --- http response --- 19:12:00 DEBUG --- status code --- 19:12:00 DEBUG --- status code --- 19:12:00 DEBUG --- http response --- 19:12:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:12:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:12:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:12:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:12:00 INFO [http_cmd]: http status code OK 19:12:00 INFO [http_cmd]: http status code OK 19:12:00 DEBUG --- http response --- 19:12:00 DEBUG --- http response --- 19:12:00 DEBUG --- status code --- 19:12:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:12:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:12:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:12:00 DEBUG --- status code --- 19:12:00 DEBUG --- http response --- 19:12:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:12:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:12:00 DEBUG --- http response --- 19:12:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:12:00 INFO [http_cmd]: http status code OK 19:12:00 DEBUG --- status code --- 19:12:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:12:00 DEBUG --- http response --- 19:12:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:12:00 DEBUG ----- output ----- 19:12:00 DEBUG { 19:12:00 DEBUG "status": "success", 19:12:00 DEBUG "data": { 19:12:00 DEBUG "resultType": "vector", 19:12:00 DEBUG "result": [ 19:12:00 DEBUG { 19:12:00 DEBUG "metric": { 19:12:00 DEBUG "pod": "ds-cts-0" 19:12:00 DEBUG }, 19:12:00 DEBUG "value": [ 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG [...] 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:12:00 DEBUG }, 19:12:00 DEBUG "value": [ 19:12:00 DEBUG 1734462720, 19:12:00 DEBUG "0.0036370390573552173" 19:12:00 DEBUG ] 19:12:00 DEBUG } 19:12:00 DEBUG ] 19:12:00 DEBUG } 19:12:00 DEBUG } 19:12:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462720" 19:12:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462720" 19:12:00 INFO [http_cmd]: http status code OK 19:12:00 DEBUG --- status code --- 19:12:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:12:00 DEBUG --- http response --- 19:12:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:12:00 DEBUG ----- output ----- 19:12:00 DEBUG { 19:12:00 DEBUG "status": "success", 19:12:00 DEBUG "data": { 19:12:00 DEBUG "resultType": "vector", 19:12:00 DEBUG "result": [ 19:12:00 DEBUG { 19:12:00 DEBUG "metric": { 19:12:00 DEBUG "instance": "10.142.0.11:9100" 19:12:00 DEBUG }, 19:12:00 DEBUG "value": [ 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG [...] 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG "instance": "10.142.0.51:9100" 19:12:00 DEBUG }, 19:12:00 DEBUG "value": [ 19:12:00 DEBUG 1734462720, 19:12:00 DEBUG "90440.43333333336" 19:12:00 DEBUG ] 19:12:00 DEBUG } 19:12:00 DEBUG ] 19:12:00 DEBUG } 19:12:00 DEBUG } 19:12:00 INFO [http_cmd]: http status code OK 19:12:00 DEBUG --- status code --- 19:12:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:12:00 DEBUG --- http response --- 19:12:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:12:00 DEBUG ----- output ----- 19:12:00 DEBUG { 19:12:00 DEBUG "status": "success", 19:12:00 DEBUG "data": { 19:12:00 DEBUG "resultType": "vector", 19:12:00 DEBUG "result": [ 19:12:00 DEBUG { 19:12:00 DEBUG "metric": { 19:12:00 DEBUG "instance": "10.142.0.11:9100" 19:12:00 DEBUG }, 19:12:00 DEBUG "value": [ 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG [...] 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG "instance": "10.142.0.51:9100" 19:12:00 DEBUG }, 19:12:00 DEBUG "value": [ 19:12:00 DEBUG 1734462720, 19:12:00 DEBUG "0" 19:12:00 DEBUG ] 19:12:00 DEBUG } 19:12:00 DEBUG ] 19:12:00 DEBUG } 19:12:00 DEBUG } 19:12:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462720" 19:12:00 INFO [http_cmd]: http status code OK 19:12:00 DEBUG --- status code --- 19:12:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:12:00 DEBUG --- http response --- 19:12:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:12:00 DEBUG ----- output ----- 19:12:00 DEBUG { 19:12:00 DEBUG "status": "success", 19:12:00 DEBUG "data": { 19:12:00 DEBUG "resultType": "vector", 19:12:00 DEBUG "result": [ 19:12:00 DEBUG { 19:12:00 DEBUG "metric": { 19:12:00 DEBUG "instance": "10.142.0.11:9100" 19:12:00 DEBUG }, 19:12:00 DEBUG "value": [ 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG [...] 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG "instance": "10.142.0.51:9100" 19:12:00 DEBUG }, 19:12:00 DEBUG "value": [ 19:12:00 DEBUG 1734462720, 19:12:00 DEBUG "0.0004666666666665266" 19:12:00 DEBUG ] 19:12:00 DEBUG } 19:12:00 DEBUG ] 19:12:00 DEBUG } 19:12:00 DEBUG } 19:12:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462720" 19:12:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462720" 19:12:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734462720" 19:12:00 INFO [http_cmd]: http status code OK 19:12:00 DEBUG --- status code --- 19:12:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:12:00 DEBUG --- http response --- 19:12:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:12:00 DEBUG ----- output ----- 19:12:00 DEBUG { 19:12:00 DEBUG "status": "success", 19:12:00 DEBUG "data": { 19:12:00 DEBUG "resultType": "vector", 19:12:00 DEBUG "result": [ 19:12:00 DEBUG { 19:12:00 DEBUG "metric": { 19:12:00 DEBUG "pod": "ds-cts-0" 19:12:00 DEBUG }, 19:12:00 DEBUG "value": [ 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG [...] 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:12:00 DEBUG }, 19:12:00 DEBUG "value": [ 19:12:00 DEBUG 1734462720, 19:12:00 DEBUG "0" 19:12:00 DEBUG ] 19:12:00 DEBUG } 19:12:00 DEBUG ] 19:12:00 DEBUG } 19:12:00 DEBUG } 19:12:00 INFO [http_cmd]: http status code OK 19:12:00 DEBUG --- status code --- 19:12:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:12:00 DEBUG --- http response --- 19:12:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:12:00 DEBUG ----- output ----- 19:12:00 DEBUG { 19:12:00 DEBUG "status": "success", 19:12:00 DEBUG "data": { 19:12:00 DEBUG "resultType": "vector", 19:12:00 DEBUG "result": [ 19:12:00 DEBUG { 19:12:00 DEBUG "metric": { 19:12:00 DEBUG "pod": "ds-cts-0" 19:12:00 DEBUG }, 19:12:00 DEBUG "value": [ 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG [...] 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:12:00 DEBUG }, 19:12:00 DEBUG "value": [ 19:12:00 DEBUG 1734462720, 19:12:00 DEBUG "1343.7959366254272" 19:12:00 DEBUG ] 19:12:00 DEBUG } 19:12:00 DEBUG ] 19:12:00 DEBUG } 19:12:00 DEBUG } 19:12:00 INFO [http_cmd]: http status code OK 19:12:00 DEBUG --- status code --- 19:12:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:12:00 DEBUG --- http response --- 19:12:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:12:00 DEBUG ----- output ----- 19:12:00 DEBUG { 19:12:00 DEBUG "status": "success", 19:12:00 DEBUG "data": { 19:12:00 DEBUG "resultType": "vector", 19:12:00 DEBUG "result": [ 19:12:00 DEBUG { 19:12:00 DEBUG "metric": { 19:12:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 19:12:00 DEBUG }, 19:12:00 DEBUG "value": [ 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG [...] 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 19:12:00 DEBUG }, 19:12:00 DEBUG "value": [ 19:12:00 DEBUG 1734462720, 19:12:00 DEBUG "2567618560" 19:12:00 DEBUG ] 19:12:00 DEBUG } 19:12:00 DEBUG ] 19:12:00 DEBUG } 19:12:00 DEBUG } 19:12:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734462720" 19:12:00 INFO [http_cmd]: http status code OK 19:12:00 DEBUG --- status code --- 19:12:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:12:00 DEBUG --- http response --- 19:12:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:12:00 DEBUG ----- output ----- 19:12:00 DEBUG { 19:12:00 DEBUG "status": "success", 19:12:00 DEBUG "data": { 19:12:00 DEBUG "resultType": "vector", 19:12:00 DEBUG "result": [ 19:12:00 DEBUG { 19:12:00 DEBUG "metric": { 19:12:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 19:12:00 DEBUG }, 19:12:00 DEBUG "value": [ 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG [...] 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 19:12:00 DEBUG }, 19:12:00 DEBUG "value": [ 19:12:00 DEBUG 1734462720, 19:12:00 DEBUG "1.7616428926496739" 19:12:00 DEBUG ] 19:12:00 DEBUG } 19:12:00 DEBUG ] 19:12:00 DEBUG } 19:12:00 DEBUG } 19:12:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462720" 19:12:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462720" 19:12:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462720" 19:12:00 INFO [http_cmd]: http status code OK 19:12:00 DEBUG --- status code --- 19:12:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:12:00 DEBUG --- http response --- 19:12:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:12:00 DEBUG ----- output ----- 19:12:00 DEBUG { 19:12:00 DEBUG "status": "success", 19:12:00 DEBUG "data": { 19:12:00 DEBUG "resultType": "vector", 19:12:00 DEBUG "result": [ 19:12:00 DEBUG { 19:12:00 DEBUG "metric": { 19:12:00 DEBUG "instance": "10.142.0.11:9100" 19:12:00 DEBUG }, 19:12:00 DEBUG "value": [ 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG [...] 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG "instance": "10.142.0.51:9100" 19:12:00 DEBUG }, 19:12:00 DEBUG "value": [ 19:12:00 DEBUG 1734462720, 19:12:00 DEBUG "17749.333333333336" 19:12:00 DEBUG ] 19:12:00 DEBUG } 19:12:00 DEBUG ] 19:12:00 DEBUG } 19:12:00 DEBUG } 19:12:00 INFO [http_cmd]: http status code OK 19:12:00 DEBUG --- status code --- 19:12:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:12:00 DEBUG --- http response --- 19:12:00 INFO [http_cmd]: http status code OK 19:12:00 DEBUG --- status code --- 19:12:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:12:00 DEBUG --- http response --- 19:12:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:12:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:12:00 DEBUG ----- output ----- 19:12:00 DEBUG ----- output ----- 19:12:00 DEBUG { 19:12:00 DEBUG { 19:12:00 DEBUG "status": "success", 19:12:00 DEBUG "status": "success", 19:12:00 DEBUG "data": { 19:12:00 DEBUG "data": { 19:12:00 DEBUG "resultType": "vector", 19:12:00 DEBUG "resultType": "vector", 19:12:00 DEBUG "result": [ 19:12:00 DEBUG "result": [ 19:12:00 DEBUG { 19:12:00 DEBUG { 19:12:00 DEBUG "metric": { 19:12:00 DEBUG "metric": { 19:12:00 DEBUG "pod": "ds-cts-0" 19:12:00 DEBUG "instance": "10.142.0.11:9100" 19:12:00 DEBUG }, 19:12:00 DEBUG }, 19:12:00 DEBUG "value": [ 19:12:00 DEBUG "value": [ 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG [...] 19:12:00 DEBUG [...] 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG 19:12:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:12:00 DEBUG "instance": "10.142.0.51:9100" 19:12:00 DEBUG }, 19:12:00 DEBUG }, 19:12:00 DEBUG "value": [ 19:12:00 DEBUG "value": [ 19:12:00 DEBUG 1734462720, 19:12:00 DEBUG 1734462720, 19:12:00 DEBUG "696766464" 19:12:00 DEBUG "76812.93333333335" 19:12:00 DEBUG ] 19:12:00 DEBUG ] 19:12:00 DEBUG } 19:12:00 DEBUG } 19:12:00 DEBUG ] 19:12:00 DEBUG ] 19:12:00 DEBUG } 19:12:00 DEBUG } 19:12:00 DEBUG } 19:12:00 DEBUG } 19:12:03 INFO 19:12:03 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 19:12:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:12:03 INFO [loop_until]: OK (rc = 0) 19:12:03 DEBUG --- stdout --- 19:12:03 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 546Mi ds-idrepo-0 7m 741Mi rcs-5d696ff5b-7ttvr 3m 297Mi rcs-5d696ff5b-wwlbr 3m 331Mi 19:12:03 DEBUG --- stderr --- 19:12:03 DEBUG 19:12:04 INFO 19:12:04 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 19:12:04 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:12:04 INFO [loop_until]: OK (rc = 0) 19:12:04 DEBUG --- stdout --- 19:12:04 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 96m 0% 2263Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1812m 5% 3656Mi 12% 19:12:04 DEBUG --- stderr --- 19:12:04 DEBUG 19:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462779" 19:12:59 INFO [http_cmd]: http status code OK 19:12:59 DEBUG --- status code --- 19:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:12:59 DEBUG --- http response --- 19:12:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734462779" 19:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462779" 19:12:59 INFO [http_cmd]: http status code OK 19:12:59 DEBUG --- status code --- 19:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:12:59 DEBUG --- http response --- 19:12:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:12:59 INFO [http_cmd]: http status code OK 19:12:59 DEBUG --- status code --- 19:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:12:59 DEBUG --- http response --- 19:12:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:13:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462780" 19:13:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462780" 19:13:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462780" 19:13:00 INFO [http_cmd]: http status code OK 19:13:00 DEBUG --- status code --- 19:13:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:00 INFO [http_cmd]: http status code OK 19:13:00 DEBUG --- http response --- 19:13:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462780" 19:13:00 DEBUG --- status code --- 19:13:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:13:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462780" 19:13:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462780" 19:13:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462780" 19:13:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462780" 19:13:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462780" 19:13:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462780" 19:13:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734462780" 19:13:00 DEBUG --- http response --- 19:13:00 INFO [http_cmd]: http status code OK 19:13:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462780" 19:13:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:13:00 DEBUG --- status code --- 19:13:00 INFO [http_cmd]: http status code OK 19:13:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:00 DEBUG --- status code --- 19:13:00 DEBUG --- http response --- 19:13:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:13:00 DEBUG --- http response --- 19:13:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:13:00 INFO [http_cmd]: http status code OK 19:13:00 DEBUG --- status code --- 19:13:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:00 DEBUG --- http response --- 19:13:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:13:00 INFO [http_cmd]: http status code OK 19:13:00 DEBUG --- status code --- 19:13:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:00 DEBUG --- http response --- 19:13:00 INFO [http_cmd]: http status code OK 19:13:00 INFO [http_cmd]: http status code OK 19:13:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:13:00 DEBUG --- status code --- 19:13:00 DEBUG --- status code --- 19:13:00 INFO [http_cmd]: http status code OK 19:13:00 INFO [http_cmd]: http status code OK 19:13:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:00 INFO [http_cmd]: http status code OK 19:13:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:00 ERROR [http_cmd]: ERROR 19:13:00 DEBUG --- status code --- 19:13:00 DEBUG --- status code --- 19:13:00 DEBUG --- http response --- 19:13:00 DEBUG --- status code --- 19:13:00 DEBUG --- http response --- 19:13:00 ERROR --- status code --- 19:13:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:13:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:13:00 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:13:00 DEBUG --- http response --- 19:13:00 DEBUG --- http response --- 19:13:00 DEBUG --- http response --- 19:13:00 ERROR --- http response --- 19:13:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:13:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:13:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:13:00 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:13:00 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:13:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462780" 19:13:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462780" 19:13:00 INFO [http_cmd]: http status code OK 19:13:00 DEBUG --- status code --- 19:13:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:00 DEBUG --- http response --- 19:13:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:13:00 INFO [http_cmd]: http status code OK 19:13:00 DEBUG --- status code --- 19:13:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:00 DEBUG --- http response --- 19:13:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:13:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462780" 19:13:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462780" 19:13:00 INFO [http_cmd]: http status code OK 19:13:00 DEBUG --- status code --- 19:13:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:00 DEBUG --- http response --- 19:13:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:13:00 INFO [http_cmd]: http status code OK 19:13:00 DEBUG --- status code --- 19:13:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:00 DEBUG --- http response --- 19:13:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:13:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462780" 19:13:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734462780" 19:13:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734462780" 19:13:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462780" 19:13:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462780" 19:13:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462780" 19:13:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734462780" 19:13:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462780" 19:13:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462780" 19:13:00 INFO [http_cmd]: http status code OK 19:13:00 DEBUG --- status code --- 19:13:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:00 DEBUG --- http response --- 19:13:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:13:00 INFO [http_cmd]: http status code OK 19:13:00 DEBUG --- status code --- 19:13:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:00 DEBUG --- http response --- 19:13:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:13:00 INFO [http_cmd]: http status code OK 19:13:00 INFO [http_cmd]: http status code OK 19:13:00 DEBUG --- status code --- 19:13:00 DEBUG --- status code --- 19:13:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:00 INFO [http_cmd]: http status code OK 19:13:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:00 INFO [http_cmd]: http status code OK 19:13:00 DEBUG --- http response --- 19:13:00 DEBUG --- status code --- 19:13:00 DEBUG --- http response --- 19:13:00 DEBUG --- status code --- 19:13:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:13:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:00 INFO [http_cmd]: http status code OK 19:13:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:13:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:00 DEBUG --- http response --- 19:13:00 DEBUG --- status code --- 19:13:00 DEBUG --- http response --- 19:13:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:13:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:13:00 DEBUG --- http response --- 19:13:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:13:00 INFO [http_cmd]: http status code OK 19:13:00 DEBUG --- status code --- 19:13:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:00 DEBUG --- http response --- 19:13:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:13:00 DEBUG ----- output ----- 19:13:00 DEBUG { 19:13:00 DEBUG "status": "success", 19:13:00 DEBUG "data": { 19:13:00 DEBUG "resultType": "vector", 19:13:00 DEBUG "result": [ 19:13:00 DEBUG { 19:13:00 DEBUG "metric": { 19:13:00 DEBUG "pod": "ds-cts-0" 19:13:00 DEBUG }, 19:13:00 DEBUG "value": [ 19:13:00 DEBUG 19:13:00 DEBUG 19:13:00 DEBUG 19:13:00 DEBUG [...] 19:13:00 DEBUG 19:13:00 DEBUG 19:13:00 DEBUG 19:13:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:13:00 DEBUG }, 19:13:00 DEBUG "value": [ 19:13:00 DEBUG 1734462780, 19:13:00 INFO [http_cmd]: http status code OK 19:13:00 DEBUG "0.0026621472779109" 19:13:00 DEBUG --- status code --- 19:13:00 DEBUG ] 19:13:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:00 DEBUG } 19:13:00 DEBUG --- http response --- 19:13:00 DEBUG ] 19:13:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:13:00 DEBUG } 19:13:00 DEBUG } 19:13:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462780" 19:13:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462780" 19:13:00 INFO [http_cmd]: http status code OK 19:13:00 DEBUG --- status code --- 19:13:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:00 DEBUG --- http response --- 19:13:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:13:00 DEBUG ----- output ----- 19:13:00 DEBUG { 19:13:00 DEBUG "status": "success", 19:13:00 DEBUG "data": { 19:13:00 DEBUG "resultType": "vector", 19:13:00 DEBUG "result": [ 19:13:00 DEBUG { 19:13:00 DEBUG "metric": { 19:13:00 DEBUG "instance": "10.142.0.11:9100" 19:13:00 DEBUG }, 19:13:00 DEBUG "value": [ 19:13:00 DEBUG 19:13:00 DEBUG 19:13:00 DEBUG 19:13:00 DEBUG [...] 19:13:00 DEBUG 19:13:00 DEBUG 19:13:00 DEBUG 19:13:00 DEBUG "instance": "10.142.0.51:9100" 19:13:00 DEBUG }, 19:13:00 DEBUG "value": [ 19:13:00 DEBUG 1734462780, 19:13:00 DEBUG "88172.26666666668" 19:13:00 DEBUG ] 19:13:00 DEBUG } 19:13:00 DEBUG ] 19:13:00 DEBUG } 19:13:00 DEBUG } 19:13:00 INFO [http_cmd]: http status code OK 19:13:00 DEBUG --- status code --- 19:13:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:00 DEBUG --- http response --- 19:13:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:13:00 DEBUG ----- output ----- 19:13:00 DEBUG { 19:13:00 DEBUG "status": "success", 19:13:00 DEBUG "data": { 19:13:00 DEBUG "resultType": "vector", 19:13:00 DEBUG "result": [ 19:13:00 DEBUG { 19:13:00 DEBUG "metric": { 19:13:00 DEBUG "instance": "10.142.0.11:9100" 19:13:00 DEBUG }, 19:13:00 DEBUG "value": [ 19:13:00 DEBUG 19:13:00 DEBUG 19:13:00 DEBUG 19:13:00 DEBUG [...] 19:13:00 DEBUG 19:13:00 DEBUG 19:13:00 DEBUG 19:13:00 DEBUG "instance": "10.142.0.51:9100" 19:13:00 DEBUG }, 19:13:00 DEBUG "value": [ 19:13:00 DEBUG 1734462780, 19:13:00 DEBUG "0" 19:13:00 DEBUG ] 19:13:00 DEBUG } 19:13:00 DEBUG ] 19:13:00 DEBUG } 19:13:00 DEBUG } 19:13:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462780" 19:13:00 INFO [http_cmd]: http status code OK 19:13:00 DEBUG --- status code --- 19:13:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:00 DEBUG --- http response --- 19:13:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:13:00 DEBUG ----- output ----- 19:13:00 DEBUG { 19:13:00 DEBUG "status": "success", 19:13:00 DEBUG "data": { 19:13:00 DEBUG "resultType": "vector", 19:13:00 DEBUG "result": [ 19:13:00 DEBUG { 19:13:00 DEBUG "metric": { 19:13:00 DEBUG "instance": "10.142.0.11:9100" 19:13:00 DEBUG }, 19:13:00 DEBUG "value": [ 19:13:00 DEBUG 19:13:00 DEBUG 19:13:00 DEBUG 19:13:00 DEBUG [...] 19:13:00 DEBUG 19:13:00 DEBUG 19:13:00 DEBUG 19:13:00 DEBUG "instance": "10.142.0.51:9100" 19:13:00 DEBUG }, 19:13:00 DEBUG "value": [ 19:13:00 DEBUG 1734462780, 19:13:00 DEBUG "0.00040000000000238375" 19:13:00 DEBUG ] 19:13:00 DEBUG } 19:13:00 DEBUG ] 19:13:00 DEBUG } 19:13:00 DEBUG } 19:13:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462780" 19:13:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462780" 19:13:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734462780" 19:13:00 INFO [http_cmd]: http status code OK 19:13:00 DEBUG --- status code --- 19:13:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:00 DEBUG --- http response --- 19:13:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:13:00 DEBUG ----- output ----- 19:13:00 DEBUG { 19:13:00 DEBUG "status": "success", 19:13:00 DEBUG "data": { 19:13:00 DEBUG "resultType": "vector", 19:13:00 DEBUG "result": [ 19:13:00 DEBUG { 19:13:00 DEBUG "metric": { 19:13:00 DEBUG "pod": "ds-cts-0" 19:13:00 DEBUG }, 19:13:00 DEBUG "value": [ 19:13:00 DEBUG 19:13:00 DEBUG 19:13:00 DEBUG 19:13:00 DEBUG [...] 19:13:00 DEBUG 19:13:00 DEBUG 19:13:00 DEBUG 19:13:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:13:00 DEBUG }, 19:13:00 DEBUG "value": [ 19:13:00 DEBUG 1734462780, 19:13:00 DEBUG "0" 19:13:00 DEBUG ] 19:13:00 DEBUG } 19:13:00 DEBUG ] 19:13:00 DEBUG } 19:13:00 DEBUG } 19:13:00 INFO [http_cmd]: http status code OK 19:13:00 DEBUG --- status code --- 19:13:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:00 DEBUG --- http response --- 19:13:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:13:00 DEBUG ----- output ----- 19:13:00 DEBUG { 19:13:00 DEBUG "status": "success", 19:13:00 DEBUG "data": { 19:13:00 DEBUG "resultType": "vector", 19:13:00 DEBUG "result": [ 19:13:00 DEBUG { 19:13:00 DEBUG "metric": { 19:13:00 DEBUG "pod": "ds-cts-0" 19:13:00 DEBUG }, 19:13:00 DEBUG "value": [ 19:13:00 DEBUG 19:13:00 DEBUG 19:13:00 DEBUG 19:13:00 DEBUG [...] 19:13:00 DEBUG 19:13:00 DEBUG 19:13:00 DEBUG 19:13:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:13:00 DEBUG }, 19:13:00 DEBUG "value": [ 19:13:00 DEBUG 1734462780, 19:13:00 DEBUG "576.0900140646976" 19:13:00 DEBUG ] 19:13:00 DEBUG } 19:13:00 DEBUG ] 19:13:00 DEBUG } 19:13:00 DEBUG } 19:13:00 INFO [http_cmd]: http status code OK 19:13:00 DEBUG --- status code --- 19:13:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:00 DEBUG --- http response --- 19:13:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:13:00 DEBUG ----- output ----- 19:13:00 DEBUG { 19:13:00 DEBUG "status": "success", 19:13:00 DEBUG "data": { 19:13:00 DEBUG "resultType": "vector", 19:13:00 DEBUG "result": [ 19:13:00 DEBUG { 19:13:00 DEBUG "metric": { 19:13:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 19:13:00 DEBUG }, 19:13:00 DEBUG "value": [ 19:13:00 DEBUG 19:13:00 DEBUG 19:13:00 DEBUG 19:13:00 DEBUG [...] 19:13:00 DEBUG 19:13:00 DEBUG 19:13:00 DEBUG 19:13:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 19:13:00 DEBUG }, 19:13:00 DEBUG "value": [ 19:13:00 DEBUG 1734462780, 19:13:00 DEBUG "2558816256" 19:13:00 DEBUG ] 19:13:00 DEBUG } 19:13:00 DEBUG ] 19:13:00 DEBUG } 19:13:00 DEBUG } 19:13:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734462780" 19:13:00 INFO [http_cmd]: http status code OK 19:13:00 DEBUG --- status code --- 19:13:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:00 DEBUG --- http response --- 19:13:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:13:00 DEBUG ----- output ----- 19:13:00 DEBUG { 19:13:00 DEBUG "status": "success", 19:13:00 DEBUG "data": { 19:13:00 DEBUG "resultType": "vector", 19:13:00 DEBUG "result": [ 19:13:00 DEBUG { 19:13:00 DEBUG "metric": { 19:13:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 19:13:00 DEBUG }, 19:13:00 DEBUG "value": [ 19:13:00 DEBUG 19:13:00 DEBUG 19:13:00 DEBUG 19:13:00 DEBUG [...] 19:13:00 DEBUG 19:13:00 DEBUG 19:13:00 DEBUG 19:13:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 19:13:00 DEBUG }, 19:13:00 DEBUG "value": [ 19:13:00 DEBUG 1734462780, 19:13:00 DEBUG "1.7605000554832577" 19:13:00 DEBUG ] 19:13:00 DEBUG } 19:13:00 DEBUG ] 19:13:00 DEBUG } 19:13:00 DEBUG } 19:13:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462780" 19:13:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462780" 19:13:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462780" 19:13:00 INFO [http_cmd]: http status code OK 19:13:00 DEBUG --- status code --- 19:13:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:00 DEBUG --- http response --- 19:13:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:13:00 DEBUG ----- output ----- 19:13:00 DEBUG { 19:13:00 DEBUG "status": "success", 19:13:00 DEBUG "data": { 19:13:00 DEBUG "resultType": "vector", 19:13:00 DEBUG "result": [ 19:13:00 DEBUG { 19:13:00 DEBUG "metric": { 19:13:00 DEBUG "instance": "10.142.0.11:9100" 19:13:00 DEBUG }, 19:13:00 DEBUG "value": [ 19:13:00 DEBUG 19:13:00 DEBUG 19:13:00 DEBUG 19:13:00 DEBUG [...] 19:13:00 DEBUG 19:13:00 DEBUG 19:13:00 DEBUG 19:13:00 DEBUG "instance": "10.142.0.51:9100" 19:13:00 DEBUG }, 19:13:00 DEBUG "value": [ 19:13:00 DEBUG 1734462780, 19:13:00 DEBUG "18022.400000000005" 19:13:00 DEBUG ] 19:13:00 DEBUG } 19:13:00 DEBUG ] 19:13:00 DEBUG } 19:13:00 DEBUG } 19:13:00 INFO [http_cmd]: http status code OK 19:13:00 DEBUG --- status code --- 19:13:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:00 DEBUG --- http response --- 19:13:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:13:00 DEBUG ----- output ----- 19:13:00 DEBUG { 19:13:00 DEBUG "status": "success", 19:13:00 DEBUG "data": { 19:13:00 INFO [http_cmd]: http status code OK 19:13:00 DEBUG "resultType": "vector", 19:13:00 DEBUG --- status code --- 19:13:00 DEBUG "result": [ 19:13:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:00 DEBUG { 19:13:00 DEBUG --- http response --- 19:13:00 DEBUG "metric": { 19:13:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:13:00 DEBUG "pod": "ds-cts-0" 19:13:00 DEBUG ----- output ----- 19:13:00 DEBUG }, 19:13:00 DEBUG { 19:13:00 DEBUG "value": [ 19:13:00 DEBUG "status": "success", 19:13:00 DEBUG 19:13:00 DEBUG "data": { 19:13:00 DEBUG 19:13:00 DEBUG "resultType": "vector", 19:13:00 DEBUG 19:13:00 DEBUG "result": [ 19:13:00 DEBUG [...] 19:13:00 DEBUG { 19:13:00 DEBUG 19:13:00 DEBUG "metric": { 19:13:00 DEBUG 19:13:00 DEBUG "instance": "10.142.0.11:9100" 19:13:00 DEBUG 19:13:00 DEBUG }, 19:13:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:13:00 DEBUG "value": [ 19:13:00 DEBUG }, 19:13:00 DEBUG 19:13:00 DEBUG "value": [ 19:13:00 DEBUG 19:13:00 DEBUG 1734462780, 19:13:00 DEBUG 19:13:00 DEBUG "696934400" 19:13:00 DEBUG [...] 19:13:00 DEBUG ] 19:13:00 DEBUG 19:13:00 DEBUG } 19:13:00 DEBUG 19:13:00 DEBUG ] 19:13:00 DEBUG 19:13:00 DEBUG } 19:13:00 DEBUG "instance": "10.142.0.51:9100" 19:13:00 DEBUG } 19:13:00 DEBUG }, 19:13:00 DEBUG "value": [ 19:13:00 DEBUG 1734462780, 19:13:00 DEBUG "72229.36666666667" 19:13:00 DEBUG ] 19:13:00 DEBUG } 19:13:00 DEBUG ] 19:13:00 DEBUG } 19:13:00 DEBUG } 19:13:03 INFO 19:13:03 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 19:13:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:13:03 INFO [loop_until]: OK (rc = 0) 19:13:03 DEBUG --- stdout --- 19:13:03 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 545Mi ds-idrepo-0 7m 741Mi rcs-5d696ff5b-7ttvr 3m 297Mi rcs-5d696ff5b-wwlbr 2m 332Mi 19:13:03 DEBUG --- stderr --- 19:13:03 DEBUG 19:13:04 INFO 19:13:04 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 19:13:04 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:13:04 INFO [loop_until]: OK (rc = 0) 19:13:04 DEBUG --- stdout --- 19:13:04 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 95m 0% 2285Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1826m 5% 3656Mi 12% 19:13:04 DEBUG --- stderr --- 19:13:04 DEBUG 19:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462839" 19:13:59 INFO [http_cmd]: http status code OK 19:13:59 DEBUG --- status code --- 19:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:59 DEBUG --- http response --- 19:13:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462839" 19:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734462839" 19:13:59 INFO [http_cmd]: http status code OK 19:13:59 INFO [http_cmd]: http status code OK 19:13:59 DEBUG --- status code --- 19:13:59 DEBUG --- status code --- 19:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:13:59 DEBUG --- http response --- 19:13:59 DEBUG --- http response --- 19:13:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:13:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:14:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462840" 19:14:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462840" 19:14:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462840" 19:14:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462840" 19:14:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462840" 19:14:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462840" 19:14:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462840" 19:14:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462840" 19:14:00 INFO [http_cmd]: http status code OK 19:14:00 DEBUG --- status code --- 19:14:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:00 DEBUG --- http response --- 19:14:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:14:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462840" 19:14:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734462840" 19:14:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462840" 19:14:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462840" 19:14:00 INFO [http_cmd]: http status code OK 19:14:00 INFO [http_cmd]: http status code OK 19:14:00 DEBUG --- status code --- 19:14:00 DEBUG --- status code --- 19:14:00 INFO [http_cmd]: http status code OK 19:14:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:00 DEBUG --- status code --- 19:14:00 DEBUG --- http response --- 19:14:00 DEBUG --- http response --- 19:14:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:14:00 INFO [http_cmd]: http status code OK 19:14:00 DEBUG --- http response --- 19:14:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:14:00 DEBUG --- status code --- 19:14:00 INFO [http_cmd]: http status code OK 19:14:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:14:00 INFO [http_cmd]: http status code OK 19:14:00 INFO [http_cmd]: http status code OK 19:14:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:00 INFO [http_cmd]: http status code OK 19:14:00 DEBUG --- status code --- 19:14:00 DEBUG --- status code --- 19:14:00 ERROR [http_cmd]: ERROR 19:14:00 ERROR --- status code --- 19:14:00 DEBUG --- status code --- 19:14:00 DEBUG --- http response --- 19:14:00 DEBUG --- status code --- 19:14:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:00 INFO [http_cmd]: http status code OK 19:14:00 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:14:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:14:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:00 DEBUG --- http response --- 19:14:00 INFO [http_cmd]: http status code OK 19:14:00 DEBUG --- http response --- 19:14:00 DEBUG --- status code --- 19:14:00 ERROR --- http response --- 19:14:00 DEBUG --- http response --- 19:14:00 DEBUG --- http response --- 19:14:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:14:00 DEBUG --- status code --- 19:14:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:00 DEBUG --- http response --- 19:14:00 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:14:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:14:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:14:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:14:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:14:00 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:14:00 DEBUG --- http response --- 19:14:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:14:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462840" 19:14:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462840" 19:14:00 INFO [http_cmd]: http status code OK 19:14:00 DEBUG --- status code --- 19:14:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:00 DEBUG --- http response --- 19:14:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:14:00 INFO [http_cmd]: http status code OK 19:14:00 DEBUG --- status code --- 19:14:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:00 DEBUG --- http response --- 19:14:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:14:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462840" 19:14:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462840" 19:14:00 INFO [http_cmd]: http status code OK 19:14:00 DEBUG --- status code --- 19:14:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:00 DEBUG --- http response --- 19:14:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:14:00 INFO [http_cmd]: http status code OK 19:14:00 DEBUG --- status code --- 19:14:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:00 DEBUG --- http response --- 19:14:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:14:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462840" 19:14:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734462840" 19:14:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462840" 19:14:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462840" 19:14:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462840" 19:14:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462840" 19:14:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734462840" 19:14:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734462840" 19:14:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462840" 19:14:00 INFO [http_cmd]: http status code OK 19:14:00 DEBUG --- status code --- 19:14:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:00 DEBUG --- http response --- 19:14:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:14:00 INFO [http_cmd]: http status code OK 19:14:00 DEBUG --- status code --- 19:14:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:00 DEBUG --- http response --- 19:14:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:14:00 INFO [http_cmd]: http status code OK 19:14:00 DEBUG --- status code --- 19:14:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:00 DEBUG --- http response --- 19:14:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:14:00 INFO [http_cmd]: http status code OK 19:14:00 DEBUG --- status code --- 19:14:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:00 DEBUG --- http response --- 19:14:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:14:00 INFO [http_cmd]: http status code OK 19:14:00 INFO [http_cmd]: http status code OK 19:14:00 DEBUG --- status code --- 19:14:00 DEBUG --- status code --- 19:14:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:00 DEBUG --- http response --- 19:14:00 DEBUG --- http response --- 19:14:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:14:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:14:00 INFO [http_cmd]: http status code OK 19:14:00 DEBUG --- status code --- 19:14:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:00 DEBUG --- http response --- 19:14:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:14:00 INFO [http_cmd]: http status code OK 19:14:00 DEBUG --- status code --- 19:14:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:00 DEBUG --- http response --- 19:14:00 INFO [http_cmd]: http status code OK 19:14:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:14:00 DEBUG --- status code --- 19:14:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:00 DEBUG --- http response --- 19:14:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:14:00 DEBUG ----- output ----- 19:14:00 DEBUG { 19:14:00 DEBUG "status": "success", 19:14:00 DEBUG "data": { 19:14:00 DEBUG "resultType": "vector", 19:14:00 DEBUG "result": [ 19:14:00 DEBUG { 19:14:00 DEBUG "metric": { 19:14:00 DEBUG "pod": "ds-cts-0" 19:14:00 DEBUG }, 19:14:00 DEBUG "value": [ 19:14:00 DEBUG 19:14:00 DEBUG 19:14:00 DEBUG 19:14:00 DEBUG [...] 19:14:00 DEBUG 19:14:00 DEBUG 19:14:00 DEBUG 19:14:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:14:00 DEBUG }, 19:14:00 DEBUG "value": [ 19:14:00 DEBUG 1734462840, 19:14:00 DEBUG "0.002402767829762206" 19:14:00 DEBUG ] 19:14:00 DEBUG } 19:14:00 DEBUG ] 19:14:00 DEBUG } 19:14:00 DEBUG } 19:14:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462840" 19:14:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462840" 19:14:00 INFO [http_cmd]: http status code OK 19:14:00 DEBUG --- status code --- 19:14:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:00 DEBUG --- http response --- 19:14:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:14:00 DEBUG ----- output ----- 19:14:00 DEBUG { 19:14:00 DEBUG "status": "success", 19:14:00 DEBUG "data": { 19:14:00 DEBUG "resultType": "vector", 19:14:00 DEBUG "result": [ 19:14:00 DEBUG { 19:14:00 DEBUG "metric": { 19:14:00 DEBUG "instance": "10.142.0.11:9100" 19:14:00 DEBUG }, 19:14:00 DEBUG "value": [ 19:14:00 DEBUG 19:14:00 DEBUG 19:14:00 DEBUG 19:14:00 DEBUG [...] 19:14:00 DEBUG 19:14:00 DEBUG 19:14:00 DEBUG 19:14:00 DEBUG "instance": "10.142.0.51:9100" 19:14:00 DEBUG }, 19:14:00 DEBUG "value": [ 19:14:00 DEBUG 1734462840, 19:14:00 DEBUG "89402.6666666667" 19:14:00 DEBUG ] 19:14:00 DEBUG } 19:14:00 DEBUG ] 19:14:00 DEBUG } 19:14:00 DEBUG } 19:14:00 INFO [http_cmd]: http status code OK 19:14:00 DEBUG --- status code --- 19:14:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:00 DEBUG --- http response --- 19:14:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:14:00 DEBUG ----- output ----- 19:14:00 DEBUG { 19:14:00 DEBUG "status": "success", 19:14:00 DEBUG "data": { 19:14:00 DEBUG "resultType": "vector", 19:14:00 DEBUG "result": [ 19:14:00 DEBUG { 19:14:00 DEBUG "metric": { 19:14:00 DEBUG "instance": "10.142.0.11:9100" 19:14:00 DEBUG }, 19:14:00 DEBUG "value": [ 19:14:00 DEBUG 19:14:00 DEBUG 19:14:00 DEBUG 19:14:00 DEBUG [...] 19:14:00 DEBUG 19:14:00 DEBUG 19:14:00 DEBUG 19:14:00 DEBUG "instance": "10.142.0.51:9100" 19:14:00 DEBUG }, 19:14:00 DEBUG "value": [ 19:14:00 DEBUG 1734462840, 19:14:00 DEBUG "0" 19:14:00 DEBUG ] 19:14:00 DEBUG } 19:14:00 DEBUG ] 19:14:00 DEBUG } 19:14:00 DEBUG } 19:14:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462840" 19:14:00 INFO [http_cmd]: http status code OK 19:14:00 DEBUG --- status code --- 19:14:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:00 DEBUG --- http response --- 19:14:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:14:00 DEBUG ----- output ----- 19:14:00 DEBUG { 19:14:00 DEBUG "status": "success", 19:14:00 DEBUG "data": { 19:14:00 DEBUG "resultType": "vector", 19:14:00 DEBUG "result": [ 19:14:00 DEBUG { 19:14:00 DEBUG "metric": { 19:14:00 DEBUG "instance": "10.142.0.11:9100" 19:14:00 DEBUG }, 19:14:00 DEBUG "value": [ 19:14:00 DEBUG 19:14:00 DEBUG 19:14:00 DEBUG 19:14:00 DEBUG [...] 19:14:00 DEBUG 19:14:00 DEBUG 19:14:00 DEBUG 19:14:00 DEBUG "instance": "10.142.0.51:9100" 19:14:00 DEBUG }, 19:14:00 DEBUG "value": [ 19:14:00 DEBUG 1734462840, 19:14:00 DEBUG "0.0005333333333335115" 19:14:00 DEBUG ] 19:14:00 DEBUG } 19:14:00 DEBUG ] 19:14:00 DEBUG } 19:14:00 DEBUG } 19:14:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462840" 19:14:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462840" 19:14:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734462840" 19:14:00 INFO [http_cmd]: http status code OK 19:14:00 DEBUG --- status code --- 19:14:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:00 DEBUG --- http response --- 19:14:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:14:00 DEBUG ----- output ----- 19:14:00 DEBUG { 19:14:00 DEBUG "status": "success", 19:14:00 DEBUG "data": { 19:14:00 DEBUG "resultType": "vector", 19:14:00 DEBUG "result": [ 19:14:00 DEBUG { 19:14:00 DEBUG "metric": { 19:14:00 DEBUG "pod": "ds-cts-0" 19:14:00 DEBUG }, 19:14:00 DEBUG "value": [ 19:14:00 DEBUG 19:14:00 DEBUG 19:14:00 DEBUG 19:14:00 DEBUG [...] 19:14:00 DEBUG 19:14:00 DEBUG 19:14:00 DEBUG 19:14:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:14:00 DEBUG }, 19:14:00 DEBUG "value": [ 19:14:00 DEBUG 1734462840, 19:14:00 DEBUG "0" 19:14:00 DEBUG ] 19:14:00 DEBUG } 19:14:00 DEBUG ] 19:14:00 DEBUG } 19:14:00 DEBUG } 19:14:00 INFO [http_cmd]: http status code OK 19:14:00 DEBUG --- status code --- 19:14:00 INFO [http_cmd]: http status code OK 19:14:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:00 DEBUG --- status code --- 19:14:00 DEBUG --- http response --- 19:14:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:14:00 DEBUG --- http response --- 19:14:00 DEBUG ----- output ----- 19:14:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:14:00 DEBUG { 19:14:00 DEBUG ----- output ----- 19:14:00 DEBUG "status": "success", 19:14:00 DEBUG { 19:14:00 DEBUG "data": { 19:14:00 DEBUG "status": "success", 19:14:00 DEBUG "resultType": "vector", 19:14:00 DEBUG "data": { 19:14:00 DEBUG "result": [ 19:14:00 DEBUG "resultType": "vector", 19:14:00 DEBUG { 19:14:00 DEBUG "result": [ 19:14:00 DEBUG "metric": { 19:14:00 DEBUG { 19:14:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 19:14:00 DEBUG "metric": { 19:14:00 DEBUG }, 19:14:00 DEBUG "pod": "ds-cts-0" 19:14:00 DEBUG "value": [ 19:14:00 DEBUG }, 19:14:00 DEBUG 19:14:00 DEBUG "value": [ 19:14:00 DEBUG 19:14:00 DEBUG 19:14:00 DEBUG 19:14:00 DEBUG 19:14:00 DEBUG [...] 19:14:00 DEBUG 19:14:00 DEBUG 19:14:00 DEBUG [...] 19:14:00 DEBUG 19:14:00 DEBUG 19:14:00 DEBUG 19:14:00 DEBUG 19:14:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 19:14:00 DEBUG 19:14:00 DEBUG }, 19:14:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:14:00 DEBUG "value": [ 19:14:00 DEBUG }, 19:14:00 DEBUG 1734462840, 19:14:00 DEBUG "value": [ 19:14:00 DEBUG "2562924544" 19:14:00 DEBUG 1734462840, 19:14:00 DEBUG ] 19:14:00 DEBUG "1435.2521051979368" 19:14:00 DEBUG } 19:14:00 DEBUG ] 19:14:00 DEBUG ] 19:14:00 DEBUG } 19:14:00 DEBUG } 19:14:00 DEBUG ] 19:14:00 DEBUG } 19:14:00 DEBUG } 19:14:00 DEBUG } 19:14:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734462840" 19:14:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462840" 19:14:00 INFO [http_cmd]: http status code OK 19:14:00 DEBUG --- status code --- 19:14:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:00 DEBUG --- http response --- 19:14:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:14:00 DEBUG ----- output ----- 19:14:00 DEBUG { 19:14:00 DEBUG "status": "success", 19:14:00 DEBUG "data": { 19:14:00 DEBUG "resultType": "vector", 19:14:00 DEBUG "result": [ 19:14:00 DEBUG { 19:14:00 DEBUG "metric": { 19:14:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 19:14:00 DEBUG }, 19:14:00 DEBUG "value": [ 19:14:00 DEBUG 19:14:00 DEBUG 19:14:00 DEBUG 19:14:00 DEBUG [...] 19:14:00 DEBUG 19:14:00 DEBUG 19:14:00 DEBUG 19:14:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 19:14:00 DEBUG }, 19:14:00 DEBUG "value": [ 19:14:00 DEBUG 1734462840, 19:14:00 DEBUG "1.7603288418874317" 19:14:00 DEBUG ] 19:14:00 DEBUG } 19:14:00 DEBUG ] 19:14:00 DEBUG } 19:14:00 DEBUG } 19:14:00 INFO [http_cmd]: http status code OK 19:14:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462840" 19:14:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462840" 19:14:00 DEBUG --- status code --- 19:14:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:00 DEBUG --- http response --- 19:14:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:14:00 DEBUG ----- output ----- 19:14:00 DEBUG { 19:14:00 DEBUG "status": "success", 19:14:00 DEBUG "data": { 19:14:00 DEBUG "resultType": "vector", 19:14:00 DEBUG "result": [ 19:14:00 DEBUG { 19:14:00 DEBUG "metric": { 19:14:00 DEBUG "instance": "10.142.0.11:9100" 19:14:00 DEBUG }, 19:14:00 DEBUG "value": [ 19:14:00 DEBUG 19:14:00 DEBUG 19:14:00 DEBUG 19:14:00 DEBUG [...] 19:14:00 DEBUG 19:14:00 DEBUG 19:14:00 DEBUG 19:14:00 DEBUG "instance": "10.142.0.51:9100" 19:14:00 DEBUG }, 19:14:00 DEBUG "value": [ 19:14:00 DEBUG 1734462840, 19:14:00 DEBUG "33314.13333333334" 19:14:00 DEBUG ] 19:14:00 DEBUG } 19:14:00 DEBUG ] 19:14:00 DEBUG } 19:14:00 DEBUG } 19:14:00 INFO [http_cmd]: http status code OK 19:14:00 DEBUG --- status code --- 19:14:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:00 INFO [http_cmd]: http status code OK 19:14:00 DEBUG --- http response --- 19:14:00 DEBUG --- status code --- 19:14:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:14:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:00 DEBUG ----- output ----- 19:14:00 DEBUG --- http response --- 19:14:00 DEBUG { 19:14:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:14:00 DEBUG "status": "success", 19:14:00 DEBUG ----- output ----- 19:14:00 DEBUG "data": { 19:14:00 DEBUG { 19:14:00 DEBUG "resultType": "vector", 19:14:00 DEBUG "status": "success", 19:14:00 DEBUG "result": [ 19:14:00 DEBUG "data": { 19:14:00 DEBUG { 19:14:00 DEBUG "resultType": "vector", 19:14:00 DEBUG "metric": { 19:14:00 DEBUG "result": [ 19:14:00 DEBUG "pod": "ds-cts-0" 19:14:00 DEBUG { 19:14:00 DEBUG }, 19:14:00 DEBUG "metric": { 19:14:00 DEBUG "value": [ 19:14:00 DEBUG "instance": "10.142.0.11:9100" 19:14:00 DEBUG 19:14:00 DEBUG }, 19:14:00 DEBUG 19:14:00 DEBUG "value": [ 19:14:00 DEBUG 19:14:00 DEBUG 19:14:00 DEBUG [...] 19:14:00 DEBUG 19:14:00 DEBUG 19:14:00 DEBUG 19:14:00 DEBUG 19:14:00 DEBUG [...] 19:14:00 DEBUG 19:14:00 DEBUG 19:14:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:14:00 DEBUG 19:14:00 DEBUG }, 19:14:00 DEBUG 19:14:00 DEBUG "value": [ 19:14:00 DEBUG "instance": "10.142.0.51:9100" 19:14:00 DEBUG 1734462840, 19:14:00 DEBUG }, 19:14:00 DEBUG "696930304" 19:14:00 DEBUG "value": [ 19:14:00 DEBUG ] 19:14:00 DEBUG 1734462840, 19:14:00 DEBUG } 19:14:00 DEBUG "75654.06666666668" 19:14:00 DEBUG ] 19:14:00 DEBUG ] 19:14:00 DEBUG } 19:14:00 DEBUG } 19:14:00 DEBUG } 19:14:00 DEBUG ] 19:14:00 DEBUG } 19:14:00 DEBUG } 19:14:03 INFO 19:14:03 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 19:14:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:14:03 INFO [loop_until]: OK (rc = 0) 19:14:03 DEBUG --- stdout --- 19:14:03 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 546Mi ds-idrepo-0 6m 741Mi rcs-5d696ff5b-7ttvr 3m 297Mi rcs-5d696ff5b-wwlbr 2m 332Mi 19:14:03 DEBUG --- stderr --- 19:14:03 DEBUG 19:14:04 INFO 19:14:04 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 19:14:04 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:14:04 INFO [loop_until]: OK (rc = 0) 19:14:04 DEBUG --- stdout --- 19:14:04 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 96m 0% 2284Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1823m 5% 3654Mi 12% 19:14:04 DEBUG --- stderr --- 19:14:04 DEBUG 19:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462899" 19:14:59 INFO [http_cmd]: http status code OK 19:14:59 DEBUG --- status code --- 19:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:14:59 DEBUG --- http response --- 19:14:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734462899" 19:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462899" 19:15:00 INFO [http_cmd]: http status code OK 19:15:00 DEBUG --- status code --- 19:15:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:00 DEBUG --- http response --- 19:15:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:15:00 INFO [http_cmd]: http status code OK 19:15:00 DEBUG --- status code --- 19:15:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:00 DEBUG --- http response --- 19:15:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:15:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462900" 19:15:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462900" 19:15:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462900" 19:15:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462900" 19:15:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462900" 19:15:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462900" 19:15:00 INFO [http_cmd]: http status code OK 19:15:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734462900" 19:15:00 DEBUG --- status code --- 19:15:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462900" 19:15:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462900" 19:15:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462900" 19:15:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:00 DEBUG --- http response --- 19:15:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462900" 19:15:00 INFO [http_cmd]: http status code OK 19:15:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462900" 19:15:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:15:00 INFO [http_cmd]: http status code OK 19:15:00 INFO [http_cmd]: http status code OK 19:15:00 INFO [http_cmd]: http status code OK 19:15:00 DEBUG --- status code --- 19:15:00 DEBUG --- status code --- 19:15:00 DEBUG --- status code --- 19:15:00 DEBUG --- status code --- 19:15:00 INFO [http_cmd]: http status code OK 19:15:00 INFO [http_cmd]: http status code OK 19:15:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:00 ERROR [http_cmd]: ERROR 19:15:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:00 DEBUG --- status code --- 19:15:00 DEBUG --- status code --- 19:15:00 DEBUG --- http response --- 19:15:00 ERROR --- status code --- 19:15:00 DEBUG --- http response --- 19:15:00 DEBUG --- http response --- 19:15:00 DEBUG --- http response --- 19:15:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:15:00 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:15:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:15:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:15:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:15:00 DEBUG --- http response --- 19:15:00 DEBUG --- http response --- 19:15:00 ERROR --- http response --- 19:15:00 INFO [http_cmd]: http status code OK 19:15:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:15:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:15:00 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:15:00 DEBUG --- status code --- 19:15:00 INFO [http_cmd]: http status code OK 19:15:00 INFO [http_cmd]: http status code OK 19:15:00 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:15:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:00 DEBUG --- status code --- 19:15:00 DEBUG --- status code --- 19:15:00 DEBUG --- http response --- 19:15:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:15:00 DEBUG --- http response --- 19:15:00 DEBUG --- http response --- 19:15:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:15:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:15:00 INFO [http_cmd]: http status code OK 19:15:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462900" 19:15:00 DEBUG --- status code --- 19:15:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:00 DEBUG --- http response --- 19:15:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:15:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462900" 19:15:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462900" 19:15:00 INFO [http_cmd]: http status code OK 19:15:00 DEBUG --- status code --- 19:15:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:00 DEBUG --- http response --- 19:15:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:15:00 INFO [http_cmd]: http status code OK 19:15:00 DEBUG --- status code --- 19:15:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:00 DEBUG --- http response --- 19:15:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:15:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462900" 19:15:00 INFO [http_cmd]: http status code OK 19:15:00 DEBUG --- status code --- 19:15:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:00 DEBUG --- http response --- 19:15:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:15:00 INFO [http_cmd]: http status code OK 19:15:00 DEBUG --- status code --- 19:15:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:00 DEBUG --- http response --- 19:15:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:15:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462900" 19:15:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734462900" 19:15:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462900" 19:15:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462900" 19:15:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462900" 19:15:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462900" 19:15:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734462900" 19:15:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734462900" 19:15:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462900" 19:15:00 INFO [http_cmd]: http status code OK 19:15:00 DEBUG --- status code --- 19:15:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:00 DEBUG --- http response --- 19:15:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:15:00 INFO [http_cmd]: http status code OK 19:15:00 DEBUG --- status code --- 19:15:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:00 DEBUG --- http response --- 19:15:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:15:00 INFO [http_cmd]: http status code OK 19:15:00 DEBUG --- status code --- 19:15:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:00 DEBUG --- http response --- 19:15:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:15:00 INFO [http_cmd]: http status code OK 19:15:00 DEBUG --- status code --- 19:15:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:00 DEBUG --- http response --- 19:15:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:15:00 INFO [http_cmd]: http status code OK 19:15:00 DEBUG --- status code --- 19:15:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:00 DEBUG --- http response --- 19:15:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:15:00 INFO [http_cmd]: http status code OK 19:15:00 DEBUG --- status code --- 19:15:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:00 DEBUG --- http response --- 19:15:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:15:00 INFO [http_cmd]: http status code OK 19:15:00 DEBUG --- status code --- 19:15:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:00 DEBUG --- http response --- 19:15:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:15:00 INFO [http_cmd]: http status code OK 19:15:00 DEBUG --- status code --- 19:15:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:00 DEBUG --- http response --- 19:15:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:15:00 DEBUG ----- output ----- 19:15:00 DEBUG { 19:15:00 DEBUG "status": "success", 19:15:00 DEBUG "data": { 19:15:00 DEBUG "resultType": "vector", 19:15:00 DEBUG "result": [ 19:15:00 DEBUG { 19:15:00 DEBUG "metric": { 19:15:00 DEBUG "pod": "ds-cts-0" 19:15:00 DEBUG }, 19:15:00 DEBUG "value": [ 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG [...] 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:15:00 DEBUG }, 19:15:00 DEBUG "value": [ 19:15:00 DEBUG 1734462900, 19:15:00 DEBUG "0.0026200614019692182" 19:15:00 DEBUG ] 19:15:00 DEBUG } 19:15:00 DEBUG ] 19:15:00 DEBUG } 19:15:00 DEBUG } 19:15:00 INFO [http_cmd]: http status code OK 19:15:00 DEBUG --- status code --- 19:15:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:00 DEBUG --- http response --- 19:15:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:15:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462900" 19:15:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462900" 19:15:00 INFO [http_cmd]: http status code OK 19:15:00 DEBUG --- status code --- 19:15:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:00 DEBUG --- http response --- 19:15:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:15:00 DEBUG ----- output ----- 19:15:00 DEBUG { 19:15:00 DEBUG "status": "success", 19:15:00 DEBUG "data": { 19:15:00 DEBUG "resultType": "vector", 19:15:00 DEBUG "result": [ 19:15:00 DEBUG { 19:15:00 DEBUG "metric": { 19:15:00 DEBUG "instance": "10.142.0.11:9100" 19:15:00 DEBUG }, 19:15:00 DEBUG "value": [ 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG [...] 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG "instance": "10.142.0.51:9100" 19:15:00 DEBUG }, 19:15:00 DEBUG "value": [ 19:15:00 DEBUG 1734462900, 19:15:00 DEBUG "88507.90000000001" 19:15:00 DEBUG ] 19:15:00 DEBUG } 19:15:00 DEBUG ] 19:15:00 DEBUG } 19:15:00 DEBUG } 19:15:00 INFO [http_cmd]: http status code OK 19:15:00 DEBUG --- status code --- 19:15:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:00 DEBUG --- http response --- 19:15:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:15:00 DEBUG ----- output ----- 19:15:00 DEBUG { 19:15:00 DEBUG "status": "success", 19:15:00 DEBUG "data": { 19:15:00 DEBUG "resultType": "vector", 19:15:00 DEBUG "result": [ 19:15:00 DEBUG { 19:15:00 DEBUG "metric": { 19:15:00 DEBUG "instance": "10.142.0.11:9100" 19:15:00 DEBUG }, 19:15:00 DEBUG "value": [ 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG [...] 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG "instance": "10.142.0.51:9100" 19:15:00 DEBUG }, 19:15:00 DEBUG "value": [ 19:15:00 DEBUG 1734462900, 19:15:00 DEBUG "0.0003333333333333855" 19:15:00 DEBUG ] 19:15:00 DEBUG } 19:15:00 DEBUG ] 19:15:00 DEBUG } 19:15:00 DEBUG } 19:15:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462900" 19:15:00 INFO [http_cmd]: http status code OK 19:15:00 DEBUG --- status code --- 19:15:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:00 DEBUG --- http response --- 19:15:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:15:00 DEBUG ----- output ----- 19:15:00 DEBUG { 19:15:00 DEBUG "status": "success", 19:15:00 DEBUG "data": { 19:15:00 DEBUG "resultType": "vector", 19:15:00 DEBUG "result": [ 19:15:00 DEBUG { 19:15:00 DEBUG "metric": { 19:15:00 DEBUG "instance": "10.142.0.11:9100" 19:15:00 DEBUG }, 19:15:00 DEBUG "value": [ 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG [...] 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG "instance": "10.142.0.51:9100" 19:15:00 DEBUG }, 19:15:00 DEBUG "value": [ 19:15:00 DEBUG 1734462900, 19:15:00 DEBUG "0.0004333333333320867" 19:15:00 DEBUG ] 19:15:00 DEBUG } 19:15:00 DEBUG ] 19:15:00 DEBUG } 19:15:00 DEBUG } 19:15:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462900" 19:15:00 INFO [http_cmd]: http status code OK 19:15:00 DEBUG --- status code --- 19:15:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:00 DEBUG --- http response --- 19:15:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:15:00 DEBUG ----- output ----- 19:15:00 DEBUG { 19:15:00 DEBUG "status": "success", 19:15:00 DEBUG "data": { 19:15:00 DEBUG "resultType": "vector", 19:15:00 DEBUG "result": [ 19:15:00 DEBUG { 19:15:00 DEBUG "metric": { 19:15:00 DEBUG "pod": "ds-cts-0" 19:15:00 DEBUG }, 19:15:00 DEBUG "value": [ 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG [...] 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:15:00 DEBUG }, 19:15:00 DEBUG "value": [ 19:15:00 DEBUG 1734462900, 19:15:00 DEBUG "0" 19:15:00 DEBUG ] 19:15:00 DEBUG } 19:15:00 DEBUG ] 19:15:00 DEBUG } 19:15:00 DEBUG } 19:15:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462900" 19:15:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734462900" 19:15:00 INFO [http_cmd]: http status code OK 19:15:00 DEBUG --- status code --- 19:15:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:00 DEBUG --- http response --- 19:15:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:15:00 DEBUG ----- output ----- 19:15:00 DEBUG { 19:15:00 DEBUG "status": "success", 19:15:00 DEBUG "data": { 19:15:00 DEBUG "resultType": "vector", 19:15:00 DEBUG "result": [ 19:15:00 DEBUG { 19:15:00 DEBUG "metric": { 19:15:00 DEBUG "pod": "ds-cts-0" 19:15:00 DEBUG }, 19:15:00 DEBUG "value": [ 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG [...] 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:15:00 DEBUG }, 19:15:00 DEBUG "value": [ 19:15:00 DEBUG 1734462900, 19:15:00 DEBUG "1744.3506609562805" 19:15:00 DEBUG ] 19:15:00 DEBUG } 19:15:00 DEBUG ] 19:15:00 DEBUG } 19:15:00 DEBUG } 19:15:00 INFO [http_cmd]: http status code OK 19:15:00 DEBUG --- status code --- 19:15:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:00 DEBUG --- http response --- 19:15:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:15:00 DEBUG ----- output ----- 19:15:00 DEBUG { 19:15:00 DEBUG "status": "success", 19:15:00 DEBUG "data": { 19:15:00 DEBUG "resultType": "vector", 19:15:00 DEBUG "result": [ 19:15:00 DEBUG { 19:15:00 DEBUG "metric": { 19:15:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 19:15:00 DEBUG }, 19:15:00 DEBUG "value": [ 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG [...] 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 19:15:00 DEBUG }, 19:15:00 DEBUG "value": [ 19:15:00 DEBUG 1734462900, 19:15:00 DEBUG "2568384512" 19:15:00 DEBUG ] 19:15:00 DEBUG } 19:15:00 DEBUG ] 19:15:00 DEBUG } 19:15:00 DEBUG } 19:15:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734462900" 19:15:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462900" 19:15:00 INFO [http_cmd]: http status code OK 19:15:00 DEBUG --- status code --- 19:15:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:00 DEBUG --- http response --- 19:15:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:15:00 DEBUG ----- output ----- 19:15:00 DEBUG { 19:15:00 DEBUG "status": "success", 19:15:00 DEBUG "data": { 19:15:00 DEBUG "resultType": "vector", 19:15:00 DEBUG "result": [ 19:15:00 DEBUG { 19:15:00 DEBUG "metric": { 19:15:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 19:15:00 DEBUG }, 19:15:00 DEBUG "value": [ 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG [...] 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 19:15:00 DEBUG }, 19:15:00 DEBUG "value": [ 19:15:00 DEBUG 1734462900, 19:15:00 DEBUG "1.7555003813423928" 19:15:00 DEBUG ] 19:15:00 DEBUG } 19:15:00 DEBUG ] 19:15:00 DEBUG } 19:15:00 DEBUG } 19:15:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462900" 19:15:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462900" 19:15:00 INFO [http_cmd]: http status code OK 19:15:00 DEBUG --- status code --- 19:15:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:00 DEBUG --- http response --- 19:15:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:15:00 DEBUG ----- output ----- 19:15:00 DEBUG { 19:15:00 DEBUG "status": "success", 19:15:00 DEBUG "data": { 19:15:00 DEBUG "resultType": "vector", 19:15:00 DEBUG "result": [ 19:15:00 DEBUG { 19:15:00 DEBUG "metric": { 19:15:00 DEBUG "instance": "10.142.0.11:9100" 19:15:00 DEBUG }, 19:15:00 DEBUG "value": [ 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG [...] 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG "instance": "10.142.0.51:9100" 19:15:00 DEBUG }, 19:15:00 DEBUG "value": [ 19:15:00 DEBUG 1734462900, 19:15:00 DEBUG "19797.333333333336" 19:15:00 DEBUG ] 19:15:00 DEBUG } 19:15:00 DEBUG ] 19:15:00 DEBUG } 19:15:00 DEBUG } 19:15:00 INFO [http_cmd]: http status code OK 19:15:00 DEBUG --- status code --- 19:15:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:00 DEBUG --- http response --- 19:15:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:15:00 DEBUG ----- output ----- 19:15:00 DEBUG { 19:15:00 DEBUG "status": "success", 19:15:00 DEBUG "data": { 19:15:00 DEBUG "resultType": "vector", 19:15:00 DEBUG "result": [ 19:15:00 DEBUG { 19:15:00 DEBUG "metric": { 19:15:00 DEBUG "pod": "ds-cts-0" 19:15:00 DEBUG }, 19:15:00 DEBUG "value": [ 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG [...] 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:15:00 DEBUG }, 19:15:00 DEBUG "value": [ 19:15:00 DEBUG 1734462900, 19:15:00 DEBUG "696950784" 19:15:00 DEBUG ] 19:15:00 DEBUG } 19:15:00 DEBUG ] 19:15:00 DEBUG } 19:15:00 DEBUG } 19:15:00 INFO [http_cmd]: http status code OK 19:15:00 DEBUG --- status code --- 19:15:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:00 DEBUG --- http response --- 19:15:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:15:00 DEBUG ----- output ----- 19:15:00 DEBUG { 19:15:00 DEBUG "status": "success", 19:15:00 DEBUG "data": { 19:15:00 DEBUG "resultType": "vector", 19:15:00 DEBUG "result": [ 19:15:00 DEBUG { 19:15:00 DEBUG "metric": { 19:15:00 DEBUG "instance": "10.142.0.11:9100" 19:15:00 DEBUG }, 19:15:00 DEBUG "value": [ 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG [...] 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG 19:15:00 DEBUG "instance": "10.142.0.51:9100" 19:15:00 DEBUG }, 19:15:00 DEBUG "value": [ 19:15:00 DEBUG 1734462900, 19:15:00 DEBUG "71979.90000000002" 19:15:00 DEBUG ] 19:15:00 DEBUG } 19:15:00 DEBUG ] 19:15:00 DEBUG } 19:15:00 DEBUG } 19:15:03 INFO 19:15:03 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 19:15:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:15:03 INFO [loop_until]: OK (rc = 0) 19:15:03 DEBUG --- stdout --- 19:15:03 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 546Mi ds-idrepo-0 7m 742Mi rcs-5d696ff5b-7ttvr 2m 297Mi rcs-5d696ff5b-wwlbr 2m 333Mi 19:15:03 DEBUG --- stderr --- 19:15:03 DEBUG 19:15:04 INFO 19:15:04 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 19:15:04 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:15:04 INFO [loop_until]: OK (rc = 0) 19:15:04 DEBUG --- stdout --- 19:15:04 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 92m 0% 2269Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1809m 5% 3652Mi 12% 19:15:04 DEBUG --- stderr --- 19:15:04 DEBUG 19:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou-rcs%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462959" 19:15:59 INFO [http_cmd]: http status code OK 19:15:59 DEBUG --- status code --- 19:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:15:59 DEBUG --- http response --- 19:15:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462960" 19:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_replayed_updates_conflicts_resolved%29by%28pod%29&time=1734462960" 19:16:00 INFO [http_cmd]: http status code OK 19:16:00 DEBUG --- status code --- 19:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:00 DEBUG --- http response --- 19:16:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:16:00 INFO [http_cmd]: http status code OK 19:16:00 DEBUG --- status code --- 19:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:00 DEBUG --- http response --- 19:16:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_rejected_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462960" 19:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_blocked_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462960" 19:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amIdentityStore%27%2Cpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462960" 19:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462960" 19:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462960" 19:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462960" 19:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1734462960" 19:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_max_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462960" 19:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462960" 19:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462960" 19:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou-rcs%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462960" 19:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_memory_usage_pools_metaspace_usage%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462960" 19:16:00 INFO [http_cmd]: http status code OK 19:16:00 DEBUG --- status code --- 19:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:00 DEBUG --- http response --- 19:16:00 INFO [http_cmd]: http status code OK 19:16:00 INFO [http_cmd]: http status code OK 19:16:00 ERROR [http_cmd]: ERROR 19:16:00 INFO [http_cmd]: http status code OK 19:16:00 INFO [http_cmd]: http status code OK 19:16:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:16:00 DEBUG --- status code --- 19:16:00 INFO [http_cmd]: http status code OK 19:16:00 DEBUG --- status code --- 19:16:00 INFO [http_cmd]: http status code OK 19:16:00 ERROR --- status code --- 19:16:00 DEBUG --- status code --- 19:16:00 INFO [http_cmd]: http status code OK 19:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=max%28ds_disk_free_space_bytes%7Bdisk%3D%27/opt/opendj/db%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462960" 19:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462960" 19:16:00 DEBUG --- status code --- 19:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:00 INFO [http_cmd]: http status code OK 19:16:00 DEBUG --- status code --- 19:16:00 INFO [http_cmd]: http status code OK 19:16:00 INFO [http_cmd]: http status code OK 19:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:00 DEBUG --- status code --- 19:16:00 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:00 DEBUG --- status code --- 19:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:00 DEBUG --- http response --- 19:16:00 DEBUG --- status code --- 19:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:00 DEBUG --- status code --- 19:16:00 DEBUG --- status code --- 19:16:00 DEBUG --- http response --- 19:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:00 ERROR --- http response --- 19:16:00 DEBUG --- http response --- 19:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:00 DEBUG --- http response --- 19:16:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:00 DEBUG --- http response --- 19:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:16:00 DEBUG --- http response --- 19:16:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_jvm_thread_state_timed_waiting_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462960" 19:16:00 DEBUG --- http response --- 19:16:00 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:16:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:16:00 DEBUG --- http response --- 19:16:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:16:00 DEBUG --- http response --- 19:16:00 DEBUG --- http response --- 19:16:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:16:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:16:00 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:16:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462960" 19:16:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:16:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:16:00 INFO [http_cmd]: http status code OK 19:16:00 INFO [http_cmd]: http status code OK 19:16:00 DEBUG --- status code --- 19:16:00 DEBUG --- status code --- 19:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:00 DEBUG --- http response --- 19:16:00 DEBUG --- http response --- 19:16:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:16:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:16:00 INFO [http_cmd]: http status code OK 19:16:00 DEBUG --- status code --- 19:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:00 DEBUG --- http response --- 19:16:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:16:00 INFO [http_cmd]: http status code OK 19:16:00 DEBUG --- status code --- 19:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:00 DEBUG --- http response --- 19:16:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_work_queue_requests_in_queue%7Bpod%3D~%27ds-idrepo.%2A%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462960" 19:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28am_scripting_threadpool_threads_count%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462960" 19:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28idm_managed_seconds%7Bnamespace%3D%27xlou-rcs%27%2Cmanaged_object%3D~%27.%2Auser.%2A%27%2Coperation%3D~%27%28create%7Cpatch%7Cupdate%29%27%2Cquantile%3D%270.5%27%7D%29by%28pod%29&time=1734462960" 19:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou-rcs%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%29by%28pod%29&time=1734462960" 19:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1734462960" 19:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou-rcs%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462960" 19:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462960" 19:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462960" 19:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28ds_connection_handlers_ldap_active_connections_count%7Bldap_handler%3D%27LDAP%27%2Cnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462960" 19:16:00 INFO [http_cmd]: http status code OK 19:16:00 DEBUG --- status code --- 19:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:00 INFO [http_cmd]: http status code OK 19:16:00 INFO [http_cmd]: http status code OK 19:16:00 DEBUG --- http response --- 19:16:00 DEBUG --- status code --- 19:16:00 DEBUG --- status code --- 19:16:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:00 INFO [http_cmd]: http status code OK 19:16:00 DEBUG --- http response --- 19:16:00 INFO [http_cmd]: http status code OK 19:16:00 DEBUG --- http response --- 19:16:00 INFO [http_cmd]: http status code OK 19:16:00 DEBUG --- status code --- 19:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:00 DEBUG --- http response --- 19:16:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:16:00 INFO [http_cmd]: http status code OK 19:16:00 INFO [http_cmd]: http status code OK 19:16:00 DEBUG --- status code --- 19:16:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:16:00 DEBUG --- status code --- 19:16:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:16:00 DEBUG --- status code --- 19:16:00 DEBUG --- status code --- 19:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:00 DEBUG --- http response --- 19:16:00 DEBUG --- http response --- 19:16:00 DEBUG --- http response --- 19:16:00 DEBUG --- http response --- 19:16:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:16:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:16:00 INFO [http_cmd]: http status code OK 19:16:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:16:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:16:00 DEBUG --- status code --- 19:16:00 DEBUG ----- output ----- 19:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:00 DEBUG { 19:16:00 DEBUG --- http response --- 19:16:00 DEBUG "status": "success", 19:16:00 DEBUG "data": { 19:16:00 DEBUG "resultType": "vector", 19:16:00 DEBUG "result": [ 19:16:00 DEBUG { 19:16:00 DEBUG "metric": { 19:16:00 DEBUG "pod": "ds-cts-0" 19:16:00 DEBUG }, 19:16:00 DEBUG "value": [ 19:16:00 DEBUG 19:16:00 DEBUG 19:16:00 DEBUG 19:16:00 DEBUG [...] 19:16:00 DEBUG 19:16:00 DEBUG 19:16:00 DEBUG 19:16:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:16:00 DEBUG }, 19:16:00 DEBUG "value": [ 19:16:00 DEBUG 1734462960, 19:16:00 DEBUG "0.0025111767598694395" 19:16:00 DEBUG ] 19:16:00 DEBUG } 19:16:00 DEBUG ] 19:16:00 DEBUG } 19:16:00 DEBUG } 19:16:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462960" 19:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462960" 19:16:00 INFO [http_cmd]: http status code OK 19:16:00 DEBUG --- status code --- 19:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:00 DEBUG --- http response --- 19:16:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:16:00 DEBUG ----- output ----- 19:16:00 DEBUG { 19:16:00 DEBUG "status": "success", 19:16:00 DEBUG "data": { 19:16:00 DEBUG "resultType": "vector", 19:16:00 DEBUG "result": [ 19:16:00 DEBUG { 19:16:00 DEBUG "metric": { 19:16:00 DEBUG "instance": "10.142.0.11:9100" 19:16:00 DEBUG }, 19:16:00 DEBUG "value": [ 19:16:00 DEBUG 19:16:00 DEBUG 19:16:00 DEBUG 19:16:00 DEBUG [...] 19:16:00 DEBUG 19:16:00 DEBUG 19:16:00 DEBUG 19:16:00 DEBUG "instance": "10.142.0.51:9100" 19:16:00 DEBUG }, 19:16:00 DEBUG "value": [ 19:16:00 DEBUG 1734462960, 19:16:00 DEBUG "80665.03333333335" 19:16:00 DEBUG ] 19:16:00 DEBUG } 19:16:00 DEBUG ] 19:16:00 DEBUG } 19:16:00 DEBUG } 19:16:00 INFO [http_cmd]: http status code OK 19:16:00 DEBUG --- status code --- 19:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:00 DEBUG --- http response --- 19:16:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:16:00 DEBUG ----- output ----- 19:16:00 DEBUG { 19:16:00 DEBUG "status": "success", 19:16:00 DEBUG "data": { 19:16:00 DEBUG "resultType": "vector", 19:16:00 DEBUG "result": [ 19:16:00 DEBUG { 19:16:00 DEBUG "metric": { 19:16:00 DEBUG "instance": "10.142.0.11:9100" 19:16:00 DEBUG }, 19:16:00 DEBUG "value": [ 19:16:00 DEBUG 19:16:00 DEBUG 19:16:00 DEBUG 19:16:00 DEBUG [...] 19:16:00 DEBUG 19:16:00 DEBUG 19:16:00 DEBUG 19:16:00 DEBUG "instance": "10.142.0.51:9100" 19:16:00 DEBUG }, 19:16:00 DEBUG "value": [ 19:16:00 DEBUG 1734462960, 19:16:00 DEBUG "0.0003333333333333855" 19:16:00 DEBUG ] 19:16:00 DEBUG } 19:16:00 DEBUG ] 19:16:00 DEBUG } 19:16:00 DEBUG } 19:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462960" 19:16:00 INFO [http_cmd]: http status code OK 19:16:00 DEBUG --- status code --- 19:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:00 DEBUG --- http response --- 19:16:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:16:00 DEBUG ----- output ----- 19:16:00 DEBUG { 19:16:00 DEBUG "status": "success", 19:16:00 DEBUG "data": { 19:16:00 DEBUG "resultType": "vector", 19:16:00 DEBUG "result": [ 19:16:00 DEBUG { 19:16:00 DEBUG "metric": { 19:16:00 DEBUG "instance": "10.142.0.11:9100" 19:16:00 DEBUG }, 19:16:00 DEBUG "value": [ 19:16:00 DEBUG 19:16:00 DEBUG 19:16:00 DEBUG 19:16:00 DEBUG [...] 19:16:00 DEBUG 19:16:00 DEBUG 19:16:00 DEBUG 19:16:00 DEBUG "instance": "10.142.0.51:9100" 19:16:00 DEBUG }, 19:16:00 DEBUG "value": [ 19:16:00 DEBUG 1734462960, 19:16:00 DEBUG "0.00039999999999954156" 19:16:00 DEBUG ] 19:16:00 DEBUG } 19:16:00 DEBUG ] 19:16:00 DEBUG } 19:16:00 DEBUG } 19:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462960" 19:16:00 INFO [http_cmd]: http status code OK 19:16:00 DEBUG --- status code --- 19:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:00 DEBUG --- http response --- 19:16:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:16:00 DEBUG ----- output ----- 19:16:00 DEBUG { 19:16:00 DEBUG "status": "success", 19:16:00 DEBUG "data": { 19:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou-rcs%27%7D%5B60s%5D%29%29by%28pod%29&time=1734462960" 19:16:00 DEBUG "resultType": "vector", 19:16:00 DEBUG "result": [ 19:16:00 DEBUG { 19:16:00 DEBUG "metric": { 19:16:00 DEBUG "pod": "ds-cts-0" 19:16:00 DEBUG }, 19:16:00 DEBUG "value": [ 19:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1734462960" 19:16:00 DEBUG 19:16:00 DEBUG 19:16:00 DEBUG 19:16:00 DEBUG [...] 19:16:00 DEBUG 19:16:00 DEBUG 19:16:00 DEBUG 19:16:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:16:00 DEBUG }, 19:16:00 DEBUG "value": [ 19:16:00 DEBUG 1734462960, 19:16:00 DEBUG "0" 19:16:00 DEBUG ] 19:16:00 DEBUG } 19:16:00 DEBUG ] 19:16:00 DEBUG } 19:16:00 DEBUG } 19:16:00 INFO [http_cmd]: http status code OK 19:16:00 DEBUG --- status code --- 19:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:00 DEBUG --- http response --- 19:16:00 INFO [http_cmd]: http status code OK 19:16:00 DEBUG --- status code --- 19:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:16:00 DEBUG --- http response --- 19:16:00 DEBUG ----- output ----- 19:16:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:16:00 DEBUG { 19:16:00 DEBUG ----- output ----- 19:16:00 DEBUG "status": "success", 19:16:00 DEBUG { 19:16:00 DEBUG "data": { 19:16:00 DEBUG "status": "success", 19:16:00 DEBUG "resultType": "vector", 19:16:00 DEBUG "data": { 19:16:00 DEBUG "result": [ 19:16:00 DEBUG "resultType": "vector", 19:16:00 DEBUG { 19:16:00 DEBUG "result": [ 19:16:00 DEBUG "metric": { 19:16:00 DEBUG { 19:16:00 DEBUG "pod": "ds-cts-0" 19:16:00 DEBUG "metric": { 19:16:00 DEBUG }, 19:16:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 19:16:00 DEBUG "value": [ 19:16:00 DEBUG }, 19:16:00 DEBUG 19:16:00 DEBUG "value": [ 19:16:00 DEBUG 19:16:00 DEBUG 19:16:00 DEBUG 19:16:00 DEBUG 19:16:00 DEBUG [...] 19:16:00 DEBUG 19:16:00 DEBUG 19:16:00 DEBUG [...] 19:16:00 DEBUG 19:16:00 DEBUG 19:16:00 DEBUG 19:16:00 DEBUG 19:16:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:16:00 DEBUG 19:16:00 DEBUG }, 19:16:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 19:16:00 DEBUG "value": [ 19:16:00 DEBUG }, 19:16:00 DEBUG 1734462960, 19:16:00 DEBUG "value": [ 19:16:00 DEBUG "1304.9274547093728" 19:16:00 DEBUG 1734462960, 19:16:00 DEBUG ] 19:16:00 DEBUG "2563010560" 19:16:00 DEBUG } 19:16:00 DEBUG ] 19:16:00 DEBUG ] 19:16:00 DEBUG } 19:16:00 DEBUG } 19:16:00 DEBUG ] 19:16:00 DEBUG } 19:16:00 DEBUG } 19:16:00 DEBUG } 19:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1734462960" 19:16:00 INFO [http_cmd]: http status code OK 19:16:00 DEBUG --- status code --- 19:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:00 DEBUG --- http response --- 19:16:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:16:00 DEBUG ----- output ----- 19:16:00 DEBUG { 19:16:00 DEBUG "status": "success", 19:16:00 DEBUG "data": { 19:16:00 DEBUG "resultType": "vector", 19:16:00 DEBUG "result": [ 19:16:00 DEBUG { 19:16:00 DEBUG "metric": { 19:16:00 DEBUG "node": "gke-xlou-cdm-default-pool-9d1e5395-7hzu" 19:16:00 DEBUG }, 19:16:00 DEBUG "value": [ 19:16:00 DEBUG 19:16:00 DEBUG 19:16:00 DEBUG 19:16:00 DEBUG [...] 19:16:00 DEBUG 19:16:00 DEBUG 19:16:00 DEBUG 19:16:00 DEBUG "node": "gke-xlou-cdm-frontend-18a44ad7-wxs9" 19:16:00 DEBUG }, 19:16:00 DEBUG "value": [ 19:16:00 DEBUG 1734462960, 19:16:00 DEBUG "1.7570426967552597" 19:16:00 DEBUG ] 19:16:00 DEBUG } 19:16:00 DEBUG ] 19:16:00 DEBUG } 19:16:00 DEBUG } 19:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462960" 19:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou-rcs%27%7D%29by%28pod%29&time=1734462960" 19:16:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1734462960" 19:16:00 INFO [http_cmd]: http status code OK 19:16:00 DEBUG --- status code --- 19:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:00 DEBUG --- http response --- 19:16:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:16:00 DEBUG ----- output ----- 19:16:00 DEBUG { 19:16:00 DEBUG "status": "success", 19:16:00 DEBUG "data": { 19:16:00 DEBUG "resultType": "vector", 19:16:00 DEBUG "result": [ 19:16:00 DEBUG { 19:16:00 DEBUG "metric": { 19:16:00 DEBUG "instance": "10.142.0.11:9100" 19:16:00 DEBUG }, 19:16:00 DEBUG "value": [ 19:16:00 DEBUG 19:16:00 DEBUG 19:16:00 DEBUG 19:16:00 DEBUG [...] 19:16:00 DEBUG 19:16:00 DEBUG 19:16:00 DEBUG 19:16:00 DEBUG "instance": "10.142.0.51:9100" 19:16:00 DEBUG }, 19:16:00 DEBUG "value": [ 19:16:00 DEBUG 1734462960, 19:16:00 DEBUG "25395.200000000004" 19:16:00 DEBUG ] 19:16:00 DEBUG } 19:16:00 DEBUG ] 19:16:00 DEBUG } 19:16:00 DEBUG } 19:16:00 INFO [http_cmd]: http status code OK 19:16:00 DEBUG --- status code --- 19:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:00 DEBUG --- http response --- 19:16:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (44 lines): 19:16:00 DEBUG ----- output ----- 19:16:00 DEBUG { 19:16:00 DEBUG "status": "success", 19:16:00 DEBUG "data": { 19:16:00 DEBUG "resultType": "vector", 19:16:00 DEBUG "result": [ 19:16:00 DEBUG { 19:16:00 DEBUG "metric": { 19:16:00 DEBUG "pod": "ds-cts-0" 19:16:00 DEBUG }, 19:16:00 DEBUG "value": [ 19:16:00 DEBUG 19:16:00 INFO [http_cmd]: http status code OK 19:16:00 DEBUG 19:16:00 DEBUG --- status code --- 19:16:00 DEBUG 19:16:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:16:00 DEBUG [...] 19:16:00 DEBUG --- http response --- 19:16:00 DEBUG 19:16:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (26 lines): 19:16:00 DEBUG 19:16:00 DEBUG ----- output ----- 19:16:00 DEBUG 19:16:00 DEBUG { 19:16:00 DEBUG "pod": "rcs-5d696ff5b-wwlbr" 19:16:00 DEBUG "status": "success", 19:16:00 DEBUG }, 19:16:00 DEBUG "data": { 19:16:00 DEBUG "value": [ 19:16:00 DEBUG "resultType": "vector", 19:16:00 DEBUG 1734462960, 19:16:00 DEBUG "result": [ 19:16:00 DEBUG "696971264" 19:16:00 DEBUG { 19:16:00 DEBUG ] 19:16:00 DEBUG "metric": { 19:16:00 DEBUG } 19:16:00 DEBUG "instance": "10.142.0.11:9100" 19:16:00 DEBUG ] 19:16:00 DEBUG }, 19:16:00 DEBUG } 19:16:00 DEBUG "value": [ 19:16:00 DEBUG } 19:16:00 DEBUG 19:16:00 DEBUG 19:16:00 DEBUG 19:16:00 DEBUG [...] 19:16:00 DEBUG 19:16:00 DEBUG 19:16:00 DEBUG 19:16:00 DEBUG "instance": "10.142.0.51:9100" 19:16:00 DEBUG }, 19:16:00 DEBUG "value": [ 19:16:00 DEBUG 1734462960, 19:16:00 DEBUG "64456.600000000006" 19:16:00 DEBUG ] 19:16:00 DEBUG } 19:16:00 DEBUG ] 19:16:00 DEBUG } 19:16:00 DEBUG } 19:16:03 INFO 19:16:03 INFO [loop_until]: kubectl --namespace=xlou-rcs top pods 19:16:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:16:03 INFO [loop_until]: OK (rc = 0) 19:16:03 DEBUG --- stdout --- 19:16:03 DEBUG NAME CPU(cores) MEMORY(bytes) ds-cts-0 6m 547Mi ds-idrepo-0 7m 743Mi rcs-5d696ff5b-7ttvr 2m 297Mi rcs-5d696ff5b-wwlbr 2m 332Mi 19:16:03 DEBUG --- stderr --- 19:16:03 DEBUG 19:16:04 INFO 19:16:04 INFO [loop_until]: kubectl --namespace=xlou-rcs top node 19:16:04 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:16:04 INFO [loop_until]: OK (rc = 0) 19:16:04 DEBUG --- stdout --- 19:16:04 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-9d1e5395-7hzu 93m 0% 2267Mi 3% gke-xlou-cdm-frontend-18a44ad7-wxs9 1825m 5% 3655Mi 12% 19:16:04 DEBUG --- stderr --- 19:16:04 DEBUG 19:16:30 INFO Finished: True 19:16:30 INFO Waiting for threads to register finish flag 19:17:04 INFO Done. Have a nice day! :) 127.0.0.1 - - [17/Dec/2024 19:17:04] "GET /monitoring/stop HTTP/1.1" 200 - 19:17:06 WARNING File /tmp/lodemon_data-20241217_175018/node_disk_read_bytes_total_file.json does not exist. Skipping graph generation... WARNING:bokeh.core.validation.check:W-1000 (MISSING_RENDERERS): Plot has no renderers: figure(id='p3146', ...) WARNING:bokeh.core.validation.check:W-1000 (MISSING_RENDERERS): Plot has no renderers: figure(id='p2901', ...) WARNING:bokeh.core.validation.check:W-1000 (MISSING_RENDERERS): Plot has no renderers: figure(id='p3048', ...) WARNING:bokeh.core.validation.check:W-1000 (MISSING_RENDERERS): Plot has no renderers: figure(id='p2460', ...) WARNING:bokeh.core.validation.check:W-1000 (MISSING_RENDERERS): Plot has no renderers: figure(id='p2999', ...) WARNING:bokeh.core.validation.check:W-1000 (MISSING_RENDERERS): Plot has no renderers: figure(id='p2754', ...) WARNING:bokeh.core.validation.check:W-1000 (MISSING_RENDERERS): Plot has no renderers: figure(id='p3313', ...) WARNING:bokeh.core.validation.check:W-1000 (MISSING_RENDERERS): Plot has no renderers: figure(id='p2950', ...) WARNING:bokeh.core.validation.check:W-1000 (MISSING_RENDERERS): Plot has no renderers: figure(id='p2558', ...) WARNING:bokeh.core.validation.check:W-1000 (MISSING_RENDERERS): Plot has no renderers: figure(id='p2509', ...) WARNING:bokeh.core.validation.check:W-1000 (MISSING_RENDERERS): Plot has no renderers: figure(id='p2705', ...) WARNING:bokeh.core.validation.check:W-1000 (MISSING_RENDERERS): Plot has no renderers: figure(id='p2166', ...) WARNING:bokeh.core.validation.check:W-1000 (MISSING_RENDERERS): Plot has no renderers: figure(id='p3264', ...) WARNING:bokeh.core.validation.check:W-1000 (MISSING_RENDERERS): Plot has no renderers: figure(id='p2607', ...) WARNING:bokeh.core.validation.check:W-1000 (MISSING_RENDERERS): Plot has no renderers: figure(id='p3411', ...) WARNING:bokeh.core.validation.check:W-1000 (MISSING_RENDERERS): Plot has no renderers: figure(id='p2215', ...) WARNING:bokeh.core.validation.check:W-1000 (MISSING_RENDERERS): Plot has no renderers: figure(id='p2411', ...) WARNING:bokeh.core.validation.check:W-1000 (MISSING_RENDERERS): Plot has no renderers: figure(id='p2264', ...) WARNING:bokeh.core.validation.check:W-1000 (MISSING_RENDERERS): Plot has no renderers: figure(id='p2803', ...) WARNING:bokeh.core.validation.check:W-1000 (MISSING_RENDERERS): Plot has no renderers: figure(id='p2656', ...) WARNING:bokeh.core.validation.check:W-1000 (MISSING_RENDERERS): Plot has no renderers: figure(id='p2852', ...) WARNING:bokeh.core.validation.check:W-1000 (MISSING_RENDERERS): Plot has no renderers: figure(id='p3097', ...) WARNING:bokeh.core.validation.check:W-1000 (MISSING_RENDERERS): Plot has no renderers: figure(id='p2362', ...) WARNING:bokeh.core.validation.check:W-1000 (MISSING_RENDERERS): Plot has no renderers: figure(id='p2313', ...) WARNING:bokeh.core.validation.check:W-1000 (MISSING_RENDERERS): Plot has no renderers: figure(id='p3460', ...) WARNING:bokeh.core.validation.check:W-1000 (MISSING_RENDERERS): Plot has no renderers: figure(id='p3362', ...) 127.0.0.1 - - [17/Dec/2024 19:17:07] "GET /monitoring/process HTTP/1.1" 200 -