From 14e7f6f76009fc804e3a01ee1f33f355db618ce6 Mon Sep 17 00:00:00 2001
From: eeff <33566201+eeff@users.noreply.github.com>
Date: Mon, 5 Feb 2024 16:47:32 +0800
Subject: [PATCH] fix(restful): force clean stale rolling metrics

---
 plugins/restful/metric_handle.c | 35 +++++++++++++++++++++++++++++----
 1 file changed, 31 insertions(+), 4 deletions(-)

diff --git a/plugins/restful/metric_handle.c b/plugins/restful/metric_handle.c
index d1e56d97..d18884b0 100644
--- a/plugins/restful/metric_handle.c
+++ b/plugins/restful/metric_handle.c
@@ -19,8 +19,11 @@
 
 #include <stdio.h>
 
+#include <pthread.h>
+
 #include "define.h"
 #include "metrics.h"
+#include "plugin.h"
 #include "utils/asprintf.h"
 #include "utils/http.h"
 #include "utils/http_handler.h"
@@ -164,8 +167,8 @@ static inline void gen_global_metrics(const neu_metrics_t *metrics,
             metrics->south_running_nodes, metrics->south_disconnected_nodes);
 }
 
-static inline void
-gen_single_node_metrics(const neu_node_metrics_t *node_metrics, FILE *stream)
+static inline void gen_single_node_metrics(neu_node_metrics_t *node_metrics,
+                                           FILE *              stream)
 {
     fprintf(stream,
             "# HELP node_type Driver(1) or APP(2)\n"
@@ -175,8 +178,13 @@ gen_single_node_metrics(const neu_node_metrics_t *node_metrics, FILE *stream)
 
     neu_metric_entry_t *e = NULL;
 
+    pthread_mutex_lock(&node_metrics->lock);
     HASH_LOOP(hh, node_metrics->entries, e)
     {
+        if (NEU_METRIC_TYPE_ROLLING_COUNTER == e->type) {
+            // force clean stale value
+            e->value = neu_rolling_counter_inc(e->rcnt, global_timestamp, 0);
+        }
         fprintf(stream,
                 "# HELP %s %s\n# TYPE %s %s\n%s{node=\"%s\"} %" PRIu64 "\n",
                 e->name, e->help, e->name, neu_metric_type_str(e->type),
@@ -188,6 +196,11 @@ gen_single_node_metrics(const neu_node_metrics_t *node_metrics, FILE *stream)
     {
         HASH_LOOP(hh, g->entries, e)
         {
+            if (NEU_METRIC_TYPE_ROLLING_COUNTER == e->type) {
+                // force clean stale value
+                e->value =
+                    neu_rolling_counter_inc(e->rcnt, global_timestamp, 0);
+            }
             fprintf(stream,
                     "# HELP %s %s\n# TYPE %s %s\n%s{node=\"%s\",group=\"%s\"} "
                     "%" PRIu64 "\n",
@@ -195,6 +208,7 @@ gen_single_node_metrics(const neu_node_metrics_t *node_metrics, FILE *stream)
                     e->name, node_metrics->name, g->name, e->value);
         }
     }
+    pthread_mutex_unlock(&node_metrics->lock);
 }
 
 static inline bool has_entry(neu_node_metrics_t *node_metrics, const char *name)
@@ -216,7 +230,7 @@ static inline bool has_entry(neu_node_metrics_t *node_metrics, const char *name)
     return false;
 }
 
-static void gen_all_node_metrics(const neu_metrics_t *metrics, int type_filter,
+static void gen_all_node_metrics(neu_metrics_t *metrics, int type_filter,
                                  FILE *stream)
 {
     neu_metric_entry_t * e = NULL, *r = NULL;
@@ -261,11 +275,18 @@ static void gen_all_node_metrics(const neu_metrics_t *metrics, int type_filter,
                         r->help, r->name, neu_metric_type_str(r->type));
             }
 
+            pthread_mutex_lock(&n->lock);
             HASH_FIND_STR(n->entries, r->name, e);
             if (e) {
+                if (NEU_METRIC_TYPE_ROLLING_COUNTER == e->type) {
+                    // force clean stale value
+                    e->value =
+                        neu_rolling_counter_inc(e->rcnt, global_timestamp, 0);
+                }
                 fprintf(stream, "%s{node=\"%s\"} %" PRIu64 "\n", e->name,
                         n->name, e->value);
 
+                pthread_mutex_unlock(&n->lock);
                 continue;
             }
 
@@ -273,11 +294,17 @@ static void gen_all_node_metrics(const neu_metrics_t *metrics, int type_filter,
             {
                 HASH_FIND_STR(g->entries, r->name, e);
                 if (e) {
+                    if (NEU_METRIC_TYPE_ROLLING_COUNTER == e->type) {
+                        // force clean stale value
+                        e->value = neu_rolling_counter_inc(e->rcnt,
+                                                           global_timestamp, 0);
+                    }
                     fprintf(stream,
                             "%s{node=\"%s\",group=\"%s\"} %" PRIu64 "\n",
                             e->name, n->name, g->name, e->value);
                 }
             }
+            pthread_mutex_unlock(&n->lock);
         }
     }
 }
@@ -289,7 +316,7 @@ struct context {
     const char *node;
 };
 
-static void gen_node_metrics(const neu_metrics_t *metrics, struct context *ctx)
+static void gen_node_metrics(neu_metrics_t *metrics, struct context *ctx)
 {
     if (ctx->node[0]) {
         neu_node_metrics_t *n = NULL;