[#1407] feat(rust): fix + add total grpc request metrics (#1516)

### What changes were proposed in this pull request?

1. fix grpc queue size metric 
2. add total grpc request metrics

### Why are the changes needed?

For #1407

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Needn't
diff --git a/rust/experimental/server/src/grpc.rs b/rust/experimental/server/src/grpc.rs
index 7557512..1ff4bd4 100644
--- a/rust/experimental/server/src/grpc.rs
+++ b/rust/experimental/server/src/grpc.rs
@@ -720,7 +720,7 @@
 }
 
 pub mod metrics_middleware {
-    use crate::metric::GAUGE_GRPC_REQUEST_QUEUE_SIZE;
+    use crate::metric::{GAUGE_GRPC_REQUEST_QUEUE_SIZE, TOTAL_GRPC_REQUEST};
     use hyper::service::Service;
     use hyper::Body;
     use prometheus::HistogramVec;
@@ -769,6 +769,7 @@
         }
 
         fn call(&mut self, req: hyper::Request<Body>) -> Self::Future {
+            TOTAL_GRPC_REQUEST.inc();
             GAUGE_GRPC_REQUEST_QUEUE_SIZE.inc();
 
             // This is necessary because tonic internally uses `tower::buffer::Buffer`.
@@ -787,7 +788,7 @@
 
                 timer.observe_duration();
 
-                GAUGE_GRPC_REQUEST_QUEUE_SIZE.inc();
+                GAUGE_GRPC_REQUEST_QUEUE_SIZE.dec();
 
                 Ok(response)
             })
diff --git a/rust/experimental/server/src/metric.rs b/rust/experimental/server/src/metric.rs
index 03e53bc..a7998a8 100644
--- a/rust/experimental/server/src/metric.rs
+++ b/rust/experimental/server/src/metric.rs
@@ -245,8 +245,11 @@
 pub static GAUGE_IN_SPILL_DATA_SIZE: Lazy<IntGauge> =
     Lazy::new(|| IntGauge::new("in_spill_data_size", "total data size in spill").unwrap());
 
+pub static TOTAL_GRPC_REQUEST: Lazy<IntCounter> =
+    Lazy::new(|| IntCounter::new("total_grpc_request_number", "total request number").expect(""));
+
 pub static GAUGE_GRPC_REQUEST_QUEUE_SIZE: Lazy<IntGauge> =
-    Lazy::new(|| IntGauge::new("grpc_request_queue_size", "grpc request queue size").unwrap());
+    Lazy::new(|| IntGauge::new("grpc_request_number", "current grpc request queue size").unwrap());
 
 pub static TOTAL_SPILL_EVENTS_DROPPED: Lazy<IntCounter> = Lazy::new(|| {
     IntCounter::new(
@@ -258,6 +261,14 @@
 
 fn register_custom_metrics() {
     REGISTRY
+        .register(Box::new(TOTAL_GRPC_REQUEST.clone()))
+        .expect("");
+
+    REGISTRY
+        .register(Box::new(GAUGE_GRPC_REQUEST_QUEUE_SIZE.clone()))
+        .expect("");
+
+    REGISTRY
         .register(Box::new(TOTAL_SPILL_EVENTS_DROPPED.clone()))
         .expect("");