Fix performance of tracing rate limiter

Cargo.toml

@@ -181,10 +181,11 @@ thread_local = { version = "1.1.9", default-features = false, features = [] }
 tokio = { version = "1.52.3", default-features = false, features = [] }
 tokio-util = { version = "0.7.18", default-features = false, features = [] }
 tonic = { version = "0.14.6", default-features = false, features = [] }
-tracing = { version = "0.1.44", default-features = false, features = [] }
+tracing = { version = "0.1.44", default-features = false, features = ["release_max_level_debug"] }
 tracing-error = { version = "0.2.1", default-features = false, features = [] }
 tracing-subscriber = { version = "0.3.23", default-features = false, features = [] }
 tracing-test = { version = "0.2.6", default-features = false, features = [] }
+tracing-throttle = { version = "0.4.2", default-features = false, features = [] }
 ureq = { version = "3.3.0", default-features = false, features = [] }
 url = { version = "2.5.8", default-features = false, features = [] }
 uuid = { version = "1.23.1", default-features = false, features = [] }

args/src/lib.rs

@@ -84,6 +84,23 @@ pub struct InterfaceArg {
     pub port: Option<PortArg>,
 }
 
+#[derive(
+    Debug,
+    PartialEq,
+    Eq,
+    Clone,
+    serde::Serialize,
+    rkyv::Serialize,
+    rkyv::Deserialize,
+    rkyv::Archive,
+    CheckBytes,
+)]
+#[rkyv(attr(derive(PartialEq, Eq, Debug)))]
+pub struct TracingRateLimit {
+    pub burst: u32,
+    pub replenish_per_second: u32,
+}
+
 impl FromStr for PortArg {
     type Err = String;
     fn from_str(input: &str) -> Result<Self, Self::Err> {
@@ -131,6 +148,34 @@ impl FromStr for InterfaceArg {
     }
 }
 
+impl FromStr for TracingRateLimit {
+    type Err = String;
+    fn from_str(input: &str) -> Result<Self, Self::Err> {
+        let (burst, replenish_per_second) = input
+            .split_once(':')
+            .ok_or("Bad syntax: missing :".to_string())?;
+
+        let burst = burst
+            .parse::<u32>()
+            .map_err(|e| format!("Bad burst value: {e}"))?;
+        let replenish_per_second = replenish_per_second
+            .parse::<u32>()
+            .map_err(|e| format!("Bad replenish-per-second value: {e}"))?;
+
+        if burst == 0 {
+            return Err("Burst must be greater than 0".to_string());
+        }
+        if replenish_per_second == 0 {
+            return Err("Replenish-per-second must be greater than 0".to_string());
+        }
+
+        Ok(Self {
+            burst,
+            replenish_per_second,
+        })
+    }
+}
+
 use tracing::instrument;
 
 use bytecheck::CheckBytes;
@@ -463,6 +508,8 @@ pub struct TracingConfigSection {
     pub show: TracingShowSection,
     /// Tracing configuration string (e.g., "default=info,nat=debug")
     pub config: Option<String>, // TODO: stronger typing on this config?
+    /// Optional rate limit for lower-severity tracing output
+    pub rate_limit: Option<TracingRateLimit>,
 }
 
 /// Display option for trace metadata elements.
@@ -1146,6 +1193,7 @@ impl TryFrom<CmdArgs> for LaunchConfiguration {
                     },
                 },
                 config: value.tracing.clone(),
+                rate_limit: value.tracing_rate_limit.clone(),
             },
             metrics: MetricsConfigSection {
                 address: value.metrics_address(),
@@ -1266,6 +1314,16 @@ E.g. default=error,all=info,nat=debug will set the default target to error, and
     )]
     tracing: Option<String>,
 
+    #[arg(
+        long,
+        value_name = "BURST:REPLENISH_PER_SECOND",
+        value_parser=TracingRateLimit::from_str,
+        help = "Optional rate limit for lower-severity tracing output. Syntax: BURST:REPLENISH_PER_SECOND.
+Example: --tracing-rate-limit 50:5 allows bursts up to 50 repeated messages and replenishes 5 messages per second.
+If omitted, tracing output is not rate-limited."
+    )]
+    tracing_rate_limit: Option<TracingRateLimit>,
+
     #[arg(long, help = "Set the name of this gateway")]
     name: Option<String>,
 
@@ -1363,6 +1421,11 @@ impl CmdArgs {
         self.tracing.as_ref()
     }
 
+    #[must_use]
+    pub fn tracing_rate_limit(&self) -> Option<&TracingRateLimit> {
+        self.tracing_rate_limit.as_ref()
+    }
+
     /// Get the number of worker threads for the kernel driver.
     ///
     /// This value comes from the `--num-workers` argument (default: 1, range: 1-64).
@@ -1478,6 +1541,7 @@ mod tests {
     use hardware::pci::function::Function;
     use net::interface::InterfaceName;
 
+    use super::TracingRateLimit;
     use crate::{InterfaceArg, PortArg};
     use std::str::FromStr;
 
@@ -1520,4 +1584,29 @@ mod tests {
         // bad discriminant
         assert!(InterfaceArg::from_str("GbEth1.9000=foo@0000:02:01.7").is_err());
     }
+    #[test]
+    fn tracing_rate_limit_parses_valid_values() {
+        let rate_limit = TracingRateLimit::from_str("10:20").unwrap();
+        assert_eq!(rate_limit.burst, 10);
+        assert_eq!(rate_limit.replenish_per_second, 20);
+    }
+    #[test]
+    fn tracing_rate_limit_rejects_missing_separator() {
+        let err = TracingRateLimit::from_str("10").unwrap_err();
+        assert_eq!(err, "Bad syntax: missing :");
+    }
+    #[test]
+    fn tracing_rate_limit_rejects_non_numeric_values() {
+        let err = TracingRateLimit::from_str("abc:20").unwrap_err();
+        assert!(err.starts_with("Bad burst value:"));
+        let err = TracingRateLimit::from_str("10:def").unwrap_err();
+        assert!(err.starts_with("Bad replenish-per-second value:"));
+    }
+    #[test]
+    fn tracing_rate_limit_rejects_zero_values() {
+        let err = TracingRateLimit::from_str("0:20").unwrap_err();
+        assert_eq!(err, "Burst must be greater than 0");
+        let err = TracingRateLimit::from_str("10:0").unwrap_err();
+        assert_eq!(err, "Replenish-per-second must be greater than 0");
+    }
 }

dataplane/src/main.rs

@@ -21,7 +21,9 @@ use pyroscope::backend::{BackendConfig, PprofConfig, pprof_backend};
 use pyroscope::pyroscope::{PyroscopeAgentBuilder, PyroscopeConfig};
 
 use routing::{BmpServerParams, RouterParamsBuilder};
-use tracectl::{custom_target, get_trace_ctl, trace_target};
+use tracectl::{
+    TracingControl, TracingRateLimitConfig, custom_target, get_trace_ctl, trace_target,
+};
 
 use tracing::{error, info, level_filters::LevelFilter};
 
@@ -52,15 +54,24 @@ fn init_name(args: &CmdArgs) -> Result<String, String> {
         Ok(name.to_string())
     }
 }
-fn init_logging(gwname: &str) {
+fn init_logging(args: &CmdArgs, gwname: &str) {
+    TracingControl::init_with_rate_limit(args.tracing_rate_limit().map(|rate_limit| {
+        TracingRateLimitConfig {
+            burst: rate_limit.burst,
+            replenish_per_second: rate_limit.replenish_per_second,
+        }
+    }));
+
     let tctl = get_trace_ctl();
     info!(
         " ━━━━━━ Starting dataplane for gateway '{gwname}' (Version = {}) ━━━━━━",
         option_env!("VERSION").unwrap_or("dev").to_string()
     );
 
-    tctl.set_default_level(LevelFilter::DEBUG)
-        .expect("Setting default loglevel failed");
+    if args.tracing().is_none() {
+        tctl.set_default_level(LevelFilter::DEBUG)
+            .expect("Setting default loglevel failed");
+    }
 }
 
 fn process_tracing_cmds(args: &CmdArgs) {
@@ -133,7 +144,7 @@ fn main() {
             std::process::exit(1);
         }
     };
-    init_logging(&gwname);
+    init_logging(&args, &gwname);
 
     let (bmp_server_params, bmp_client_opts) = parse_bmp_params(&args);
 

tracectl/Cargo.toml

@@ -6,6 +6,7 @@ publish.workspace = true
 version.workspace = true
 
 [dependencies]
+arc-swap = { workspace = true }
 color-eyre = { workspace = true , features = [ "capture-spantrace", "color-spantrace", "tracing-error", "track-caller" ] }
 common = { workspace = true }
 linkme = { workspace = true }
@@ -14,6 +15,8 @@ thiserror = { workspace = true }
 tracing = { workspace = true }
 tracing-error = { workspace = true, features = ["traced-error"] }
 tracing-subscriber = { workspace = true, features = ["registry", "std", "env-filter", "fmt"] }
+tracing-throttle = { workspace = true }
 
 [dev-dependencies]
 serial_test = { workspace = true }
+tracing-test = { workspace = true }