feat: instrument command latency in Sentry

.gitmodules

@@ -3,4 +3,4 @@
 	url = https://github.com/Statsify/public-assets
 [submodule "assets/private"]
 	path = assets/private
-	url = https://github.com/Statsify/assets
+	url = https://github.com/Statsify/assets

apps/api/package.json

@@ -20,6 +20,7 @@
     "@nestjs/platform-fastify": "^11.1.6",
     "@nestjs/swagger": "^11.2.0",
     "@sentry/node": "^7.118.0",
+    "@sentry/profiling-node": "^7.120.4",
     "@statsify/api-client": "workspace:^",
     "@statsify/assets": "workspace:^",
     "@statsify/logger": "workspace:^",

apps/api/src/hypixel/hypixel.service.ts

@@ -6,7 +6,6 @@
  * https://github.com/Statsify/statsify/blob/main/LICENSE
  */
 
-import * as Sentry from "@sentry/node";
 import { CacheLevel } from "@statsify/api-client";
 import {
   GameCounts,
@@ -18,8 +17,8 @@ import {
 } from "@statsify/schemas";
 import { HttpService } from "@nestjs/axios";
 import { Injectable } from "@nestjs/common";
-import { Logger } from "@statsify/logger";
-import { Observable, catchError, lastValueFrom, map, of, tap, throwError } from "rxjs";
+import { Logger, startSentrySpan } from "@statsify/logger";
+import { Observable, catchError, finalize, lastValueFrom, map, of, tap, throwError } from "rxjs";
 import type { APIData } from "@statsify/util";
 
 @Injectable()
@@ -142,27 +141,30 @@ export class HypixelService {
   }
 
   private request<T>(url: string, params?: Record<string, unknown>): Observable<T> {
-    const transaction = Sentry.getCurrentHub().getScope()?.getTransaction();
-
-    const child = transaction?.startChild({
-      op: "http.client",
+    const span = startSentrySpan({
+      op: "hypixel.fetch",
       description: `GET ${this.httpService.axiosRef.getUri({ url })}`,
+      data: {
+        "http.method": "GET",
+        "http.route": url,
+      },
     });
 
     return this.httpService.get(url, { params }).pipe(
       tap((res) => {
-        child?.setHttpStatus(res.status);
-        child?.finish();
+        span?.setHttpStatus(res.status);
+        span?.setData("hypixel.ratelimit.limit", res.headers["ratelimit-limit"]);
+        span?.setData("hypixel.ratelimit.remaining", res.headers["ratelimit-remaining"]);
+        span?.setData("hypixel.ratelimit.reset", res.headers["ratelimit-reset"]);
       }),
       map((res) => res.data),
-      catchError((err) =>
-        throwError(
-          () =>
-            new Error(`Fetching ${url} failed with reason: ${err.message}`, {
-              cause: err,
-            })
-        )
-      )
+      catchError((err) => throwError(
+        () =>
+          new Error(`Fetching ${url} failed with reason: ${err.message}`, {
+            cause: err,
+          })
+      )),
+      finalize(() => span?.finish())
     );
   }
 }

apps/api/src/index.ts

@@ -14,13 +14,14 @@ import { DocumentBuilder, SwaggerModule } from "@nestjs/swagger";
 import { FastifyAdapter, NestFastifyApplication } from "@nestjs/platform-fastify";
 import { Logger } from "@statsify/logger";
 import { NestFactory } from "@nestjs/core";
-import { SentryInterceptor } from "./sentry/index.js";
+import { SentryInterceptor, instrumentMongooseQueries } from "./sentry/index.js";
 import { Severity, setGlobalOptions } from "@typegoose/typegoose";
 import { ValidationPipe } from "@nestjs/common";
 import { config } from "@statsify/util";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
 import { mkdir } from "node:fs/promises";
+import { nodeProfilingIntegration } from "@sentry/profiling-node";
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 
@@ -31,20 +32,29 @@ process.on("uncaughtException", handleError);
 process.on("unhandledRejection", handleError);
 
 const sentryDsn = await config("sentry.apiDsn", { required: false });
+const sentryTracesSampleRate =
+  await config("sentry.tracesSampleRate", { required: false }) ?? 0;
+const sentryProfilesSampleRate =
+  await config("sentry.profilesSampleRate", { required: false }) ??
+  sentryTracesSampleRate;
 
 if (sentryDsn) {
   Sentry.init({
     dsn: sentryDsn,
     integrations: [
       new Sentry.Integrations.Http({ tracing: false, breadcrumbs: true }),
       new Sentry.Integrations.Mongo({ useMongoose: true }),
+      nodeProfilingIntegration(),
     ],
     normalizeDepth: 3,
-    tracesSampleRate: await config("sentry.tracesSampleRate"),
+    tracesSampleRate: sentryTracesSampleRate,
+    profilesSampleRate: sentryProfilesSampleRate,
     environment: await config("environment"),
   });
 }
 
+instrumentMongooseQueries();
+
 const mediaRoot = await config("api.mediaRoot");
 
 await mkdir(join(mediaRoot, "badges"), { recursive: true });

apps/api/src/leaderboards/leaderboard.service.ts

@@ -6,14 +6,14 @@
  * https://github.com/Statsify/statsify/blob/main/LICENSE
  */
 
-import * as Sentry from "@sentry/node";
 import { Constructor, Flatten } from "@statsify/util";
 import { DateTime } from "luxon";
 import { InjectRedis } from "#redis";
 import { Injectable, InternalServerErrorException } from "@nestjs/common";
 import { LeaderboardEnabledMetadata, LeaderboardScanner } from "@statsify/schemas";
 import { LeaderboardQuery } from "@statsify/api-client";
 import { Redis } from "ioredis";
+import { withSentrySpan } from "@statsify/logger";
 
 const DAYS_IN_WEEK = {
   monday: 0,
@@ -38,12 +38,6 @@ export abstract class LeaderboardService {
     remove = false
   ) {
     const fields = LeaderboardScanner.getLeaderboardFields(constructor);
-    const transaction = Sentry.getCurrentHub().getScope()?.getTransaction();
-
-    const child = transaction?.startChild({
-      op: "redis",
-      description: `add ${constructor.name} leaderboards`,
-    });
 
     const pipeline = this.redis.pipeline();
     const name = constructor.name.toLowerCase();
@@ -67,9 +61,10 @@ export abstract class LeaderboardService {
       }
     }
 
-    await pipeline.exec();
-
-    child?.finish();
+    await withSentrySpan({
+      op: "redis.write",
+      description: `add ${constructor.name} leaderboards`,
+    }, () => pipeline.exec());
   }
 
   public async getLeaderboard<T>(
@@ -194,13 +189,6 @@ export abstract class LeaderboardService {
     fields: string[],
     id: string
   ) {
-    const transaction = Sentry.getCurrentHub().getScope()?.getTransaction();
-
-    const child = transaction?.startChild({
-      op: "redis",
-      description: `get ${constructor.name} rankings`,
-    });
-
     const pipeline = this.redis.pipeline();
     const constructorName = constructor.name.toLowerCase();
 
@@ -221,9 +209,10 @@ export abstract class LeaderboardService {
       }
     });
 
-    const responses = await pipeline.exec();
-
-    child?.finish();
+    const responses = await withSentrySpan({
+      op: "redis.get",
+      description: `get ${constructor.name} rankings`,
+    }, () => pipeline.exec());
 
     if (!responses) throw new InternalServerErrorException();
 
@@ -274,21 +263,17 @@ export abstract class LeaderboardService {
     bottom: number,
     sort = "DESC"
   ) {
-    const transaction = Sentry.getCurrentHub().getScope()?.getTransaction();
-
-    const child = transaction?.startChild({
-      op: "redis",
-      description: `get ${constructor.name} leaderboards`,
-    });
-
     const name = constructor.name.toLowerCase();
     field = `${name}.${field}`;
 
-    const scores = await (sort === "ASC" ?
-      this.redis.zrange(field, top, bottom, "WITHSCORES") :
-      this.redis.zrevrange(field, top, bottom, "WITHSCORES"));
-
-    child?.finish();
+    const scores = await withSentrySpan({
+      op: "redis.get",
+      description: `get ${constructor.name} leaderboards`,
+    }, () =>
+      sort === "ASC" ?
+        this.redis.zrange(field, top, bottom, "WITHSCORES") :
+        this.redis.zrevrange(field, top, bottom, "WITHSCORES")
+    );
 
     const response: { id: string; score: number; index: number }[] = [];
 

apps/api/src/redis/redis.utils.ts

@@ -12,8 +12,38 @@ import {
   REDIS_MODULE_OPTIONS_TOKEN,
 } from "./redis.constants.js";
 import { Redis } from "ioredis";
+import { startSentrySpan } from "@statsify/logger";
 import type { RedisModuleOptions } from "./redis.interfaces.js";
 
+const REDIS_READ_COMMANDS = new Set([
+  "exists",
+  "get",
+  "hget",
+  "hgetall",
+  "hmget",
+  "mget",
+  "ttl",
+  "zrank",
+  "zrange",
+  "zrevrank",
+  "zrevrange",
+  "zscore",
+  "ft.sugget",
+]);
+
+const REDIS_WRITE_COMMANDS = new Set([
+  "del",
+  "expire",
+  "expireat",
+  "hset",
+  "hmset",
+  "set",
+  "zadd",
+  "zrem",
+  "ft.sugadd",
+  "ft.sugdel",
+]);
+
 export function getRedisOptionsToken(connection?: string): string {
   return `${connection || REDIS_MODULE_CONNECTION}_${REDIS_MODULE_OPTIONS_TOKEN}`;
 }
@@ -24,5 +54,32 @@ export function getRedisConnectionToken(connection?: string): string {
 
 export function createRedisConnection(options: RedisModuleOptions) {
   const { config } = options;
-  return config.url ? new Redis(config.url, config) : new Redis(config);
+  const redis = config.url ? new Redis(config.url, config) : new Redis(config);
+  const sendCommand = redis.sendCommand.bind(redis);
+
+  redis.sendCommand = ((command, stream) => {
+    const commandName = String((command as { name: string }).name).toLowerCase();
+    const span = startSentrySpan({
+      op: getRedisSpanOperation(commandName),
+      description: commandName,
+      data: { "redis.command": commandName },
+    });
+
+    try {
+      return (sendCommand(command, stream) as Promise<unknown>).finally(() =>
+        span?.finish()
+      );
+    } catch (error) {
+      span?.finish();
+      throw error;
+    }
+  }) as Redis["sendCommand"];
+
+  return redis;
+}
+
+function getRedisSpanOperation(commandName: string) {
+  if (REDIS_READ_COMMANDS.has(commandName)) return "redis.get";
+  if (REDIS_WRITE_COMMANDS.has(commandName)) return "redis.write";
+  return "redis.command";
 }

apps/api/src/sentry/index.ts

@@ -7,3 +7,4 @@
  */
 
 export * from "./sentry.interceptor.js";
+export * from "./mongoose.js";

apps/api/src/sentry/mongoose.ts

@@ -0,0 +1,93 @@
+/**
+ * Copyright (c) Statsify
+ *
+ * This source code is licensed under the GNU GPL v3 license found in the
+ * LICENSE file in the root directory of this source tree.
+ * https://github.com/Statsify/statsify/blob/main/LICENSE
+ */
+
+import { Aggregate, Query } from "mongoose";
+import { startSentrySpan } from "@statsify/logger";
+
+let mongooseInstrumented = false;
+
+type InstrumentedQuery = Query<unknown, unknown> & {
+  mongooseCollection?: { name?: string };
+  op?: string;
+};
+
+type InstrumentedAggregate = Aggregate<unknown> & {
+  _model?: {
+    collection?: { name?: string };
+    modelName?: string;
+  };
+};
+
+export function instrumentMongooseQueries() {
+  if (mongooseInstrumented) return;
+  mongooseInstrumented = true;
+
+  instrumentQueryExec();
+  instrumentAggregateExec();
+}
+
+function instrumentQueryExec() {
+  const exec = Query.prototype.exec;
+
+  Query.prototype.exec = function instrumentedExec(
+    this: InstrumentedQuery,
+    ...args: Parameters<typeof exec>
+  ): ReturnType<typeof exec> {
+    const collection = this.mongooseCollection?.name ?? this.model.collection.name;
+    const operation = this.op ?? "query";
+    const span = startSentrySpan({
+      op: "mongo.query",
+      description: `${collection}.${operation}`,
+      data: {
+        "db.collection": collection,
+        "db.operation": operation,
+        "db.system": "mongodb",
+        "mongoose.model": this.model.modelName,
+      },
+    });
+
+    try {
+      return exec.apply(this, args).finally(() => span?.finish()) as ReturnType<
+        typeof exec
+      >;
+    } catch (error) {
+      span?.finish();
+      throw error;
+    }
+  };
+}
+
+function instrumentAggregateExec() {
+  const exec = Aggregate.prototype.exec;
+
+  Aggregate.prototype.exec = function instrumentedExec(
+    this: InstrumentedAggregate,
+    ...args: Parameters<typeof exec>
+  ): ReturnType<typeof exec> {
+    const collection = this._model?.collection?.name ?? "unknown";
+    const span = startSentrySpan({
+      op: "mongo.query",
+      description: `${collection}.aggregate`,
+      data: {
+        "db.collection": collection,
+        "db.operation": "aggregate",
+        "db.system": "mongodb",
+        "mongoose.model": this._model?.modelName ?? "unknown",
+      },
+    });
+
+    try {
+      return exec.apply(this, args).finally(() => span?.finish()) as ReturnType<
+        typeof exec
+      >;
+    } catch (error) {
+      span?.finish();
+      throw error;
+    }
+  };
+}