diff --git a/docs/understand/weblogs/end-to-end_weblog.md b/docs/understand/weblogs/end-to-end_weblog.md index 08dabf6b078..60b470abf88 100644 --- a/docs/understand/weblogs/end-to-end_weblog.md +++ b/docs/understand/weblogs/end-to-end_weblog.md @@ -198,6 +198,20 @@ if the request to `internal_server` is a failure, it must return a json body wit - `status` the status code of the `internal_server` response if available or a null value - `error` a string describing the error, for debug purposes +### GET /external_request/body_limit/{failure_reason} +### POST /external_request/body_limit/{failure_reason} +### TRACE /external_request/body_limit/{failure_reason} +### PUT /external_request/body_limit/{failure_reason} + +Same behavior as `/external_request`, but the downstream call targets `http://internal_server:8089/downstream_response/{failure_reason}`. + +Supported `{failure_reason}` values (defined in `internal_server`): +- `invalid_content_type` +- `content_length_missing` +- `content_length_too_big` + +Unknown values must return HTTP 404 with a json error body. + ### GET /external_request/redirect This endpoint tests HTTP redirect chains with downstream requests, using the fastapi application in `/utils/build/docker/internal_server/app.py` diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index 3ea27c8774d..81a89778757 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -95,6 +95,7 @@ refs: - &ref_5_101_0 '>=5.101.0' - &ref_5_103_0 '>=5.103.0' - &ref_5_104_0 '>=5.104.0' + - &ref_5_106_0 '>=5.106.0' # downstream response body limits - &ref_6_0_0 '>=6.0.0-pre' manifest: tests/ai_guard/test_ai_guard_sdk.py::Test_AIGuardEvent_Tag: missing_feature (APPSEC-62217) @@ -697,6 +698,21 @@ manifest: tests/appsec/rasp/test_api10.py::Test_API10_request_headers: *ref_5_87_0 tests/appsec/rasp/test_api10.py::Test_API10_request_method: *ref_5_87_0 tests/appsec/rasp/test_api10.py::Test_API10_response_body: *ref_5_87_0 + tests/appsec/rasp/test_api10.py::Test_API10_response_body_ignored_content_length_missing: + - weblog_declaration: + express4: *ref_5_106_0 + express5: *ref_5_106_0 + "*": irrelevant + tests/appsec/rasp/test_api10.py::Test_API10_response_body_ignored_content_length_too_big: + - weblog_declaration: + express4: *ref_5_106_0 + express5: *ref_5_106_0 + "*": irrelevant + tests/appsec/rasp/test_api10.py::Test_API10_response_body_ignored_content_type: + - weblog_declaration: + express4: *ref_5_106_0 + express5: *ref_5_106_0 + "*": irrelevant tests/appsec/rasp/test_api10.py::Test_API10_response_headers: *ref_5_87_0 tests/appsec/rasp/test_api10.py::Test_API10_response_status: *ref_5_87_0 tests/appsec/rasp/test_api10.py::Test_API10_without_downstream_body_analysis_using_max: *ref_5_87_0 diff --git a/tests/appsec/rasp/test_api10.py b/tests/appsec/rasp/test_api10.py index b4e2c6aefb2..98d6fd81654 100644 --- a/tests/appsec/rasp/test_api10.py +++ b/tests/appsec/rasp/test_api10.py @@ -339,6 +339,78 @@ def test_api10_res_body(self): interfaces.library.validate_one_span(self.r, validator=self.validate_absence) +class API10ResponseBodyIgnored(API10): + """Base for downstream response body guard tests.""" + + TAGS_NOT_EXPECTED = ["_dd.appsec.trace.res_body"] + + def validate_ignored(self, span: DataDogLibrarySpan): + if span.get("parent_id") not in (0, None): + return None + + if not self.validate_metric(span): + return None + + for tag in self.TAGS_NOT_EXPECTED: + assert tag not in span.get("meta", {}), f"Tag {tag} should NOT be present in span's meta" + + return True + + +@rfc("https://docs.google.com/document/d/1gCXU3LvTH9en3Bww0AC2coSJWz1m7HcavZjvMLuDCWg/edit#heading=h.giijrtyn1fdx") +@features.api10 +@scenarios.appsec_rasp +class Test_API10_response_body_ignored_content_type(API10ResponseBodyIgnored): + """Downstream response body ignored when content-type is unsupported.""" + + TAGS_EXPECTED_METRIC = [ + ("_dd.appsec.downstream_request.response_body_ignored.content_type_invalid", "1"), + ] + + def setup_api10_response_body_ignored_content_type(self): + self.r = weblog.get("/external_request/body_limit/invalid_content_type") + + def test_api10_response_body_ignored_content_type(self): + assert self.r.status_code == 200 + interfaces.library.validate_one_span(self.r, validator=self.validate_ignored) + + +@rfc("https://docs.google.com/document/d/1gCXU3LvTH9en3Bww0AC2coSJWz1m7HcavZjvMLuDCWg/edit#heading=h.giijrtyn1fdx") +@features.api10 +@scenarios.appsec_rasp +class Test_API10_response_body_ignored_content_length_missing(API10ResponseBodyIgnored): + """Downstream response body ignored when content-length is missing.""" + + TAGS_EXPECTED_METRIC = [ + ("_dd.appsec.downstream_request.response_body_ignored.content_length_missing", "1"), + ] + + def setup_api10_response_body_ignored_content_length_missing(self): + self.r = weblog.get("/external_request/body_limit/content_length_missing") + + def test_api10_response_body_ignored_content_length_missing(self): + assert self.r.status_code == 200 + interfaces.library.validate_one_span(self.r, validator=self.validate_ignored) + + +@rfc("https://docs.google.com/document/d/1gCXU3LvTH9en3Bww0AC2coSJWz1m7HcavZjvMLuDCWg/edit#heading=h.giijrtyn1fdx") +@features.api10 +@scenarios.appsec_rasp +class Test_API10_response_body_ignored_content_length_too_big(API10ResponseBodyIgnored): + """Downstream response body ignored when content-length exceeds maxBytes.""" + + TAGS_EXPECTED_METRIC = [ + ("_dd.appsec.downstream_request.response_body_ignored.content_length_too_big", "1"), + ] + + def setup_api10_response_body_ignored_content_length_too_big(self): + self.r = weblog.get("/external_request/body_limit/content_length_too_big") + + def test_api10_response_body_ignored_content_length_too_big(self): + assert self.r.status_code == 200 + interfaces.library.validate_one_span(self.r, validator=self.validate_ignored) + + @rfc("https://docs.google.com/document/d/1gCXU3LvTH9en3Bww0AC2coSJWz1m7HcavZjvMLuDCWg/edit#heading=h.giijrtyn1fdx") @features.api10 @scenarios.appsec_rasp diff --git a/utils/_context/_scenarios/appsec_rasp.py b/utils/_context/_scenarios/appsec_rasp.py index 4d797e034a6..200bb3413f8 100644 --- a/utils/_context/_scenarios/appsec_rasp.py +++ b/utils/_context/_scenarios/appsec_rasp.py @@ -24,6 +24,7 @@ def __init__( # added to test Test_ExtendedRequestBodyCollection "DD_APPSEC_RASP_COLLECT_REQUEST_BODY": "true", "DD_API_SECURITY_DOWNSTREAM_BODY_ANALYSIS_SAMPLE_RATE": "1.0", + "DD_API_SECURITY_MAX_DOWNSTREAM_BODY_BYTES": "128", "OPENAI_BASE_URL": "http://internal_server:8089", } merged_env = default_env | weblog_env diff --git a/utils/build/docker/internal_server/app.py b/utils/build/docker/internal_server/app.py index ecec8be5eac..3810ba46795 100644 --- a/utils/build/docker/internal_server/app.py +++ b/utils/build/docker/internal_server/app.py @@ -40,6 +40,45 @@ async def mirror(status: int, request: fastapi.Request): return fastapi.responses.JSONResponse({"status": "OK", "payload": body}, status_code=status, headers=query) +_DOWNSTREAM_RESPONSE_BODY = b'{"ok":true}' +_DOWNSTREAM_RESPONSE_BODY_TOO_BIG = b'{"payload":"' + b"x" * 180 + b'"}' + + +@app.get("/downstream_response/{profile}") +async def downstream_response(profile: str): + """Controlled downstream responses for API10 response body limit guards.""" + + if profile == "invalid_content_type": + return fastapi.responses.Response( + content=_DOWNSTREAM_RESPONSE_BODY, + status_code=200, + media_type="text/html", + headers={"content-length": str(len(_DOWNSTREAM_RESPONSE_BODY))}, + ) + + if profile == "content_length_missing": + + async def body_stream(): + yield _DOWNSTREAM_RESPONSE_BODY + + return fastapi.responses.StreamingResponse( + body_stream(), + status_code=200, + media_type="application/json", + ) + + if profile == "content_length_too_big": + body = _DOWNSTREAM_RESPONSE_BODY_TOO_BIG + return fastapi.responses.Response( + content=body, + status_code=200, + media_type="application/json", + headers={"content-length": str(len(body))}, + ) + + return fastapi.responses.JSONResponse({"error": "unknown profile"}, status_code=404) + + @app.get("/redirect", response_class=fastapi.responses.RedirectResponse) async def redirect(request: fastapi.Request): """Redirect endpoint for testing API 10 with redirects diff --git a/utils/build/docker/nodejs/express/app.js b/utils/build/docker/nodejs/express/app.js index e21e24ddeae..a8e7a7589f7 100644 --- a/utils/build/docker/nodejs/express/app.js +++ b/utils/build/docker/nodejs/express/app.js @@ -664,12 +664,20 @@ app.get('/add_event', (req, res) => { res.status(200).json({ message: 'Event added' }) }) -app.all('/external_request', (req, res) => { +const DOWNSTREAM_RESPONSE_BODY_LIMIT_PROFILES = new Set([ + 'invalid_content_type', + 'content_length_missing', + 'content_length_too_big' +]) + +function forwardExternalRequest (req, res, downstreamPath) { const status = req.query.status || '200' const urlExtra = req.query.url_extra || '' const headers = {} + const queryParamsExcludedFromHeaders = new Set(['status', 'url_extra']) for (const [key, value] of Object.entries(req.query)) { + if (queryParamsExcludedFromHeaders.has(key)) continue headers[key] = String(value) } @@ -679,10 +687,12 @@ app.all('/external_request', (req, res) => { headers['Content-Type'] = req.headers['content-type'] || 'application/json' } + const path = downstreamPath || `/mirror/${status}${urlExtra}` + const options = { hostname: 'internal_server', port: 8089, - path: `/mirror/${status}${urlExtra}`, + path, method: req.method, headers } @@ -703,12 +713,25 @@ app.all('/external_request', (req, res) => { }) }) - // Write body if present if (body) { request.write(body) } request.end() +} + +app.all('/external_request', (req, res) => { + forwardExternalRequest(req, res) +}) + +app.all('/external_request/body_limit/:failureReason', (req, res) => { + const { failureReason } = req.params + if (!DOWNSTREAM_RESPONSE_BODY_LIMIT_PROFILES.has(failureReason)) { + res.status(404).json({ error: 'unknown failure reason' }) + return + } + + forwardExternalRequest(req, res, `/downstream_response/${failureReason}`) }) app.get('/external_request/redirect', (req, res) => { diff --git a/utils/telemetry/intake/static/config_norm_rules.json b/utils/telemetry/intake/static/config_norm_rules.json index 04b47cae964..0841c64e550 100644 --- a/utils/telemetry/intake/static/config_norm_rules.json +++ b/utils/telemetry/intake/static/config_norm_rules.json @@ -21,6 +21,7 @@ "DD_API_SECURITY_ENDPOINT_COLLECTION_MESSAGE_LIMIT": "api_security_endpoint_collection_message_limit", "DD_API_SECURITY_MAX_CONCURRENT_REQUESTS": "api_security_max_concurrent_requests", "DD_API_SECURITY_MAX_DOWNSTREAM_REQUEST_BODY_ANALYSIS": "api_security_max_downstream_request_body_analysis", + "DD_API_SECURITY_MAX_DOWNSTREAM_BODY_BYTES": "api_security_max_downstream_body_bytes", "DD_API_SECURITY_REQUEST_SAMPLE_RATE": "api_security_request_sample_rate", "DD_API_SECURITY_SAMPLE_DELAY": "api_security_sample_delay", "DD_APM_ENABLE_RARE_SAMPLER": "trace_rare_sampler_enabled",