Scaffolding: Federation sync app creation

.pre-commit-config.yaml

@@ -52,32 +52,44 @@ repos:
       hooks:
           - id: django-upgrade
             args: ["--target-version", "4.2"]
+            exclude: ^federation/
 
     # runs the ruff linter and formatter
     - repo: https://github.com/astral-sh/ruff-pre-commit
       rev: v0.15.15
       hooks:
           # linter
           - id: ruff-check # runs ruff check --force-exclude
+            exclude: ^federation/
             args:
-                # we have to pick one ruff config between gateway and sdk...
-                #   the SDK has additional constraints for having to support multiple
-                #   python versions, so it's a safer choice for a project-wide config;
-                #   though be aware that this might omit PyUpgrade improvements that
-                #   could be applied to the gateway side.
                 [
-                    # --verbose, # for debugging
-                    # --show-files, # for debugging
-                    # --show-fixes, # for debugging
                     --no-cache,
                     --fix,
                     --exit-non-zero-on-fix,
                     --config,
                     "sdk/pyproject.toml",
-                ] # --show-settings, # for debugging
+                ]
+          - id: ruff-check
+            alias: ruff-check-federation
+            name: ruff-check (federation)
+            files: ^federation/.*\.py$
+            args:
+                [
+                    --no-cache,
+                    --fix,
+                    --exit-non-zero-on-fix,
+                    --config,
+                    "federation/pyproject.toml",
+                ]
           # formatter
           - id: ruff-format # runs ruff format --force-exclude
+            exclude: ^federation/
             args: [--config, "sdk/pyproject.toml"]
+          - id: ruff-format
+            alias: ruff-format-federation
+            name: ruff-format (federation)
+            files: ^federation/.*\.py$
+            args: [--config, "federation/pyproject.toml"]
 
     # linter for django templates
     - repo: https://github.com/Riverside-Healthcare/djLint
@@ -100,7 +112,7 @@ repos:
       hooks:
           - id: deptry
             name: deptry
-            entry: bash -c "cd sdk && uv run deptry .; cd ../gateway && uv run deptry ."
+            entry: bash -c "cd sdk && uv run deptry .; cd ../gateway && uv run deptry .; cd ../federation && uv run deptry ."
             language: system
             pass_filenames: false # run once per commit, not per file
             types: [python]

federation/.envs/example/sync.env

@@ -0,0 +1 @@
+FEDERATION_GATEWAY_API_KEY=

federation/.gitignore

@@ -0,0 +1,6 @@
+# Dev PKI material — generate with scripts/generate-dev-certs.sh
+
+__pycache__/
+
+.envs/*
+!/.envs/example

federation/Dockerfile

@@ -0,0 +1,15 @@
+FROM docker.io/python:3.13-slim-trixie
+
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+
+WORKDIR /app
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+
+COPY pyproject.toml ./
+RUN uv sync --no-dev
+
+COPY sds_federation ./sds_federation
+
+EXPOSE 8000
+CMD ["uv", "run", "uvicorn", "sds_federation.main:app", "--host", "0.0.0.0", "--port", "8000"]

federation/certs/.gitignore

@@ -0,0 +1,3 @@
+# Dev PKI material — generate with scripts/generate-dev-certs.sh
+*
+!.gitignore

federation/compose.yaml

@@ -0,0 +1,31 @@
+services:
+  sds-federation-local-sync:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    image: sds-federation-local-sync
+    container_name: sds-federation-local-sync
+    env_file:
+      - .envs/local/sync.env
+    environment:
+      FEDERATION_CONFIG_PATH: /etc/sds/federation.toml
+      GATEWAY_INTERNAL_BASE_URL: http://sds-gateway-local-app:8000/api/v1
+      REDIS_URL: redis://sds-gateway-local-redis:6379/0
+      OPENSEARCH_HOST: sds-gateway-local-opensearch
+    volumes:
+      - ./federation.toml:/etc/sds/federation.toml:ro
+      - ./certs:/etc/sds/certs:ro
+    ports:
+      - "8001:8000"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+    networks:
+      - sds-network-local
+
+networks:
+  sds-network-local:
+    external: true
+    name: sds-network-local

federation/federation.toml

@@ -0,0 +1,12 @@
+[site]
+name = "crc"
+fqdn = "sds.crc.nd.edu"
+display_name = "Notre Dame CRC"
+
+[[peers]]
+name = "haystack"
+fqdn = "merrimack.haystack.mit.edu"
+display_name = "MIT Haystack"
+gateway_api_base = "https://merrimack.haystack.mit.edu/api/v1"
+sync_service_url = "https://merrimack.haystack.mit.edu/sync/"
+ca_cert_path = "/etc/sds/certs/haystack-ca.pem"

federation/justfile

@@ -0,0 +1,70 @@
+set shell := ["bash", "-eu", "-o", "pipefail", "-c"]
+
+compose_file := "compose.yaml"
+docker_compose := "COMPOSE_FILE=" + compose_file + " docker compose"
+
+alias run := up
+
+default:
+    @just --list
+
+# install runtime + dev deps (pytest)
+[group('setup')]
+sync:
+    uv sync --extra dev
+
+[group('qa')]
+test-regression +args='':
+    uv run pytest -m regression {{ args }}
+
+[group('qa')]
+test-integration +args='':
+    uv run pytest -m integration {{ args }}
+
+[group('qa')]
+test +args='':
+    uv run pytest {{ args }}
+
+[group('qa')]
+test-q:
+    uv run pytest -q
+
+# Run federation ruff hooks from repo root (requires gateway dev deps / pre-commit install)
+[group('qa')]
+pre-commit +args='':
+    cd .. && uv run --directory gateway --extra local pre-commit run ruff-check-federation --all-files {{ args }}
+    cd .. && uv run --directory gateway --extra local pre-commit run ruff-format-federation --all-files {{ args }}
+
+# dev PKI for mTLS experiments (see docs/mtls-certificates.md)
+[group('setup')]
+gen-certs +args='':
+    ./scripts/generate-dev-certs.sh all {{ args }}
+
+[group('setup')]
+gen-certs-ca:
+    ./scripts/generate-dev-certs.sh ca
+
+# publish simulated federation:events to Redis
+[group('dev')]
+simulate-redis +args='':
+    REDIS_URL="${REDIS_URL:-redis://localhost:6379/0}" uv run python scripts/simulate_redis_event.py {{ args }}
+
+[group('docker')]
+build +args='':
+    {{ docker_compose }} build {{ args }}
+
+[group('docker')]
+up +args='':
+    {{ docker_compose }} up -d --remove-orphans {{ args }}
+
+[group('docker')]
+down +args='':
+    {{ docker_compose }} down {{ args }}
+
+[group('docker')]
+logs +args='':
+    {{ docker_compose }} logs --tail 1000 -f {{ args }}
+
+[group('docker')]
+dc +args='':
+    {{ docker_compose }} {{ args }}

federation/pyproject.toml

@@ -0,0 +1,154 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["sds_federation"]
+
+[project]
+    name = "sds-federation"
+    version = "0.1.0"
+    description = "SDS Federation"
+    requires-python = ">=3.13,<3.14"
+    dependencies = [
+        "fastapi>=0.115.6",
+        "httpx>=0.28.1",
+        "loguru>=0.7.2",
+        "opensearch-py>=2.8.0",
+        "pydantic>=2.11.0",
+        "redis>=5.2.1",
+        "uvicorn[standard]>=0.34.0",
+    ]
+
+[project.optional-dependencies]
+dev = [
+    "deptry>=0.24.0",
+    "pytest>=8.3.0",
+    "pytest-asyncio>=0.25.0",
+    "ruff>=0.15.0",
+]
+
+[tool.deptry]
+    extend_exclude = ["docs/"]
+
+    [tool.deptry.per_rule_ignores]
+        DEP002 = [
+            "deptry",
+            "pytest",
+            "pytest-asyncio",
+            "ruff",
+            "uvicorn",
+        ]
+
+    [tool.deptry.package_module_name_map]
+        opensearch-py = "opensearchpy"
+
+[tool.ruff]
+    exclude = [
+        ".git",
+        ".ruff_cache",
+        ".venv",
+        "build",
+        "dist",
+        "docs/",
+    ]
+    indent-width = 4
+    line-length = 88
+    src = ["/federation/"]
+    target-version = "py313"
+
+    [tool.ruff.lint]
+        ignore = [
+            "COM812",
+            "ISC001",
+            "RUF012",
+            "S101",
+            "S104",
+            "SIM102",
+            "TRY003",
+            "EM101",
+            "EM102",
+            "TC002",
+        ]
+        select = [
+            "F",
+            "E",
+            "W",
+            "C90",
+            "I",
+            "N",
+            "UP",
+            "YTT",
+            "ASYNC",
+            "S",
+            "BLE",
+            "FBT",
+            "B",
+            "A",
+            "COM",
+            "C4",
+            "DTZ",
+            "T10",
+            "EM",
+            "EXE",
+            "ISC",
+            "ICN",
+            "LOG",
+            "G",
+            "INP",
+            "PIE",
+            "T20",
+            "PYI",
+            "PT",
+            "Q",
+            "RSE",
+            "RET",
+            "SLF",
+            "SLOT",
+            "SIM",
+            "TID",
+            "TC",
+            "PTH",
+            "ERA",
+            "PGH",
+            "PL",
+            "TRY",
+            "FLY",
+            "PERF",
+            "FURB",
+            "RUF",
+            "FAST",
+        ]
+        fixable = ["ALL"]
+
+        [tool.ruff.lint.isort]
+            force-single-line = true
+
+        [tool.ruff.lint.pylint]
+            max-args = 9
+
+        [tool.ruff.lint.per-file-ignores]
+            "**/tests/**" = [
+                "FBT001",
+                "FBT002",
+                "PLR2004",
+                "E501",
+                "PT011",
+                "B017",
+                "PLC0415",
+            ]
+            "**/scripts/**" = ["T201", "EXE001", "E501"]
+
+    [tool.ruff.format]
+        indent-style = "space"
+        line-ending = "auto"
+        quote-style = "double"
+        skip-magic-trailing-comma = false
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+testpaths = ["tests"]
+markers = [
+    "integration: tests that exercise FastAPI routes or multi-step pipeline",
+    "regression: contract and indexer behavior guards",
+]