test(docs): add end to end evaluation doc tests

deepeval/integrations/langchain/callback.py

@@ -539,4 +539,4 @@ def on_retriever_error(
         with self._ctx(run_id=run_id, parent_run_id=parent_run_id):
             retriever_span.status = TraceSpanStatus.ERRORED
             retriever_span.error = str(error)
-            exit_current_context(uuid_str=uuid_str)
+            exit_current_context(uuid_str=uuid_str)

tests/conftest.py

@@ -0,0 +1,72 @@
+try:
+    import sys
+    import pysqlite3 as sqlite3  # type: ignore
+
+    sys.modules["sqlite3"] = sqlite3
+    sys.modules["sqlite3.dbapi2"] = sqlite3.dbapi2
+except Exception:
+    pass
+
+import pytest
+
+from typing import TYPE_CHECKING
+from pathlib import Path
+
+from deepeval.tracing.tracing import trace_manager
+from deepeval.config.settings import get_settings, reset_settings
+
+
+if TYPE_CHECKING:
+    pass
+
+
+# Silence telemetry for all tests so we don't have to deal with the noise
+@pytest.fixture(autouse=True)
+def _telemetry_opt_out(monkeypatch):
+    monkeypatch.setenv("DEEPEVAL_TELEMETRY_OPT_OUT", "1")
+    yield
+
+
+@pytest.fixture(autouse=True)
+def _ensure_hidden_store_dir(tmp_path: Path):
+    d = tmp_path / ".deepeval"
+    d.mkdir(exist_ok=True)
+    # some code expects the file to be there after a run,
+    # but at minimum the directory must exist to avoid FileNotFoundError
+    yield
+
+
+@pytest.fixture
+def hidden_store_dir(tmp_path: Path) -> Path:
+    d = tmp_path / ".deepeval"
+    d.mkdir(parents=True, exist_ok=True)
+    return d
+
+
+@pytest.fixture()
+def settings():
+    settings = get_settings()
+    yield settings
+
+
+@pytest.fixture()
+def enable_dotenv(monkeypatch):
+    monkeypatch.setenv("DEEPEVAL_DISABLE_DOTENV", "0")
+    # rebuild Settings after changing the env
+    reset_settings(reload_dotenv=False)
+
+
+@pytest.fixture(autouse=True)
+def _reset_tracing_state():
+    trace_manager.clear_traces()
+    trace_manager.traces_to_evaluate_order.clear()
+    trace_manager.traces_to_evaluate.clear()
+    trace_manager.integration_traces_to_evaluate.clear()
+    trace_manager.trace_uuid_to_golden.clear()
+    try:
+        trace_manager.task_bindings.clear()
+    except Exception:
+        pass
+    trace_manager.evaluating = False
+    trace_manager.evaluation_loop = False
+    yield

tests/test_core/conftest.py

@@ -14,7 +14,6 @@
 from typing import TYPE_CHECKING
 from pathlib import Path
 
-from deepeval.tracing.tracing import trace_manager
 from deepeval.config.settings import get_settings, reset_settings, Settings
 
 
@@ -29,36 +28,6 @@
 }
 
 
-@pytest.fixture(autouse=True)
-def _ensure_hidden_store_dir(tmp_path: Path):
-    d = tmp_path / ".deepeval"
-    d.mkdir(exist_ok=True)
-    # some code expects the file to be there after a run,
-    # but at minimum the directory must exist to avoid FileNotFoundError
-    yield
-
-
-@pytest.fixture
-def hidden_store_dir(tmp_path: Path) -> Path:
-    d = tmp_path / ".deepeval"
-    d.mkdir(parents=True, exist_ok=True)
-    return d
-
-
-# Silence telemetry for all tests so we don't have to deal with the noise
-@pytest.fixture(autouse=True)
-def _telemetry_opt_out(monkeypatch):
-    monkeypatch.setenv("DEEPEVAL_TELEMETRY_OPT_OUT", "1")
-    yield
-
-
-# Run every test in its own temp CWD so .deepeval/.deepeval is sandboxed
-@pytest.fixture(autouse=True)
-def _isolate_cwd(tmp_path: Path, monkeypatch):
-    monkeypatch.chdir(tmp_path)
-    yield
-
-
 # Default dotenv path most tests can reuse; override in tests as needed
 @pytest.fixture
 def env_path(monkeypatch, tmp_path: Path) -> Path:
@@ -72,17 +41,6 @@ def env_dir(monkeypatch, tmp_path: Path) -> Path:
     return tmp_path
 
 
-@pytest.fixture(autouse=True)
-def no_sleep(monkeypatch):
-    monkeypatch.setattr(tenacity.nap, "sleep", lambda _: None, raising=True)
-
-
-@pytest.fixture()
-def settings():
-    settings = get_settings()
-    yield settings
-
-
 @pytest.fixture(scope="session")
 def _session_env_baseline():
     # capture the environment as it existed when pytest started
@@ -120,6 +78,9 @@ def _env_sandbox(_session_env_baseline, request, monkeypatch):
     for k, v in preserved.items():
         monkeypatch.setenv(k, v)
 
+    # Always silence telemetry in tests
+    monkeypatch.setenv("DEEPEVAL_TELEMETRY_OPT_OUT", "1")
+
     # Never open the Confident AI browser UI during tests
     monkeypatch.setenv("CONFIDENT_OPEN_BROWSER", "0")
 
@@ -166,13 +127,6 @@ def _core_mode_no_confident(
     yield
 
 
-@pytest.fixture()
-def enable_dotenv(monkeypatch):
-    monkeypatch.setenv("DEEPEVAL_DISABLE_DOTENV", "0")
-    # rebuild Settings after changing the env
-    reset_settings(reload_dotenv=False)
-
-
 @pytest.fixture(autouse=False)
 def unpatch_openai_after():
     from deepeval.openai.patch import unpatch_openai_classes
@@ -181,17 +135,13 @@ def unpatch_openai_after():
     unpatch_openai_classes()
 
 
+# Run every test in its own temp CWD so .deepeval/.deepeval is sandboxed
 @pytest.fixture(autouse=True)
-def _reset_tracing_state():
-    trace_manager.clear_traces()
-    trace_manager.traces_to_evaluate_order.clear()
-    trace_manager.traces_to_evaluate.clear()
-    trace_manager.integration_traces_to_evaluate.clear()
-    trace_manager.trace_uuid_to_golden.clear()
-    try:
-        trace_manager.task_bindings.clear()
-    except Exception:
-        pass
-    trace_manager.evaluating = False
-    trace_manager.evaluation_loop = False
+def _isolate_cwd(tmp_path: Path, monkeypatch):
+    monkeypatch.chdir(tmp_path)
     yield
+
+
+@pytest.fixture(autouse=True)
+def no_sleep(monkeypatch):
+    monkeypatch.setattr(tenacity.nap, "sleep", lambda _: None, raising=True)

tests/test_core/test_end_to_end/conftest.py

@@ -0,0 +1,10 @@
+import pytest
+
+
+@pytest.fixture(autouse=True)
+def _offline_deterministic_env(monkeypatch: pytest.MonkeyPatch):
+    # Prevent dotenv loading (could pull real API keys/configs) and avoid browser open.
+    monkeypatch.setenv("DEEPEVAL_DISABLE_DOTENV", "1")
+    monkeypatch.setenv("CONFIDENT_OPEN_BROWSER", "0")
+    # Keep stable even if unset.
+    monkeypatch.setenv("DEEPEVAL_RESULTS_FOLDER", "")