feat: add Benchmark

docs/docs.json

@@ -77,6 +77,7 @@
                   "observe/agent_tasks",
                   "observe/buffer",
                   "observe/disable_tasks",
+                  "observe/disable_task_status_change",
                   "observe/dashboard",
                   "observe/traces"
                 ]

docs/observe/agent_tasks.mdx

@@ -117,7 +117,10 @@ for (const task of tasks.items) {
 
 ## Next Steps
 
-<CardGroup cols={3}>
+<CardGroup cols={2}>
+<Card title="Control Task Status" icon="sliders" href="/observe/disable_task_status_change">
+Manually control task outcomes for benchmarking or human-in-the-loop workflows
+</Card>
 <Card title="Buffer" icon="clock" href="/observe/buffer">
 Understand task extraction timing
 </Card>

docs/observe/disable_task_status_change.mdx

@@ -0,0 +1,110 @@
+---
+title: "Control the Task Status"
+description: "Disable automatic status updates and manually control task outcomes"
+---
+
+Keep task extraction running while preventing the task agent from automatically marking tasks as `success` or `failed`. You decide when to finalize task outcomes via `update_task_status`, which triggers the [self-learning pipeline](/learn/self-learning).
+
+<CodeGroup>
+```python Python
+session = client.sessions.create(disable_task_status_change=True)
+```
+
+```typescript TypeScript
+const session = await client.sessions.create({ disableTaskStatusChange: true });
+```
+</CodeGroup>
+
+**What happens:**
+- Tasks are still created with descriptions and progress
+- Messages are still linked to tasks
+- Status remains at `running` — the agent cannot set `success` or `failed`
+- Learning is only triggered when you call `update_task_status`
+
+## Typical Flow
+
+<Steps>
+<Step title="Create session with manual control">
+
+<CodeGroup>
+```python Python
+import os
+from acontext import AcontextClient
+
+client = AcontextClient(api_key=os.getenv("ACONTEXT_API_KEY"))
+session = client.sessions.create(disable_task_status_change=True)
+```
+
+```typescript TypeScript
+import { AcontextClient } from '@acontext/acontext';
+
+const client = new AcontextClient({ apiKey: process.env.ACONTEXT_API_KEY });
+const session = await client.sessions.create({ disableTaskStatusChange: true });
+```
+</CodeGroup>
+</Step>
+
+<Step title="Store conversation and extract tasks">
+
+<CodeGroup>
+```python Python
+for msg in conversation_messages:
+    client.sessions.store_message(session_id=session.id, blob=msg, format="openai")
+client.sessions.flush(session.id)
+
+tasks = client.sessions.get_tasks(session.id)
+```
+
+```typescript TypeScript
+for (const msg of conversationMessages) {
+    await client.sessions.storeMessage(session.id, msg, { format: "openai" });
+}
+await client.sessions.flush(session.id);
+
+const tasks = await client.sessions.getTasks(session.id);
+```
+</CodeGroup>
+</Step>
+
+<Step title="Manually set task outcome">
+
+Setting status to `success` or `failed` triggers the self-learning pipeline.
+
+<CodeGroup>
+```python Python
+client.sessions.update_task_status(
+    session_id=session.id,
+    task_id=tasks.items[0].id,
+    status="success",
+)
+```
+
+```typescript TypeScript
+await client.sessions.updateTaskStatus(session.id, tasks.items[0].id, {
+    status: "success",
+});
+```
+</CodeGroup>
+</Step>
+</Steps>
+
+## How It Differs from `disable_task_tracking`
+
+| | `disable_task_tracking` | `disable_task_status_change` |
+|---|---|---|
+| **What's blocked** | Entire task extraction | Only status transitions to `success`/`failed` |
+| **Tasks created?** | No | Yes |
+| **Progress tracked?** | No | Yes |
+| **Messages linked?** | No | Yes |
+| **Learning triggered?** | Never | Only via manual `update_task_status` |
+
+## Next Steps
+
+<CardGroup cols={2}>
+<Card title="Agent Tasks" icon="list-check" href="/observe/agent_tasks">
+How task extraction works
+</Card>
+<Card title="Self-Learning" icon="brain" href="/learn/self-learning">
+Learn agent skills from past sessions
+</Card>
+</CardGroup>

src/client/acontext-py/src/acontext/resources/async_sessions.py

@@ -17,6 +17,7 @@
     Message,
     MessageObservingStatus,
     Session,
+    Task,
     TokenCounts,
 )
 from ..uploads import FileUpload, normalize_file_upload
@@ -84,6 +85,7 @@ async def create(
         *,
         user: str | None = None,
         disable_task_tracking: bool | None = None,
+        disable_task_status_change: bool | None = None,
         configs: Mapping[str, Any] | None = None,
         use_uuid: str | None = None,
     ) -> Session:
@@ -92,6 +94,8 @@ async def create(
         Args:
             user: Optional user identifier string. Defaults to None.
             disable_task_tracking: Whether to disable task tracking for this session. Defaults to None (server default: False).
+            disable_task_status_change: Whether to disable automatic task status changes. When True,
+                the task agent will not set tasks to success/failed automatically. Defaults to None (server default: False).
             configs: Optional session configuration dictionary. Defaults to None.
             use_uuid: Optional UUID string to use as the session ID. If not provided, a UUID will be auto-generated.
                 If a session with this UUID already exists, a 409 Conflict error will be raised.
@@ -107,6 +111,8 @@ async def create(
             payload["user"] = user
         if disable_task_tracking is not None:
             payload["disable_task_tracking"] = disable_task_tracking
+        if disable_task_status_change is not None:
+            payload["disable_task_status_change"] = disable_task_status_change
         if configs is not None:
             payload["configs"] = configs
         if use_uuid is not None:
@@ -178,6 +184,33 @@ async def get_tasks(
         )
         return GetTasksOutput.model_validate(data)
 
+    async def update_task_status(
+        self,
+        session_id: str,
+        task_id: str,
+        *,
+        status: str,
+    ) -> Task:
+        """Update a task's status.
+
+        Setting status to "success" or "failed" triggers the skill learning pipeline.
+
+        Args:
+            session_id: The UUID of the session.
+            task_id: The UUID of the task.
+            status: New status for the task. Must be one of: "success", "failed", "running", "pending".
+
+        Returns:
+            The updated Task object.
+        """
+        payload = {"status": status}
+        data = await self._requester.request(
+            "PATCH",
+            f"/session/{session_id}/task/{task_id}/status",
+            json_data=payload,
+        )
+        return Task.model_validate(data)
+
     async def get_session_summary(
         self,
         session_id: str,

src/client/acontext-py/src/acontext/resources/sessions.py

@@ -17,6 +17,7 @@
     Message,
     MessageObservingStatus,
     Session,
+    Task,
     TokenCounts,
 )
 from ..uploads import FileUpload, normalize_file_upload
@@ -84,6 +85,7 @@ def create(
         *,
         user: str | None = None,
         disable_task_tracking: bool | None = None,
+        disable_task_status_change: bool | None = None,
         configs: Mapping[str, Any] | None = None,
         use_uuid: str | None = None,
     ) -> Session:
@@ -92,6 +94,8 @@ def create(
         Args:
             user: Optional user identifier string. Defaults to None.
             disable_task_tracking: Whether to disable task tracking for this session. Defaults to None (server default: False).
+            disable_task_status_change: Whether to disable automatic task status changes. When True,
+                the task agent will not set tasks to success/failed automatically. Defaults to None (server default: False).
             configs: Optional session configuration dictionary. Defaults to None.
             use_uuid: Optional UUID string to use as the session ID. If not provided, a UUID will be auto-generated.
                 If a session with this UUID already exists, a 409 Conflict error will be raised.
@@ -107,6 +111,8 @@ def create(
             payload["user"] = user
         if disable_task_tracking is not None:
             payload["disable_task_tracking"] = disable_task_tracking
+        if disable_task_status_change is not None:
+            payload["disable_task_status_change"] = disable_task_status_change
         if configs is not None:
             payload["configs"] = configs
         if use_uuid is not None:
@@ -178,6 +184,33 @@ def get_tasks(
         )
         return GetTasksOutput.model_validate(data)
 
+    def update_task_status(
+        self,
+        session_id: str,
+        task_id: str,
+        *,
+        status: str,
+    ) -> Task:
+        """Update a task's status.
+
+        Setting status to "success" or "failed" triggers the skill learning pipeline.
+
+        Args:
+            session_id: The UUID of the session.
+            task_id: The UUID of the task.
+            status: New status for the task. Must be one of: "success", "failed", "running", "pending".
+
+        Returns:
+            The updated Task object.
+        """
+        payload = {"status": status}
+        data = self._requester.request(
+            "PATCH",
+            f"/session/{session_id}/task/{task_id}/status",
+            json_data=payload,
+        )
+        return Task.model_validate(data)
+
     def get_session_summary(
         self,
         session_id: str,

src/client/acontext-py/src/acontext/types/session.py

@@ -177,6 +177,10 @@ class Session(BaseModel):
     disable_task_tracking: bool = Field(
         False, description="Whether task tracking is disabled for this session"
     )
+    disable_task_status_change: bool = Field(
+        False,
+        description="Whether automatic task status changes are disabled for this session",
+    )
     configs: dict[str, Any] | None = Field(
         None, description="Session configuration dictionary"
     )