feat(#454): support multiple outputs

Result-datasets were changed. When making a request-task
against a cluster with multiple outputs, each output-hexagon
generates its own dataset. Because of this, the uuid of the
request-task is no longer the uuid of the resulting dataset,
because there can be multiple datasets as result. So the
datasets now have a 'task_uuid'-field, which contains the
uuid of the request-task.

Author: kitsudaiki

docs/frontend/cli_sdk_docu.md

@@ -739,10 +739,11 @@ label-file of the same dataset.
     | NUMBER OF COLUMNS | 2                                                                                                |
     | NUMBER OF ROWS    | 1723                                                                                             |
     | DESCRIPTION       | {"test_input":{"column_end":1,"column_start":0},"test_output":{"column_end":2,"column_start":1}} |
+    | TASK UUID         | 6f2bbcd2-7081-4b08-ae1d-16e6cd6f54c4                                                             |
     | VISIBILITY        | private                                                                                          |
     | OWNER ID          | asdf                                                                                             |
     | PROJECT ID        | admin                                                                                            |
-    | CREATED AT        | <nil>                                                                                            |
+    | CREATED AT        | 2025-03-15 22:02:47                                                                              |
     +-------------------+--------------------------------------------------------------------------------------------------+
     ```
 
@@ -781,16 +782,17 @@ Get information about a specific dataset.
     hanamictl dataset get 146bacb3-b5bf-485b-a2e8-d1812b57eb63
 
     +-------------------+-----------------------------------------------------------------------------------------------+
-    | UUID              | 146bacb3-b5bf-485b-a2e8-d1812b57eb63                                                          |
-    | NAME              | cli_test_dataset                                                                              |
+    | UUID              | 91c3799d-556b-4562-ae6d-96d631a46a42                                                          |
+    | NAME              | cli_test_dataset_req                                                                          |
     | VERSION           | v1.0alpha                                                                                     |
     | NUMBER OF COLUMNS | 794                                                                                           |
-    | NUMBER OF ROWS    | 60000                                                                                         |
+    | NUMBER OF ROWS    | 10000                                                                                         |
     | DESCRIPTION       | {"label":{"column_end":794,"column_start":784},"picture":{"column_end":784,"column_start":0}} |
+    | TASK UUID         | 6f2bbcd2-7081-4b08-ae1d-16e6cd6f54c4                                                          |
     | VISIBILITY        | private                                                                                       |
     | OWNER ID          | asdf                                                                                          |
     | PROJECT ID        | admin                                                                                         |
-    | CREATED AT        | <nil>                                                                                         |
+    | CREATED AT        | 2025-03-15 22:02:47                                                                           |
     +-------------------+-----------------------------------------------------------------------------------------------+
     ```
 
@@ -818,6 +820,7 @@ Get information about a specific dataset.
     #     "owner_id": "asdf",
     #     "project_id": "admin",
     #     "type": "mnist",
+    #     "task_uuid": "6f2bbcd2-7081-4b08-ae1d-16e6cd6f54c4",
     #     "uuid": "6f2bbcd2-7081-4b08-ae1d-16e6cd6f54c4",
     #     "visibility": "private"
     # }
@@ -838,13 +841,13 @@ List all visible datasets.
     ```bash
     hanamictl dataset list
 
-    +--------------------------------------+------------------------+------------+----------+------------+---------------------+
-    |                 UUID                 |          NAME          | VISIBILITY | OWNER ID | PROJECT ID |     CREATED AT      |
-    +--------------------------------------+------------------------+------------+----------+------------+---------------------+
-    | 140356ef-aebc-4069-9ef8-1c0e6d13d85f | cli_test_dataset_train | private    | asdf     | admin      | 2024-07-12 20:46:02 |
-    | 8d7ec569-fca7-4ca7-85f6-519ad05472ad | cli_test_dataset_req   | private    | asdf     | admin      | 2024-07-12 20:46:02 |
-    | 146bacb3-b5bf-485b-a2e8-d1812b57eb63 | cli_test_dataset       | private    | asdf     | admin      | 2024-07-12 20:52:21 |
-    +--------------------------------------+------------------------+------------+----------+------------+---------------------+
+    +--------------------------------------+------------------------+--------------------------------------+------------+----------+------------+---------------------+
+    |                 UUID                 |          NAME          |              TASK UUID               | VISIBILITY | OWNER ID | PROJECT ID |     CREATED AT      |
+    +--------------------------------------+------------------------+--------------------------------------+------------+----------+------------+---------------------+
+    | 8126302c-6d51-43d5-8f34-2c0a574b9ed7 | cli_test_dataset_train |                                      | private    | asdf     | admin      | 2025-03-15 22:02:45 |
+    | 91c3799d-556b-4562-ae6d-96d631a46a42 | cli_test_dataset_req   |                                      | private    | asdf     | admin      | 2025-03-15 22:02:47 |
+    | 91c3799d-556b-4562-ae6d-96d631a46a42 | cli_request_test_task  | 6f2bbcd2-7081-4b08-ae1d-16e6cd6f54c4 | private    | asdf     | admin      | 2025-03-15 22:02:47 |
+    +--------------------------------------+------------------------+--------------------------------------+------------+----------+------------+---------------------+
     ```
 
 === "Python-SDK"
@@ -864,21 +867,41 @@ List all visible datasets.
     # {
     #     "body": [
     #         [
-    #             "6f2bbcd2-7081-4b08-ae1d-16e6cd6f54c4",
+    #             "2025-03-15 21:18:52",
+    #             "329553e0-c4c4-4139-95ee-3ace764739a5",
     #             "admin",
     #             "asdf",
     #             "private",
-    #             "train_test_dataset",
-    #             "mnist"
+    #             "cli_test_dataset_train",
+    #             ""
+    #         ],
+    #         [
+    #             "2025-03-15 21:18:54",
+    #             "887d1b59-8a3e-4814-b148-8405c1d240e0",
+    #             "admin",
+    #             "asdf",
+    #             "private",
+    #             "cli_test_dataset_req",
+    #             ""
+    #         ],
+    #         [
+    #             "2025-03-15 21:20:17",
+    #             "38f464b8-d627-4ab5-944c-94391dd1962d",
+    #             "admin",
+    #             "asdf",
+    #             "private",
+    #             "cli_request_test_task",
+    #             "38f464b8-d627-4ab5-944c-94391dd1962d"
     #         ]
     #     ],
     #     "header": [
+    #         "created_at",
     #         "uuid",
     #         "project_id",
     #         "owner_id",
     #         "visibility",
     #         "name",
-    #         "type"
+    #         "task_uuid"
     #     ]
     # }
     ```

src/cli/hanamictl/resources/dataset.go

@@ -45,6 +45,7 @@ var datasetHeader = []string{
 	"number_of_columns",
 	"number_of_rows",
 	"description",
+	"task_uuid",
 	"visibility",
 	"owner_id",
 	"project_id",

src/hanami/src/api/http/dataset/csv/create_csv_dataset_v1_0.cpp

@@ -65,6 +65,11 @@ CreateCsvDataSetV1M0::CreateCsvDataSetV1M0() : Blossom("Init new csv-file datase
     registerOutputField("visibility", SAKURA_STRING_TYPE)
         .setComment("Visibility of the dataset (private, shared, public).");
 
+    registerOutputField("task_uuid", SAKURA_STRING_TYPE)
+        .setComment(
+            "In case that this dataset was created by a request-task, this contains the UUID of "
+            "the task for identifaction.");
+
     registerOutputField("uuid_input_file", SAKURA_STRING_TYPE)
         .setComment("UUID to identify the file for date upload of input-data.");
 

src/hanami/src/api/http/dataset/download_dataset_content_v1_0.cpp

@@ -78,9 +78,8 @@ DownloadDatasetContentV1M0::runTask(BlossomIO& blossomIO,
     const uint64_t rowOffset = blossomIO.input["row_offset"];
     const uint64_t numberOfRows = blossomIO.input["number_of_rows"];
 
-    DataSetFileHandle datasetFileHandle;
-
     // open files
+    DataSetFileHandle datasetFileHandle;
     ReturnStatus ret = getFileHandle(datasetFileHandle, datasetUuid, userContext, status, error);
     if (ret != OK) {
         return false;

src/hanami/src/api/http/dataset/get_dataset_v1_0.cpp

@@ -49,6 +49,11 @@ GetDataSetV1M0::GetDataSetV1M0() : Blossom("Get information of a specific datase
     registerOutputField("description", SAKURA_MAP_TYPE)
         .setComment("Description of the dataset content.");
 
+    registerOutputField("task_uuid", SAKURA_STRING_TYPE)
+        .setComment(
+            "In case that this dataset was created by a request-task, this contains the UUID of "
+            "the task for identifaction.");
+
     registerOutputField("owner_id", SAKURA_STRING_TYPE)
         .setComment("ID of the user, who created the dataset.");
 
@@ -109,6 +114,7 @@ GetDataSetV1M0::runTask(BlossomIO& blossomIO,
     blossomIO.output["uuid"] = datasetUuid;
     blossomIO.output["description"] = fileHandle.description;
     blossomIO.output["name"] = fileHandle.header.name.getName();
+    blossomIO.output["task_uuid"] = entry.taskUuid;
     blossomIO.output["owner_id"] = entry.ownerId;
     blossomIO.output["project_id"] = entry.projectId;
     blossomIO.output["visibility"] = entry.visibility;

src/hanami/src/api/http/dataset/list_dataset_v1_0.cpp

@@ -38,6 +38,7 @@ ListDataSetV1M0::ListDataSetV1M0() : Blossom("List all visible datasets.")
     headerMatch.push_back("owner_id");
     headerMatch.push_back("visibility");
     headerMatch.push_back("name");
+    headerMatch.push_back("task_uuid");
 
     registerOutputField("header", SAKURA_ARRAY_TYPE)
         .setComment("Array with the namings all columns of the table.")

src/hanami/src/api/http/dataset/mnist/create_mnist_dataset_v1_0.cpp

@@ -68,6 +68,11 @@ CreateMnistDataSetV1M0::CreateMnistDataSetV1M0() : Blossom("Init new mnist-file
     registerOutputField("visibility", SAKURA_STRING_TYPE)
         .setComment("Visibility of the dataset (private, shared, public).");
 
+    registerOutputField("task_uuid", SAKURA_STRING_TYPE)
+        .setComment(
+            "In case that this dataset was created by a request-task, this contains the UUID of "
+            "the task for identifaction.");
+
     registerOutputField("uuid_input_file", SAKURA_STRING_TYPE)
         .setComment("UUID to identify the file for date upload of input-data.");
 

src/hanami/src/api/http/task/create_request_task_v1_0.cpp

@@ -266,36 +266,14 @@ CreateRequestTaskV1M0::createResultDataset(Cluster* cluster,
     RequestInfo* taskInfo = &std::get<RequestInfo>(task->info);
 
     bool success = false;
-    std::string targetFilePath = GET_STRING_CONFIG("storage", "dataset_location", success);
+    const std::string datasetLocation = GET_STRING_CONFIG("storage", "dataset_location", success);
     if (success == false) {
         status.statusCode = INTERNAL_SERVER_ERROR_RTYPE;
         error.addMessage("file-location to store dataset is missing in the config");
         return ERROR;
     }
 
-    if (targetFilePath.at(targetFilePath.size() - 1) != '/') {
-        targetFilePath.append("/");
-    }
-    targetFilePath.append(datasetName + datasetUuid);
-
-    // create new database-entry
-    DataSetTable::DataSetDbEntry dbEntry;
-    dbEntry.name = datasetName;
-    dbEntry.ownerId = userContext.userId;
-    dbEntry.projectId = userContext.projectId;
-    dbEntry.uuid = datasetUuid;
-    dbEntry.visibility = "private";
-    dbEntry.location = targetFilePath;
-
-    // update database
-    if (DataSetTable::getInstance()->addDataSet(dbEntry, userContext, error) != OK) {
-        status.statusCode = INTERNAL_SERVER_ERROR_RTYPE;
-        return ERROR;
-    }
-
-    // create description for new dataset
-    json description;
-    uint64_t totalNumberOfOutputs = 0;
+    // precheck input
     for (const json& item : resultMetaData) {
         if (item.contains("hexagon_name") == false) {
             status.statusCode = BAD_REQUEST_RTYPE;
@@ -307,35 +285,55 @@ CreateRequestTaskV1M0::createResultDataset(Cluster* cluster,
             status.errorMessage.append("'dataset_column' is missing");
             return INVALID_INPUT;
         }
+    }
 
+    // create new dataset, one for each output-hexagon
+    for (const json& item : resultMetaData) {
         const std::string hexagonName = item["hexagon_name"];
         const std::string columnName = item["dataset_column"];
+
+        // create local file path
+        std::string targetFilePath = datasetLocation;
+        if (targetFilePath.at(targetFilePath.size() - 1) != '/') {
+            targetFilePath.append("/");
+        }
+        targetFilePath.append(datasetName + "_" + hexagonName + "_" + datasetUuid);
+
+        // create new database-entry
+        DataSetTable::DataSetDbEntry dbEntry;
+        dbEntry.uuid = datasetUuid;
+        dbEntry.name = datasetName;
+        dbEntry.ownerId = userContext.userId;
+        dbEntry.projectId = userContext.projectId;
+        dbEntry.visibility = "private";
+        dbEntry.location = targetFilePath;
+        dbEntry.taskUuid = task->uuid.toString();
+
+        // update database
+        if (DataSetTable::getInstance()->addDataSet(dbEntry, userContext, error) != OK) {
+            status.statusCode = INTERNAL_SERVER_ERROR_RTYPE;
+            return ERROR;
+        }
+
         const uint64_t numberOfOutputs = cluster->outputInterfaces[hexagonName].ioBuffer.size();
-        totalNumberOfOutputs += numberOfOutputs;
 
         // prepare description of the dataset
+        json description;
         json descriptionEntry;
         descriptionEntry["column_start"] = 0;
         descriptionEntry["column_end"] = numberOfOutputs;
         description[columnName] = descriptionEntry;
-    }
 
-    // initialize dataset-file
-    ReturnStatus ret = initNewDataSetFile(
-        targetFilePath, datasetName, description, FLOAT_TYPE, totalNumberOfOutputs, error);
-    if (ret == INVALID_INPUT) {
-        status.errorMessage = "Data-set with uuid '" + datasetUuid + "' not found";
-        status.statusCode = NOT_FOUND_RTYPE;
-    }
+        // initialize dataset-file
+        ReturnStatus ret = initNewDataSetFile(
+            targetFilePath, datasetName, description, FLOAT_TYPE, numberOfOutputs, error);
+        if (ret == INVALID_INPUT) {
+            status.errorMessage = "Data-set with uuid '" + datasetUuid + "' not found";
+            status.statusCode = NOT_FOUND_RTYPE;
+        }
 
-    // prepare io-buffer
-    for (const json& item : resultMetaData) {
-        const std::string hexagonName = item["hexagon_name"];
-        const std::string columnName = item["dataset_column"];
         DataSetFileHandle fileHandle;
-
-        const ReturnStatus ret
-            = fillTaskIo(fileHandle, userContext, columnName, datasetUuid, status, error);
+        ret = fillTaskIo(fileHandle, userContext, columnName, datasetUuid, status, error);
         if (ret != OK) {
             return ret;
         }

src/hanami/src/core/io/data_set/dataset_file_io.cpp

@@ -295,7 +295,7 @@ initNewDataSetFile(const std::string& filePath,
     // check source
     if (std::filesystem::exists(filePath) == true) {
         error.addMessage("Data-set file '" + filePath + "' already exist.");
-        return INVALID_INPUT;
+        return ERROR;
     }
 
     DataSetHeader header;

src/hanami/src/core/processing/logical_host.cpp

@@ -129,15 +129,17 @@ handleClientOutput(Cluster* cluster)
     // send output back if a client-connection is set
 
     Task* actualTask = cluster->getCurrentTask();
+    void* data = nullptr;
+
     if (actualTask != nullptr && actualTask->type == REQUEST_TASK) {
         RequestInfo* info = &std::get<RequestInfo>(actualTask->info);
 
         for (auto& [name, outputInterface] : cluster->outputInterfaces) {
             DataSetFileHandle* fileHandle = &info->results[name];
             const uint64_t ioBufferSize = convertOutputToBuffer(&outputInterface);
             // TODO: handle return status
-            appendToDataSet(
-                *fileHandle, &outputInterface.ioBuffer[0], ioBufferSize * sizeof(float), error);
+            data = &outputInterface.ioBuffer[0];
+            appendToDataSet(*fileHandle, data, ioBufferSize * sizeof(float), error);
         }
     }
     if (cluster->msgClient != nullptr) {