Merge pull request #523 from kitsudaiki/feat/update-for-multiple-inputs-and-outputs

kitsudaiki · web-flow · commit 71322b6f5cc5 · 2025-03-16T13:05:42.000+01:00
related issue: #454
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -22,10 +22,16 @@
 
 - removed the reductions-mode setting
 
+#### Database-Breaking
+
+- Added `task_uuid` to datasets
+- Datasets, which are created as result form request-task, have now a random uuid instead of the uuid of the initial request-task
+
 ### Added
 
 - delete-all functions for all resources were added to the python-sdk
 - function to wait for a task to be finished was added to the python-sdk
+- a request-task with multiple output-hexagon generates now multiple datasets for each of them
 
 ### Changed
 
diff --git a/docs/frontend/cli_sdk_docu.md b/docs/frontend/cli_sdk_docu.md
@@ -739,10 +739,11 @@ label-file of the same dataset.
     | NUMBER OF COLUMNS | 2                                                                                                |
     | NUMBER OF ROWS    | 1723                                                                                             |
     | DESCRIPTION       | {"test_input":{"column_end":1,"column_start":0},"test_output":{"column_end":2,"column_start":1}} |
+    | TASK UUID         | 6f2bbcd2-7081-4b08-ae1d-16e6cd6f54c4                                                             |
     | VISIBILITY        | private                                                                                          |
     | OWNER ID          | asdf                                                                                             |
     | PROJECT ID        | admin                                                                                            |
-    | CREATED AT        | <nil>                                                                                            |
+    | CREATED AT        | 2025-03-15 22:02:47                                                                              |
     +-------------------+--------------------------------------------------------------------------------------------------+
     ```
 
@@ -781,16 +782,17 @@ Get information about a specific dataset.
     hanamictl dataset get 146bacb3-b5bf-485b-a2e8-d1812b57eb63
 
     +-------------------+-----------------------------------------------------------------------------------------------+
-    | UUID              | 146bacb3-b5bf-485b-a2e8-d1812b57eb63                                                          |
-    | NAME              | cli_test_dataset                                                                              |
+    | UUID              | 91c3799d-556b-4562-ae6d-96d631a46a42                                                          |
+    | NAME              | cli_test_dataset_req                                                                          |
     | VERSION           | v1.0alpha                                                                                     |
     | NUMBER OF COLUMNS | 794                                                                                           |
-    | NUMBER OF ROWS    | 60000                                                                                         |
+    | NUMBER OF ROWS    | 10000                                                                                         |
     | DESCRIPTION       | {"label":{"column_end":794,"column_start":784},"picture":{"column_end":784,"column_start":0}} |
+    | TASK UUID         | 6f2bbcd2-7081-4b08-ae1d-16e6cd6f54c4                                                          |
     | VISIBILITY        | private                                                                                       |
     | OWNER ID          | asdf                                                                                          |
     | PROJECT ID        | admin                                                                                         |
-    | CREATED AT        | <nil>                                                                                         |
+    | CREATED AT        | 2025-03-15 22:02:47                                                                           |
     +-------------------+-----------------------------------------------------------------------------------------------+
     ```
 
@@ -818,6 +820,7 @@ Get information about a specific dataset.
     #     "owner_id": "asdf",
     #     "project_id": "admin",
     #     "type": "mnist",
+    #     "task_uuid": "6f2bbcd2-7081-4b08-ae1d-16e6cd6f54c4",
     #     "uuid": "6f2bbcd2-7081-4b08-ae1d-16e6cd6f54c4",
     #     "visibility": "private"
     # }
@@ -838,13 +841,13 @@ List all visible datasets.
     ```bash
     hanamictl dataset list
 
-    +--------------------------------------+------------------------+------------+----------+------------+---------------------+
-    |                 UUID                 |          NAME          | VISIBILITY | OWNER ID | PROJECT ID |     CREATED AT      |
-    +--------------------------------------+------------------------+------------+----------+------------+---------------------+
-    | 140356ef-aebc-4069-9ef8-1c0e6d13d85f | cli_test_dataset_train | private    | asdf     | admin      | 2024-07-12 20:46:02 |
-    | 8d7ec569-fca7-4ca7-85f6-519ad05472ad | cli_test_dataset_req   | private    | asdf     | admin      | 2024-07-12 20:46:02 |
-    | 146bacb3-b5bf-485b-a2e8-d1812b57eb63 | cli_test_dataset       | private    | asdf     | admin      | 2024-07-12 20:52:21 |
-    +--------------------------------------+------------------------+------------+----------+------------+---------------------+
+    +--------------------------------------+------------------------+--------------------------------------+------------+----------+------------+---------------------+
+    |                 UUID                 |          NAME          |              TASK UUID               | VISIBILITY | OWNER ID | PROJECT ID |     CREATED AT      |
+    +--------------------------------------+------------------------+--------------------------------------+------------+----------+------------+---------------------+
+    | 8126302c-6d51-43d5-8f34-2c0a574b9ed7 | cli_test_dataset_train |                                      | private    | asdf     | admin      | 2025-03-15 22:02:45 |
+    | 91c3799d-556b-4562-ae6d-96d631a46a42 | cli_test_dataset_req   |                                      | private    | asdf     | admin      | 2025-03-15 22:02:47 |
+    | 91c3799d-556b-4562-ae6d-96d631a46a42 | cli_request_test_task  | 6f2bbcd2-7081-4b08-ae1d-16e6cd6f54c4 | private    | asdf     | admin      | 2025-03-15 22:02:47 |
+    +--------------------------------------+------------------------+--------------------------------------+------------+----------+------------+---------------------+
     ```
 
 === "Python-SDK"
@@ -864,21 +867,41 @@ List all visible datasets.
     # {
     #     "body": [
     #         [
-    #             "6f2bbcd2-7081-4b08-ae1d-16e6cd6f54c4",
+    #             "2025-03-15 21:18:52",
+    #             "329553e0-c4c4-4139-95ee-3ace764739a5",
     #             "admin",
     #             "asdf",
     #             "private",
-    #             "train_test_dataset",
-    #             "mnist"
+    #             "cli_test_dataset_train",
+    #             ""
+    #         ],
+    #         [
+    #             "2025-03-15 21:18:54",
+    #             "887d1b59-8a3e-4814-b148-8405c1d240e0",
+    #             "admin",
+    #             "asdf",
+    #             "private",
+    #             "cli_test_dataset_req",
+    #             ""
+    #         ],
+    #         [
+    #             "2025-03-15 21:20:17",
+    #             "38f464b8-d627-4ab5-944c-94391dd1962d",
+    #             "admin",
+    #             "asdf",
+    #             "private",
+    #             "cli_request_test_task",
+    #             "38f464b8-d627-4ab5-944c-94391dd1962d"
     #         ]
     #     ],
     #     "header": [
+    #         "created_at",
     #         "uuid",
     #         "project_id",
     #         "owner_id",
     #         "visibility",
     #         "name",
-    #         "type"
+    #         "task_uuid"
     #     ]
     # }
     ```
diff --git a/src/cli/hanamictl/resources/dataset.go b/src/cli/hanamictl/resources/dataset.go
@@ -45,6 +45,7 @@ var datasetHeader = []string{
 	"number_of_columns",
 	"number_of_rows",
 	"description",
+	"task_uuid",
 	"visibility",
 	"owner_id",
 	"project_id",
diff --git a/src/hanami/src/api/http/dataset/csv/create_csv_dataset_v1_0.cpp b/src/hanami/src/api/http/dataset/csv/create_csv_dataset_v1_0.cpp
@@ -65,6 +65,11 @@ CreateCsvDataSetV1M0::CreateCsvDataSetV1M0() : Blossom("Init new csv-file datase
     registerOutputField("visibility", SAKURA_STRING_TYPE)
         .setComment("Visibility of the dataset (private, shared, public).");
 
+    registerOutputField("task_uuid", SAKURA_STRING_TYPE)
+        .setComment(
+            "In case that this dataset was created by a request-task, this contains the UUID of "
+            "the task for identifaction.");
+
     registerOutputField("uuid_input_file", SAKURA_STRING_TYPE)
         .setComment("UUID to identify the file for date upload of input-data.");
 
diff --git a/src/hanami/src/api/http/dataset/download_dataset_content_v1_0.cpp b/src/hanami/src/api/http/dataset/download_dataset_content_v1_0.cpp
@@ -78,9 +78,8 @@ DownloadDatasetContentV1M0::runTask(BlossomIO& blossomIO,
     const uint64_t rowOffset = blossomIO.input["row_offset"];
     const uint64_t numberOfRows = blossomIO.input["number_of_rows"];
 
-    DataSetFileHandle datasetFileHandle;
-
     // open files
+    DataSetFileHandle datasetFileHandle;
     ReturnStatus ret = getFileHandle(datasetFileHandle, datasetUuid, userContext, status, error);
     if (ret != OK) {
         return false;
diff --git a/src/hanami/src/api/http/dataset/get_dataset_v1_0.cpp b/src/hanami/src/api/http/dataset/get_dataset_v1_0.cpp
@@ -49,6 +49,11 @@ GetDataSetV1M0::GetDataSetV1M0() : Blossom("Get information of a specific datase
     registerOutputField("description", SAKURA_MAP_TYPE)
         .setComment("Description of the dataset content.");
 
+    registerOutputField("task_uuid", SAKURA_STRING_TYPE)
+        .setComment(
+            "In case that this dataset was created by a request-task, this contains the UUID of "
+            "the task for identifaction.");
+
     registerOutputField("owner_id", SAKURA_STRING_TYPE)
         .setComment("ID of the user, who created the dataset.");
 
@@ -109,6 +114,7 @@ GetDataSetV1M0::runTask(BlossomIO& blossomIO,
     blossomIO.output["uuid"] = datasetUuid;
     blossomIO.output["description"] = fileHandle.description;
     blossomIO.output["name"] = fileHandle.header.name.getName();
+    blossomIO.output["task_uuid"] = entry.taskUuid;
     blossomIO.output["owner_id"] = entry.ownerId;
     blossomIO.output["project_id"] = entry.projectId;
     blossomIO.output["visibility"] = entry.visibility;
diff --git a/src/hanami/src/api/http/dataset/list_dataset_v1_0.cpp b/src/hanami/src/api/http/dataset/list_dataset_v1_0.cpp
@@ -38,6 +38,7 @@ ListDataSetV1M0::ListDataSetV1M0() : Blossom("List all visible datasets.")
     headerMatch.push_back("owner_id");
     headerMatch.push_back("visibility");
     headerMatch.push_back("name");
+    headerMatch.push_back("task_uuid");
 
     registerOutputField("header", SAKURA_ARRAY_TYPE)
         .setComment("Array with the namings all columns of the table.")
diff --git a/src/hanami/src/api/http/dataset/mnist/create_mnist_dataset_v1_0.cpp b/src/hanami/src/api/http/dataset/mnist/create_mnist_dataset_v1_0.cpp
@@ -68,6 +68,11 @@ CreateMnistDataSetV1M0::CreateMnistDataSetV1M0() : Blossom("Init new mnist-file
     registerOutputField("visibility", SAKURA_STRING_TYPE)
         .setComment("Visibility of the dataset (private, shared, public).");
 
+    registerOutputField("task_uuid", SAKURA_STRING_TYPE)
+        .setComment(
+            "In case that this dataset was created by a request-task, this contains the UUID of "
+            "the task for identifaction.");
+
     registerOutputField("uuid_input_file", SAKURA_STRING_TYPE)
         .setComment("UUID to identify the file for date upload of input-data.");
 
diff --git a/src/hanami/src/api/http/task/create_request_task_v1_0.cpp b/src/hanami/src/api/http/task/create_request_task_v1_0.cpp
@@ -262,40 +262,17 @@ CreateRequestTaskV1M0::createResultDataset(Cluster* cluster,
                                            BlossomStatus& status,
                                            Hanami::ErrorContainer& error)
 {
-    const std::string datasetUuid = task->uuid.toString();
     RequestInfo* taskInfo = &std::get<RequestInfo>(task->info);
 
     bool success = false;
-    std::string targetFilePath = GET_STRING_CONFIG("storage", "dataset_location", success);
+    const std::string datasetLocation = GET_STRING_CONFIG("storage", "dataset_location", success);
     if (success == false) {
         status.statusCode = INTERNAL_SERVER_ERROR_RTYPE;
         error.addMessage("file-location to store dataset is missing in the config");
         return ERROR;
     }
 
-    if (targetFilePath.at(targetFilePath.size() - 1) != '/') {
-        targetFilePath.append("/");
-    }
-    targetFilePath.append(datasetName + datasetUuid);
-
-    // create new database-entry
-    DataSetTable::DataSetDbEntry dbEntry;
-    dbEntry.name = datasetName;
-    dbEntry.ownerId = userContext.userId;
-    dbEntry.projectId = userContext.projectId;
-    dbEntry.uuid = datasetUuid;
-    dbEntry.visibility = "private";
-    dbEntry.location = targetFilePath;
-
-    // update database
-    if (DataSetTable::getInstance()->addDataSet(dbEntry, userContext, error) != OK) {
-        status.statusCode = INTERNAL_SERVER_ERROR_RTYPE;
-        return ERROR;
-    }
-
-    // create description for new dataset
-    json description;
-    uint64_t totalNumberOfOutputs = 0;
+    // precheck input
     for (const json& item : resultMetaData) {
         if (item.contains("hexagon_name") == false) {
             status.statusCode = BAD_REQUEST_RTYPE;
@@ -307,35 +284,56 @@ CreateRequestTaskV1M0::createResultDataset(Cluster* cluster,
             status.errorMessage.append("'dataset_column' is missing");
             return INVALID_INPUT;
         }
+    }
 
+    // create new dataset, one for each output-hexagon
+    for (const json& item : resultMetaData) {
         const std::string hexagonName = item["hexagon_name"];
         const std::string columnName = item["dataset_column"];
+        const std::string datasetUuid = generateUuid().toString();
+
+        // create local file path
+        std::string targetFilePath = datasetLocation;
+        if (targetFilePath.at(targetFilePath.size() - 1) != '/') {
+            targetFilePath.append("/");
+        }
+        targetFilePath.append(datasetName + "_" + hexagonName + "_" + datasetUuid);
+
+        // create new database-entry
+        DataSetTable::DataSetDbEntry dbEntry;
+        dbEntry.uuid = datasetUuid;
+        dbEntry.name = datasetName;
+        dbEntry.ownerId = userContext.userId;
+        dbEntry.projectId = userContext.projectId;
+        dbEntry.visibility = "private";
+        dbEntry.location = targetFilePath;
+        dbEntry.taskUuid = task->uuid.toString();
+
+        // update database
+        if (DataSetTable::getInstance()->addDataSet(dbEntry, userContext, error) != OK) {
+            status.statusCode = INTERNAL_SERVER_ERROR_RTYPE;
+            return ERROR;
+        }
+
         const uint64_t numberOfOutputs = cluster->outputInterfaces[hexagonName].ioBuffer.size();
-        totalNumberOfOutputs += numberOfOutputs;
 
         // prepare description of the dataset
+        json description;
         json descriptionEntry;
         descriptionEntry["column_start"] = 0;
         descriptionEntry["column_end"] = numberOfOutputs;
         description[columnName] = descriptionEntry;
-    }
 
-    // initialize dataset-file
-    ReturnStatus ret = initNewDataSetFile(
-        targetFilePath, datasetName, description, FLOAT_TYPE, totalNumberOfOutputs, error);
-    if (ret == INVALID_INPUT) {
-        status.errorMessage = "Data-set with uuid '" + datasetUuid + "' not found";
-        status.statusCode = NOT_FOUND_RTYPE;
-    }
+        // initialize dataset-file
+        ReturnStatus ret = initNewDataSetFile(
+            targetFilePath, datasetName, description, FLOAT_TYPE, numberOfOutputs, error);
+        if (ret == INVALID_INPUT) {
+            status.errorMessage = "Data-set with uuid '" + datasetUuid + "' not found";
+            status.statusCode = NOT_FOUND_RTYPE;
+        }
 
-    // prepare io-buffer
-    for (const json& item : resultMetaData) {
-        const std::string hexagonName = item["hexagon_name"];
-        const std::string columnName = item["dataset_column"];
         DataSetFileHandle fileHandle;
-
-        const ReturnStatus ret
-            = fillTaskIo(fileHandle, userContext, columnName, datasetUuid, status, error);
+        ret = fillTaskIo(fileHandle, userContext, columnName, datasetUuid, status, error);
         if (ret != OK) {
             return ret;
         }
diff --git a/src/hanami/src/core/cluster/cluster_init.cpp b/src/hanami/src/core/cluster/cluster_init.cpp
@@ -237,6 +237,7 @@ connectAllHexagons(Cluster* cluster)
  * @param cluster pointer to cluster
  * @param currentHexagon actual hexagon
  * @param maxPathLength maximum path length left
+ * @param sourceHexagonId id of the source-hexagon, which initialized the search
  *
  * @return last hexagon-id of the gone path
  */
@@ -246,6 +247,7 @@ goToNextInitHexagon(Cluster* cluster,
                     int32_t& maxPathLength,
                     const uint32_t sourceHexagonId)
 {
+    std::cout << "-> " << currentHexagon->header.hexagonId << std::endl;
     // check path-length to not go too far
     maxPathLength--;
     if (maxPathLength <= 0 && currentHexagon->header.hexagonId != sourceHexagonId) {
@@ -260,20 +262,27 @@ goToNextInitHexagon(Cluster* cluster,
         return currentHexagon->header.hexagonId;
     }
 
-    // get a random possible next hexagon
+    // filter all avaialable next hexagons
+    // TODO: require a better solution, which doesn't need to filter this every time
+    std::vector<uint32_t> availableNext;
     const uint8_t possibleNextSides[7] = {9, 3, 1, 4, 11, 5, 2};
-    const uint8_t startSide = possibleNextSides[rand() % 7];
     for (uint32_t i = 0; i < 7; i++) {
-        const uint8_t side = possibleNextSides[(i + startSide) % 7];
+        const uint8_t side = possibleNextSides[i];
         const uint32_t nextHexagonId = currentHexagon->neighbors[side];
         if (nextHexagonId != UNINIT_STATE_32) {
-            return goToNextInitHexagon(
-                cluster, &cluster->hexagons[nextHexagonId], maxPathLength, sourceHexagonId);
+            availableNext.push_back(nextHexagonId);
         }
     }
 
-    // if no further next hexagon was found, the give back tha actual one as end of the path
-    return currentHexagon->header.hexagonId;
+    // handle end of the path
+    if (availableNext.size() == 0) {
+        return currentHexagon->header.hexagonId;
+    }
+
+    // select one of the filtered results
+    const uint32_t selectedNext = availableNext[rand() % availableNext.size()];
+    return goToNextInitHexagon(
+        cluster, &cluster->hexagons[selectedNext], maxPathLength, sourceHexagonId);
 }
 
 /**
@@ -285,9 +294,11 @@ void
 initializeTargetHexagonList(Cluster* cluster)
 {
     for (Hexagon& hexagon : cluster->hexagons) {
+        std::vector<uint32_t> availableNext;
         for (uint32_t counter = 0; counter < NUMBER_OF_POSSIBLE_NEXT; counter++) {
             int32_t maxPathLength = cluster->clusterHeader.settings.maxConnectionDistance;
             Hexagon* baseHexagon = &cluster->hexagons[hexagon.header.axonTarget];
+            std::cout << hexagon.header.hexagonId << std::endl;
             const uint32_t targetHexagonId = goToNextInitHexagon(
                 cluster, baseHexagon, maxPathLength, hexagon.header.hexagonId);
             if (hexagon.header.hexagonId != targetHexagonId) {
diff --git a/src/hanami/src/core/io/data_set/dataset_file_io.cpp b/src/hanami/src/core/io/data_set/dataset_file_io.cpp
@@ -295,7 +295,7 @@ initNewDataSetFile(const std::string& filePath,
     // check source
     if (std::filesystem::exists(filePath) == true) {
         error.addMessage("Data-set file '" + filePath + "' already exist.");
-        return INVALID_INPUT;
+        return ERROR;
     }
 
     DataSetHeader header;
diff --git a/src/hanami/src/core/processing/logical_host.cpp b/src/hanami/src/core/processing/logical_host.cpp
diff --git a/src/hanami/src/database/dataset_table.cpp b/src/hanami/src/database/dataset_table.cpp
diff --git a/src/hanami/src/database/dataset_table.h b/src/hanami/src/database/dataset_table.h
diff --git a/testing/go_cli_api/cli_test.sh b/testing/go_cli_api/cli_test.sh
diff --git a/testing/python_sdk_api/sdk_api_test.py b/testing/python_sdk_api/sdk_api_test.py
