diff --git a/rcl_yaml_param_parser/include/rcl_yaml_param_parser/types.h b/rcl_yaml_param_parser/include/rcl_yaml_param_parser/types.h index e0e4b5f1c..72cdf7ed2 100644 --- a/rcl_yaml_param_parser/include/rcl_yaml_param_parser/types.h +++ b/rcl_yaml_param_parser/include/rcl_yaml_param_parser/types.h @@ -76,6 +76,7 @@ typedef struct rcl_variant_s int64_t * integer_value; ///< If integer, gets stored here double * double_value; ///< If double, gets stored here char * string_value; ///< If string, gets stored here + char * yaml_value; ///< If structured YAML, gets stored here -> Is this right? rcl_byte_array_t * byte_array_value; ///< If array of bytes rcl_bool_array_t * bool_array_value; ///< If array of bool's rcl_int64_array_t * integer_array_value; ///< If array of integers diff --git a/rcl_yaml_param_parser/src/impl/parse.h b/rcl_yaml_param_parser/src/impl/parse.h index 99f5d28e7..6948277f6 100644 --- a/rcl_yaml_param_parser/src/impl/parse.h +++ b/rcl_yaml_param_parser/src/impl/parse.h @@ -49,12 +49,33 @@ rcutils_ret_t parse_value( data_types_t * seq_data_type, rcl_params_t * params_st); +RCL_YAML_PARAM_PARSER_PUBLIC +RCUTILS_WARN_UNUSED +rcutils_ret_t initialize_emitter_string( + yaml_emitter_t * emitter); + +RCL_YAML_PARAM_PARSER_PUBLIC +RCUTILS_WARN_UNUSED +rcutils_ret_t end_emitter_string( + yaml_emitter_t * emitter); + +RCL_YAML_PARAM_PARSER_PUBLIC +RCUTILS_WARN_UNUSED +rcutils_ret_t write_structured_parameter_to_string( + yaml_parser_t * parser, + char * yaml_string_buffer, + size_t * written_size, + const size_t node_index, + const size_t parameter_index, + rcl_params_t * params_st); + RCL_YAML_PARAM_PARSER_PUBLIC RCUTILS_WARN_UNUSED rcutils_ret_t parse_key( const yaml_event_t event, uint32_t * map_level, bool * is_new_map, + bool * overwrite_previous_key, size_t * node_idx, size_t * parameter_idx, namespace_tracker_t * ns_tracker, @@ -64,9 +85,17 @@ RCL_YAML_PARAM_PARSER_PUBLIC RCUTILS_WARN_UNUSED rcutils_ret_t parse_file_events( yaml_parser_t * parser, + yaml_emitter_t * emitter, + char * emitter_string_buffer, + size_t * emitter_written_bytes, namespace_tracker_t * ns_tracker, rcl_params_t * params_st); +rcutils_ret_t write_event_to_emitter( + yaml_emitter_t * emitter, + yaml_event_t * event +); + RCL_YAML_PARAM_PARSER_PUBLIC RCUTILS_WARN_UNUSED rcutils_ret_t parse_value_events( diff --git a/rcl_yaml_param_parser/src/parse.c b/rcl_yaml_param_parser/src/parse.c index 63d162a1c..55786ac0e 100644 --- a/rcl_yaml_param_parser/src/parse.c +++ b/rcl_yaml_param_parser/src/parse.c @@ -274,6 +274,8 @@ rcutils_ret_t parse_value( return RCUTILS_RET_ERROR; } + // rahul-k-a: TODO Combine this with the parameter allocation part of + // `write_structured_parameter_to_string` and put everything in a seperate functiong rcutils_ret_t ret = RCUTILS_RET_OK; switch (val_type) { case DATA_TYPE_UNKNOWN: @@ -590,6 +592,80 @@ _validate_name(const char * name, rcutils_allocator_t allocator) return ret; } +/// +/// Makes a copy of an event and writes the copy to the emitter +/// +rcutils_ret_t write_event_to_emitter( + yaml_emitter_t * emitter, + yaml_event_t * event +) +{ + RCUTILS_CHECK_ARGUMENT_FOR_NULL(emitter, RCUTILS_RET_INVALID_ARGUMENT); + RCUTILS_CHECK_ARGUMENT_FOR_NULL(event, RCUTILS_RET_INVALID_ARGUMENT); + rcutils_ret_t ret = RCUTILS_RET_OK; + // The emitter deletes the event after writing + // So we need to make a copy of it and pass the copy to the emitter + yaml_event_t event_copy; + int success; + switch (event->type) { + /** A SCALAR event. */ + case YAML_SCALAR_EVENT: + { + success = yaml_scalar_event_initialize(&event_copy, event->data.scalar.anchor, + event->data.scalar.tag, event->data.scalar.value, event->data.scalar.length, + event->data.scalar.plain_implicit, event->data.scalar.quoted_implicit, + event->data.scalar.style); + break; + } + + /** A SEQUENCE-START event. */ + case YAML_SEQUENCE_START_EVENT: + { + success = yaml_sequence_start_event_initialize(&event_copy, + event->data.sequence_start.anchor, event->data.sequence_start.tag, + event->data.sequence_start.implicit, event->data.sequence_start.style); + break; + } + /** A SEQUENCE-END event. */ + case YAML_SEQUENCE_END_EVENT: + { + success = yaml_sequence_end_event_initialize(&event_copy); + break; + } + + /** A MAPPING-START event. */ + case YAML_MAPPING_START_EVENT: + { + success = yaml_mapping_start_event_initialize(&event_copy, event->data.mapping_start.anchor, + event->data.mapping_start.tag, event->data.mapping_start.implicit, + event->data.mapping_start.style); + break; + } + /** A MAPPING-END event. */ + case YAML_MAPPING_END_EVENT: + { + success = yaml_mapping_end_event_initialize(&event_copy); + break; + } + default: + { + RCUTILS_SET_ERROR_MSG_WITH_FORMAT_STRING("Unexpected YAML token of type %d", + (int) event->type); + ret = RCUTILS_RET_ERROR; + break; + } + } + + if (success == 0) { + RCUTILS_SET_ERROR_MSG_WITH_FORMAT_STRING("Unable to duplicate YAML token of type %d", + (int) event->type); + return RCUTILS_RET_ERROR; + } + yaml_emitter_emit(emitter, &event_copy); + return ret; +} + + /// /// Parse the key part of the pair /// @@ -597,6 +673,7 @@ rcutils_ret_t parse_key( const yaml_event_t event, uint32_t * map_level, bool * is_new_map, + bool * overwrite_previous_key, size_t * node_idx, size_t * parameter_idx, namespace_tracker_t * ns_tracker, @@ -709,11 +786,16 @@ rcutils_ret_t parse_key( break; } } else { - ret = find_parameter(*node_idx, parameter_ns, params_st, parameter_idx); + if (*overwrite_previous_key == false) { + // Handle cases where the code tries to overwrite the yaml parameter name entry + ret = find_parameter(*node_idx, value, params_st, parameter_idx); + *overwrite_previous_key = true; + } else { + ret = find_parameter(*node_idx, parameter_ns, params_st, parameter_idx); + } if (ret != RCUTILS_RET_OK) { break; } - const size_t params_ns_len = strlen(parameter_ns); const size_t param_name_len = strlen(value); const size_t tot_len = (params_ns_len + param_name_len + 2U); @@ -746,16 +828,87 @@ rcutils_ret_t parse_key( return ret; } +rcutils_ret_t initialize_emitter_string( + yaml_emitter_t * emitter) +{ + rcutils_ret_t ret = RCUTILS_RET_OK; + + // Set initial events + yaml_event_t event; + yaml_stream_start_event_initialize(&event, YAML_UTF8_ENCODING); + if (!yaml_emitter_emit(emitter, &event)) { + ret = RCUTILS_RET_ERROR; + } + + yaml_document_start_event_initialize(&event, NULL, NULL, NULL, 0); + if (!yaml_emitter_emit(emitter, &event)) { + ret = RCUTILS_RET_ERROR; + } + + return ret; +} + +rcutils_ret_t end_emitter_string( + yaml_emitter_t * emitter) +{ + rcutils_ret_t ret = RCUTILS_RET_OK; + + // Set initial events + yaml_event_t event; + yaml_document_end_event_initialize(&event, 0); + if (!yaml_emitter_emit(emitter, &event)) { + ret = RCUTILS_RET_ERROR; + } + + yaml_stream_end_event_initialize(&event); + if (!yaml_emitter_emit(emitter, &event)) { + ret = RCUTILS_RET_ERROR; + } + + return ret; +} + +rcutils_ret_t write_structured_parameter_to_string( + yaml_parser_t * parser, + char * yaml_string_buffer, + size_t * written_size, + const size_t node_index, + const size_t parameter_index, + rcl_params_t * params_st) +{ + rcutils_ret_t ret = RCUTILS_RET_OK; + rcutils_allocator_t allocator = params_st->allocator; + + // rahul-k-a: TODO combine this with the parameter allocation part of `parse_value` + // and put everything in a seperate function + char * copied_yaml = rcutils_strndup(yaml_string_buffer, *written_size, allocator); + RCUTILS_CHECK_ARGUMENT_FOR_NULL(copied_yaml, RCUTILS_RET_BAD_ALLOC); + + rcl_variant_t * param_value = &(params_st->params[node_index].parameter_values[parameter_index]); + + if (param_value->yaml_value != NULL) { + // Overwriting, deallocate original + allocator.deallocate(param_value->yaml_value, allocator.state); + } + param_value->yaml_value = copied_yaml; + return ret; +} + + /// /// Get events from parsing a parameter YAML file and process them /// rcutils_ret_t parse_file_events( yaml_parser_t * parser, + yaml_emitter_t * emitter, + char * emitter_string_buffer, + size_t * emitter_written_bytes, namespace_tracker_t * ns_tracker, rcl_params_t * params_st) { int32_t done_parsing = 0; bool is_key = true; + bool is_key_value_pair_found = true; bool is_seq = false; uint32_t line_num = 0; data_types_t seq_data_type = DATA_TYPE_UNKNOWN; @@ -763,6 +916,13 @@ rcutils_ret_t parse_file_events( uint32_t map_depth = 0U; bool is_new_map = false; + uint32_t structure_detect_depth = 0; + *emitter_written_bytes = 0; + bool is_writing_structured_yaml = false; + size_t structured_yaml_param_idx = 0; + rcutils_ret_t ret = RCUTILS_RET_OK; + bool overwrite_previous_key = true; + RCUTILS_CHECK_ARGUMENT_FOR_NULL(parser, RCUTILS_RET_INVALID_ARGUMENT); RCUTILS_CHECK_ARGUMENT_FOR_NULL(params_st, RCUTILS_RET_INVALID_ARGUMENT); rcutils_allocator_t allocator = params_st->allocator; @@ -772,7 +932,6 @@ rcutils_ret_t parse_file_events( yaml_event_t event; size_t node_idx = 0; size_t parameter_idx = 0; - rcutils_ret_t ret = RCUTILS_RET_OK; while (0 == done_parsing) { if (RCUTILS_RET_OK != ret) { break; @@ -784,7 +943,16 @@ rcutils_ret_t parse_file_events( ret = RCUTILS_RET_ERROR; break; } + line_num = ((uint32_t)(event.start_mark.line) + 1U); + + if (is_writing_structured_yaml) { + if (RCUTILS_RET_ERROR == write_event_to_emitter(emitter, &event)) { + ret = RCUTILS_RET_ERROR; + break; + } + } + switch (event.type) { case YAML_STREAM_END_EVENT: done_parsing = 1; @@ -794,8 +962,23 @@ rcutils_ret_t parse_file_events( { /// Need to toggle between key and value at params level if (is_key) { + // If we're at the parameter level, set this flag to denote a + // value for the key has not been found yet + if (map_level == MAP_PARAMS_LVL) { + is_key_value_pair_found = false; + } + // Since the yaml parameter key is also considered as a + // namespace to all indented children, we must + // make sure that the parameter index of the yaml parameter is not overwritten + // By default, if a namespace is detected, then the parameter entry in the param + // table is replaced by its immediate chile + // This is done for optimization (?) - Rahul-K-A + if (is_writing_structured_yaml && (map_depth == structure_detect_depth) ) { + overwrite_previous_key = false; + } ret = parse_key( - event, &map_level, &is_new_map, &node_idx, ¶meter_idx, ns_tracker, params_st); + event, &map_level, &is_new_map, &overwrite_previous_key, &node_idx, ¶meter_idx, + ns_tracker, params_st); if (RCUTILS_RET_OK != ret) { break; } @@ -821,6 +1004,8 @@ rcutils_ret_t parse_file_events( return RCUTILS_RET_ERROR; } ret = parse_value(event, is_seq, node_idx, parameter_idx, &seq_data_type, params_st); + is_key_value_pair_found = true; + if (RCUTILS_RET_OK != ret) { break; } @@ -860,6 +1045,21 @@ rcutils_ret_t parse_file_events( { is_new_map = false; } + // Parsing nested (structured) YAML parameters + // If we're at the param level inside the YAML + // If a value has not been found for the previous key, and we get a new mapping event, + // In theory, this means we have a nested yaml struct + if (is_key_value_pair_found == false && is_writing_structured_yaml == false) { + is_writing_structured_yaml = true; + structured_yaml_param_idx = parameter_idx; + structure_detect_depth = map_depth; + initialize_emitter_string(emitter); + if (RCUTILS_RET_ERROR == write_event_to_emitter(emitter, &event)) { + ret = RCUTILS_RET_ERROR; + RCUTILS_SET_ERROR_MSG_WITH_FORMAT_STRING( + "Error adding line %d to structured yaml parameter", line_num); + } + } break; case YAML_MAPPING_END_EVENT: if (MAP_PARAMS_LVL == map_level) { @@ -888,6 +1088,20 @@ rcutils_ret_t parse_file_events( } } map_depth--; + // Terminate structured yaml parameter if needed + if (is_writing_structured_yaml) { + if (map_depth < structure_detect_depth) { + end_emitter_string(emitter); + write_structured_parameter_to_string(parser, emitter_string_buffer, + emitter_written_bytes, node_idx, structured_yaml_param_idx, params_st); + is_writing_structured_yaml = false; + is_key_value_pair_found = true; + structure_detect_depth = 0; + structured_yaml_param_idx = 0; + // Reset byte counter so that we can reuse buffer + *emitter_written_bytes = 0; + } + } break; case YAML_ALIAS_EVENT: RCUTILS_SET_ERROR_MSG_WITH_FORMAT_STRING( diff --git a/rcl_yaml_param_parser/src/parser.c b/rcl_yaml_param_parser/src/parser.c index 988e68df1..014a4e95a 100644 --- a/rcl_yaml_param_parser/src/parser.c +++ b/rcl_yaml_param_parser/src/parser.c @@ -265,6 +265,21 @@ bool rcl_parse_yaml_file( return false; } + + // Emitter for nested parameters + yaml_emitter_t emitter; + success = yaml_emitter_initialize(&emitter); + if (0 == success) { + RCUTILS_SET_ERROR_MSG("Could not initialize the emitter"); + } + + size_t max_string_length = 100000; + size_t written_size = 0; + unsigned char nested_param_string_allocator[100000]; + yaml_emitter_set_output_string(&emitter, nested_param_string_allocator, max_string_length, + &written_size); + + FILE * yaml_file = fopen(file_path, "r"); if (NULL == yaml_file) { yaml_parser_delete(&parser); @@ -276,11 +291,13 @@ bool rcl_parse_yaml_file( namespace_tracker_t ns_tracker; memset(&ns_tracker, 0, sizeof(namespace_tracker_t)); - rcutils_ret_t ret = parse_file_events(&parser, &ns_tracker, params_st); + rcutils_ret_t ret = parse_file_events(&parser, &emitter, nested_param_string_allocator, + &written_size, &ns_tracker, params_st); fclose(yaml_file); yaml_parser_delete(&parser); + yaml_emitter_delete(&emitter); rcutils_allocator_t allocator = params_st->allocator; if (NULL != ns_tracker.node_ns) { diff --git a/rcl_yaml_param_parser/src/yaml_variant.c b/rcl_yaml_param_parser/src/yaml_variant.c index d1db3cab6..9be0f3cc3 100644 --- a/rcl_yaml_param_parser/src/yaml_variant.c +++ b/rcl_yaml_param_parser/src/yaml_variant.c @@ -80,6 +80,9 @@ void rcl_yaml_variant_fini( } else if (NULL != param_var->string_value) { allocator.deallocate(param_var->string_value, allocator.state); param_var->string_value = NULL; + } else if (NULL != param_var->yaml_value) { + allocator.deallocate(param_var->yaml_value, allocator.state); + param_var->yaml_value = NULL; } else if (NULL != param_var->bool_array_value) { if (NULL != param_var->bool_array_value->values) { allocator.deallocate(param_var->bool_array_value->values, allocator.state); @@ -132,6 +135,13 @@ bool rcl_yaml_variant_copy( RCUTILS_SAFE_FWRITE_TO_STDERR("Error allocating variant mem when copying string_value\n"); return false; } + } else if (NULL != param_var->yaml_value) { + out_param_var->yaml_value = + rcutils_strdup(param_var->yaml_value, allocator); + if (NULL == out_param_var->yaml_value) { + RCUTILS_SAFE_FWRITE_TO_STDERR("Error allocating variant mem when copying yaml_value\n"); + return false; + } } else if (NULL != param_var->bool_array_value) { RCL_YAML_VARIANT_COPY_ARRAY_VALUE( out_param_var->bool_array_value, param_var->bool_array_value, allocator, diff --git a/rcl_yaml_param_parser/test/test_parse.cpp b/rcl_yaml_param_parser/test/test_parse.cpp index f7e004f20..436a17e63 100644 --- a/rcl_yaml_param_parser/test/test_parse.cpp +++ b/rcl_yaml_param_parser/test/test_parse.cpp @@ -370,6 +370,7 @@ TEST(TestParse, parse_key_bad_args) rcutils_allocator_t allocator = rcutils_get_default_allocator(); uint32_t map_level = MAP_NODE_NAME_LVL; bool is_new_map = false; + bool overwrite_yaml_param = true; size_t node_idx = 0; size_t parameter_idx = 0; namespace_tracker_t ns_tracker; @@ -387,7 +388,8 @@ TEST(TestParse, parse_key_bad_args) // map_level is nullptr EXPECT_EQ( RCUTILS_RET_INVALID_ARGUMENT, - parse_key(event, nullptr, &is_new_map, &node_idx, ¶meter_idx, &ns_tracker, params_st)) << + parse_key(event, nullptr, &is_new_map, &overwrite_yaml_param, &node_idx, ¶meter_idx, + &ns_tracker, params_st)) << rcutils_get_error_string().str; EXPECT_TRUE(rcutils_error_is_set()); rcutils_reset_error(); @@ -395,7 +397,8 @@ TEST(TestParse, parse_key_bad_args) // params_st is nullptr EXPECT_EQ( RCUTILS_RET_INVALID_ARGUMENT, - parse_key(event, &map_level, &is_new_map, &node_idx, ¶meter_idx, &ns_tracker, nullptr)) << + parse_key(event, &map_level, &is_new_map, &overwrite_yaml_param, &node_idx, ¶meter_idx, + &ns_tracker, nullptr)) << rcutils_get_error_string().str; EXPECT_TRUE(rcutils_error_is_set()); rcutils_reset_error(); @@ -406,7 +409,8 @@ TEST(TestParse, parse_key_bad_args) EXPECT_EQ( RCUTILS_RET_INVALID_ARGUMENT, parse_key( - event, &map_level, &is_new_map, &node_idx, ¶meter_idx, &ns_tracker, params_st)) << + event, &map_level, &is_new_map, &overwrite_yaml_param, &node_idx, ¶meter_idx, &ns_tracker, + params_st)) << rcutils_get_error_string().str; EXPECT_TRUE(rcutils_error_is_set()); rcutils_reset_error(); @@ -420,7 +424,8 @@ TEST(TestParse, parse_key_bad_args) EXPECT_EQ( RCUTILS_RET_ERROR, parse_key( - event, &map_level, &is_new_map, &node_idx, ¶meter_idx, &ns_tracker, params_st)) << + event, &map_level, &is_new_map, &overwrite_yaml_param, &node_idx, ¶meter_idx, &ns_tracker, + params_st)) << rcutils_get_error_string().str; EXPECT_TRUE(rcutils_error_is_set()); rcutils_reset_error(); @@ -431,7 +436,8 @@ TEST(TestParse, parse_key_bad_args) EXPECT_EQ( RCUTILS_RET_ERROR, parse_key( - event, &map_level, &is_new_map, &node_idx, ¶meter_idx, &ns_tracker, params_st)) << + event, &map_level, &is_new_map, &overwrite_yaml_param, &node_idx, ¶meter_idx, &ns_tracker, + params_st)) << rcutils_get_error_string().str; EXPECT_TRUE(rcutils_error_is_set()); rcutils_reset_error(); @@ -441,7 +447,8 @@ TEST(TestParse, parse_key_bad_args) EXPECT_EQ( RCUTILS_RET_ERROR, parse_key( - event, &map_level, &is_new_map, &node_idx, ¶meter_idx, &ns_tracker, params_st)) << + event, &map_level, &is_new_map, &overwrite_yaml_param, &node_idx, ¶meter_idx, &ns_tracker, + params_st)) << rcutils_get_error_string().str; EXPECT_TRUE(rcutils_error_is_set()); rcutils_reset_error(); @@ -453,7 +460,8 @@ TEST(TestParse, parse_key_bad_args) EXPECT_EQ( RCUTILS_RET_ERROR, parse_key( - event, &map_level, &is_new_map, &node_idx, ¶meter_idx, &ns_tracker, params_st)) << + event, &map_level, &is_new_map, &overwrite_yaml_param, &node_idx, ¶meter_idx, &ns_tracker, + params_st)) << rcutils_get_error_string().str; EXPECT_TRUE(rcutils_error_is_set()); rcutils_reset_error(); @@ -488,6 +496,20 @@ TEST(TestParse, parse_file_events_mock_yaml_parser_parse) { yaml_parser_delete(&parser); }); + size_t max_string_length = 100000; + + yaml_emitter_t emitter; + ASSERT_NE(0, yaml_emitter_initialize(&emitter)); + OSRF_TESTING_TOOLS_CPP_SCOPE_EXIT( + { + yaml_emitter_delete(&emitter); + }); + + + size_t written_size = 0; + char nested_param_string_allocator[100000]; + yaml_emitter_set_output_string(&emitter, (unsigned char *)nested_param_string_allocator, + max_string_length, &written_size); FILE * yaml_file = fopen(path, "r"); ASSERT_NE(nullptr, yaml_file); OSRF_TESTING_TOOLS_CPP_SCOPE_EXIT( @@ -505,7 +527,9 @@ TEST(TestParse, parse_file_events_mock_yaml_parser_parse) { event->type = YAML_NO_EVENT; return 1; }); - EXPECT_EQ(RCUTILS_RET_ERROR, parse_file_events(&parser, &ns_tracker, params_hdl)); + EXPECT_EQ(RCUTILS_RET_ERROR, + parse_file_events(&parser, &emitter, nested_param_string_allocator, &written_size, &ns_tracker, + params_hdl)); } TEST(TestParse, parse_value_events_mock_yaml_parser_parse) {