diff --git a/cmake/modules/yaml.cmake b/cmake/modules/yaml.cmake index 6e9b24a6edc..6be9d52c93a 100644 --- a/cmake/modules/yaml.cmake +++ b/cmake/modules/yaml.cmake @@ -491,7 +491,11 @@ function(yaml_save) zephyr_get_scoped(genex ${ARG_YAML_NAME} GENEX) zephyr_get_scoped(json_content ${ARG_YAML_NAME} JSON) - to_yaml("${json_content}" 0 yaml_out ${genex}) + if(genex) + to_yaml("${json_content}" 0 yaml_out DIRECT_GENEX) + else() + to_yaml("${json_content}" 0 yaml_out DIRECT) + endif() if(EXISTS ${yaml_file}) FILE(RENAME ${yaml_file} ${yaml_file}.bak) @@ -529,20 +533,42 @@ function(yaml_save) cmake_path(SET yaml_path "${yaml_file}") cmake_path(GET yaml_path STEM yaml_file_no_ext) - set(expanded_file ${yaml_file_no_ext}_${genex_save_count}.json) + set(expanded_file ${yaml_file_no_ext}_${genex_save_count}.yaml) set_property(TARGET ${save_target} PROPERTY expanded_file ${expanded_file}) # comment this to keep the temporary files set_property(TARGET ${save_target} APPEND PROPERTY temp_files ${expanded_file}) - FILE(GENERATE OUTPUT ${expanded_file} - CONTENT "${json_content}" - ) + to_yaml("${json_content}" 0 yaml_out TEMP_GENEX) + FILE(GENERATE OUTPUT ${expanded_file} CONTENT "${yaml_out}") endif() endfunction() -function(to_yaml in_json level yaml genex) +function(to_yaml in_json level yaml mode) zephyr_string(ESCAPE json "${in_json}") + + if(mode STREQUAL "DIRECT") + # Direct output mode, no genexes: write a standard YAML + set(expand_lists TRUE) + set(escape_quotes TRUE) + set(comment_genexes FALSE) + elseif(mode STREQUAL "DIRECT_GENEX" OR mode STREQUAL "FINAL_GENEX") + # Direct output mode with genexes enabled, or final write of post-processed + # file: write a standard YAML, comment entries with genexes if they are + # (still) present in the file + set(expand_lists TRUE) + set(escape_quotes TRUE) + set(comment_genexes TRUE) + elseif(mode STREQUAL "TEMP_GENEX") + # Temporary output mode for genex expansion: save single quotes with no + # special processing, since they will be fixed up by yaml-filter.cmake + set(expand_lists FALSE) + set(escape_quotes FALSE) + set(comment_genexes FALSE) + else() + message(FATAL_ERROR "to_yaml(... ${mode} ) is malformed.") + endif() + if(level GREATER 0) math(EXPR level_dec "${level} - 1") set(indent_${level} "${indent_${level_dec}} ") @@ -564,7 +590,7 @@ function(to_yaml in_json level yaml genex) # JSON object -> YAML dictionary set(${yaml} "${${yaml}}${indent_${level}}${member}:\n") math(EXPR sublevel "${level} + 1") - to_yaml("${subjson}" ${sublevel} ${yaml} ${genex}) + to_yaml("${subjson}" ${sublevel} ${yaml} ${mode}) elseif(type STREQUAL ARRAY) # JSON array -> YAML list set(${yaml} "${${yaml}}${indent_${level}}${member}:") @@ -581,13 +607,15 @@ function(to_yaml in_json level yaml genex) string(JSON length ERROR_VARIABLE ignore LENGTH "${item}") if(length) set(non_indent_yaml) - to_yaml("${item}" 0 non_indent_yaml FALSE) + to_yaml("${item}" 0 non_indent_yaml ${mode}) string(REGEX REPLACE "\n$" "" non_indent_yaml "${non_indent_yaml}") string(REPLACE "\n" "\n${indent_${level}} " indent_yaml "${non_indent_yaml}") set(${yaml} "${${yaml}}${indent_${level}} - ${indent_yaml}\n") else() - # Assume a string, escape single quotes. - string(REPLACE "'" "''" item "${item}") + # Assume a string, escape single quotes when required (see comment below). + if(escape_quotes) + string(REPLACE "'" "''" item "${item}") + endif() set(${yaml} "${${yaml}}${indent_${level}} - '${item}'\n") endif() endforeach() @@ -597,13 +625,17 @@ function(to_yaml in_json level yaml genex) # - with unexpanded generator expressions: save as YAML comment # - if it matches the special prefix: convert to YAML list # - otherwise: save as YAML scalar - # Single quotes must be escaped in the value. - string(REPLACE "'" "''" subjson "${subjson}") - if(subjson MATCHES "\\$<.*>" AND ${genex}) + # Single quotes must be escaped in the value _unless_ this will be used + # to expand generator expressions, because then the escaping will be + # addressed once in the yaml-filter.cmake script. + if(escape_quotes) + string(REPLACE "'" "''" subjson "${subjson}") + endif() + if(subjson MATCHES "\\$<.*>" AND comment_genexes) # Yet unexpanded generator expression: save as comment string(SUBSTRING ${indent_${level}} 1 -1 short_indent) set(${yaml} "${${yaml}}#${short_indent}${member}: '${subjson}'\n") - elseif(subjson MATCHES "^@YAML-LIST@") + elseif(subjson MATCHES "^@YAML-LIST@" AND expand_lists) # List-as-string: convert to list set(${yaml} "${${yaml}}${indent_${level}}${member}:") list(POP_FRONT subjson) diff --git a/cmake/yaml-filter.cmake b/cmake/yaml-filter.cmake index 98e914cc448..9873514d0c0 100644 --- a/cmake/yaml-filter.cmake +++ b/cmake/yaml-filter.cmake @@ -1,21 +1,28 @@ # Copyright (c) 2024 Arduino SA # SPDX-License-Identifier: Apache-2.0 -# Simple second stage filter for YAML generation, used when generator -# expressions have been used for some of the data and the conversion to -# YAML needs to happen after cmake has completed processing. +# Second stage filter for YAML generation, called when generator expressions +# have been used in some of the data and cleanup needs to happen after CMake +# has completed processing. # -# This scripts expects as input: -# - EXPANDED_FILE: the name of the input file, in JSON format, that contains -# the expanded generator expressions. +# Two issues are addressed here: +# +# - the intermediate YAML may have non-escaped single quotes in its strings. +# These may have been introduced directly via yaml_set() in the main CMake +# script or by some generator expressions; at this stage they are however +# finalized and can be escaped in one single pass. +# +# - in the input YAML, lists have been stored as a CMake-format string to +# allow generator expressions to seamlessly expand into multiple items. +# These now need to be converted back into a proper YAML list. +# +# This scripts expects as input the following variables: +# +# - EXPANDED_FILE: the name of the file that contains the expanded generator +# expressions. # - OUTPUT_FILE: the name of the final output YAML file. # - TEMP_FILES: a list of temporary files that need to be removed after # the conversion is done. -# -# This script loads the Zephyr yaml module and reuses its `to_yaml()` -# function to convert the fully expanded JSON content to YAML, taking -# into account the special format that was used to store lists. -# Temporary files are then removed. cmake_minimum_required(VERSION 3.20.0) @@ -23,13 +30,40 @@ set(ZEPHYR_BASE ${CMAKE_CURRENT_LIST_DIR}/../) list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/modules") include(yaml) -file(READ ${EXPANDED_FILE} json_content) -to_yaml("${json_content}" 0 yaml_out TRUE) +# Fix all quotes in the input YAML file. Strings in it will be in one of the +# following formats: +# +# name: 'string with 'single quotes' in it' +# - 'string with 'single quotes' in it' +# +# To address this, every single quote is duplicated, then the first and last +# occurrences of two single quotes are removed (they are the string beginning +# and end markers). The result is written to a temporary file. +file(STRINGS ${EXPANDED_FILE} yaml_content) +foreach(line ${yaml_content}) + string(REPLACE "'" "''" line "${line}") # escape every single quote in the string + string(REGEX REPLACE "^([^']* )''" "\\1'" line "${line}") # fix opening quote + string(REGEX REPLACE "''$" "'" line "${line}") # fix closing quote + # semicolons in the string are not to be confused with string separators + string(REPLACE ";" "\\;" line "${line}") + list(APPEND tmp_content "${line}") +endforeach() +list(JOIN tmp_content "\n" tmp_content) +file(WRITE ${EXPANDED_FILE}.tmp "${tmp_content}") + +# Load the now-fixed YAML file and convert the CMake-format lists back into +# proper YAML format using the to_yaml() function. The result is written to +# the final destination file. +yaml_load(FILE ${EXPANDED_FILE}.tmp NAME yaml_saved) +zephyr_get_scoped(json_content yaml_saved JSON) +to_yaml("${json_content}" 0 yaml_out FINAL_GENEX) file(WRITE ${OUTPUT_FILE} "${yaml_out}") # Remove unused temporary files. EXPANDED_FILE needs to be kept, or the -# build system will complain there is no rule to rebuild it +# build system will complain there is no rule to rebuild it, but the +# .tmp file that was just created can be removed. list(REMOVE_ITEM TEMP_FILES ${EXPANDED_FILE}) +list(APPEND TEMP_FILES ${EXPANDED_FILE}.tmp) foreach(file ${TEMP_FILES}) file(REMOVE ${file}) endforeach()