diff --git a/cmake/modules/snippets.cmake b/cmake/modules/snippets.cmake index 3d01c362fa3..c3929c86b43 100644 --- a/cmake/modules/snippets.cmake +++ b/cmake/modules/snippets.cmake @@ -4,11 +4,33 @@ # Snippets support # +# This module: +# +# - searches for snippets in zephyr and any modules +# - validates the SNIPPET input variable, if any +# +# If SNIPPET contains a snippet name that is not found, an error +# will be raised, and the list of valid snippets will be printed. +# # Outcome: # The following variables will be defined when this module completes: # # - SNIPPET_AS_LIST: CMake list of snippet names, created from the # SNIPPET variable +# - SNIPPET_ROOT: CMake list of snippet roots, deduplicated and with +# ZEPHYR_BASE appended at the end +# +# The following variables may be updated when this module completes: +# - DTC_OVERLAY_FILE +# - OVERLAY_CONFIG +# +# The following targets will be defined when this CMake module completes: +# - snippets: when invoked, a list of valid snippets will be printed +# +# Optional variables: +# - SNIPPET_ROOT: input CMake list of snippet roots (directories containing +# additional snippet implementations); this should not include ZEPHYR_BASE, +# as that will be added by this module include_guard(GLOBAL) @@ -21,15 +43,83 @@ zephyr_check_cache(SNIPPET WATCH) # parent scope. We'll set our outcome variables in the parent scope of # the function to ensure the outcome of the module. function(zephyr_process_snippets) - if (SNIPPET) - message(STATUS "Snippet(s): ${SNIPPET}") - else() + set(snippets_py ${ZEPHYR_BASE}/scripts/snippets.py) + set(snippets_generated ${CMAKE_BINARY_DIR}/zephyr/snippets_generated.cmake) + + # Set SNIPPET_AS_LIST, removing snippets_generated.cmake if we are + # running cmake again and snippets are no longer requested. + if (NOT DEFINED SNIPPET) set(SNIPPET_AS_LIST "" PARENT_SCOPE) - return() + file(REMOVE ${snippets_generated}) + else() + string(REPLACE " " ";" SNIPPET_AS_LIST "${SNIPPET}") + set(SNIPPET_AS_LIST "${SNIPPET_AS_LIST}" PARENT_SCOPE) endif() - string(REPLACE " " ";" SNIPPET_AS_LIST "${SNIPPET}") - set(SNIPPET_AS_LIST "${SNIPPET_AS_LIST}" PARENT_SCOPE) + # Set SNIPPET_ROOT. + list(APPEND SNIPPET_ROOT ${APPLICATION_SOURCE_DIR}) + list(APPEND SNIPPET_ROOT ${ZEPHYR_BASE}) + unset(real_snippet_root) + foreach(snippet_dir ${SNIPPET_ROOT}) + # The user might have put a symbolic link in here, for example. + file(REAL_PATH ${snippet_dir} real_snippet_dir) + list(APPEND real_snippet_root ${real_snippet_dir}) + endforeach() + set(SNIPPET_ROOT ${real_snippet_root}) + list(REMOVE_DUPLICATES SNIPPET_ROOT) + set(SNIPPET_ROOT "${SNIPPET_ROOT}" PARENT_SCOPE) + + # Generate and include snippets_generated.cmake. + # The Python script is responsible for checking for unknown + # snippets. + set(snippet_root_args) + foreach(root IN LISTS SNIPPET_ROOT) + list(APPEND snippet_root_args --snippet-root "${root}") + endforeach() + set(requested_snippet_args) + foreach(snippet_name ${SNIPPET_AS_LIST}) + list(APPEND requested_snippet_args --snippet "${snippet_name}") + endforeach() + execute_process(COMMAND ${PYTHON_EXECUTABLE} + ${snippets_py} + ${snippet_root_args} + ${requested_snippet_args} + --cmake-out ${snippets_generated} + OUTPUT_VARIABLE output + ERROR_VARIABLE output + RESULT_VARIABLE ret) + if(${ret}) + message(FATAL_ERROR "${output}") + endif() + include(${snippets_generated}) + + # Propagate include()d build system settings to the caller. + set(DTC_OVERLAY_FILE ${DTC_OVERLAY_FILE} PARENT_SCOPE) + set(OVERLAY_CONFIG ${OVERLAY_CONFIG} PARENT_SCOPE) + + # Create the 'snippets' target. Each snippet is printed in a + # separate command because build system files are not fond of + # newlines. + list(TRANSFORM SNIPPET_NAMES PREPEND "COMMAND;${CMAKE_COMMAND};-E;echo;" + OUTPUT_VARIABLE snippets_target_cmd) + add_custom_target(snippets ${snippets_target_cmd} USES_TERMINAL) + + # If snippets were requested, print messages for each one. + if(SNIPPET_AS_LIST) + # Print the requested snippets. + set(snippet_names "Snippet(s):") + foreach(snippet IN LISTS SNIPPET_AS_LIST) + string(APPEND snippet_names " ${snippet}") + endforeach() + message(STATUS "${snippet_names}") + endif() + + # Re-run cmake if any files we depend on changed. + set_property(DIRECTORY APPEND PROPERTY + CMAKE_CONFIGURE_DEPENDS + ${snippets_py} + ${SNIPPET_PATHS} # generated variable + ) endfunction() zephyr_process_snippets() diff --git a/scripts/schemas/snippet-schema.yml b/scripts/schemas/snippet-schema.yml new file mode 100644 index 00000000000..d6c46d54323 --- /dev/null +++ b/scripts/schemas/snippet-schema.yml @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (c) 2022, Nordic Semiconductor ASA + +# A pykwalify schema for basic validation of the snippet.yml format. + +schema;append-schema: + # Sub-schema for appending onto CMake list variables. + # See uses under 'append:' keys below. + type: map + mapping: + DTC_OVERLAY_FILE: + type: str + OVERLAY_CONFIG: + type: str + +type: map +mapping: + name: + required: true + type: str + append: + example: | + Snippet-wide appending can be done here: + + name: foo + append: + DTC_OVERLAY_FILE: m3.overlay + include: append-schema + boards: + example: | + Board-specific appending can be done here: + + name: foo + boards: + qemu_cortex_m3: + append: + DTC_OVERLAY_FILE: m3.overlay + type: map + mapping: + regex;(.*): + type: map + mapping: + append: + include: append-schema diff --git a/scripts/snippets.py b/scripts/snippets.py new file mode 100644 index 00000000000..7b838573a2c --- /dev/null +++ b/scripts/snippets.py @@ -0,0 +1,316 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2022, Nordic Semiconductor ASA +# +# SPDX-License-Identifier: Apache-2.0 + +'''Internal snippets tool. +This is part of the build system's support for snippets. +It is not meant for use outside of the build system. + +Output CMake variables: + +- SNIPPET_NAMES: CMake list of discovered snippet names +- SNIPPET_FOUND_{snippet}: one per discovered snippet +''' + +from collections import defaultdict, UserDict +from dataclasses import dataclass, field +from pathlib import Path +from typing import Dict, Iterable, List, Set +import argparse +import logging +import os +import pykwalify.core +import pykwalify.errors +import re +import sys +import textwrap +import yaml + +# Marker type for an 'append:' configuration. Maps variables +# to the list of values to append to them. +Appends = Dict[str, List[str]] + +def _new_append(): + return defaultdict(list) + +def _new_board2appends(): + return defaultdict(_new_append) + +@dataclass +class Snippet: + '''Class for keeping track of all the settings discovered for an + individual snippet.''' + + name: str + appends: Appends = field(default_factory=_new_append) + board2appends: Dict[str, Appends] = field(default_factory=_new_board2appends) + + def process_data(self, pathobj: Path, snippet_data: dict): + '''Process the data in a snippet.yml file, after it is loaded into a + python object and validated by pykwalify.''' + def append_value(variable, value): + if variable in ('DTC_OVERLAY_FILE', 'OVERLAY_CONFIG'): + path = pathobj.parent / value + if not path.is_file(): + _err(f'snippet file {pathobj}: {variable}: file not found: {path}') + return f'"{path}"' + _err(f'unknown append variable: {variable}') + + for variable, value in snippet_data.get('append', {}).items(): + self.appends[variable].append(append_value(variable, value)) + for board, settings in snippet_data.get('boards', {}).items(): + if board.startswith('/') and not board.endswith('/'): + _err(f"snippet file {pathobj}: board {board} starts with '/', so " + "it must end with '/' to use a regular expression") + for variable, value in settings.get('append', {}).items(): + self.board2appends[board][variable].append( + append_value(variable, value)) + +class Snippets(UserDict): + '''Type for all the information we have discovered about all snippets. + As a dict, this maps a snippet's name onto the Snippet object. + Any additional global attributes about all snippets go here as + instance attributes.''' + + def __init__(self, requested: Iterable[str] = None): + super().__init__() + self.paths: Set[Path] = set() + self.requested: Set[str] = set(requested or []) + +class SnippetsError(Exception): + '''Class for signalling expected errors''' + + def __init__(self, msg): + self.msg = msg + +class SnippetToCMakePrinter: + '''Helper class for printing a Snippets's semantics to a .cmake + include file for use by snippets.cmake.''' + + def __init__(self, snippets: Snippets, out_file): + self.snippets = snippets + self.out_file = out_file + self.section = '#' * 79 + + def print_cmake(self): + '''Print to the output file provided to the constructor.''' + # TODO: add source file info + snippets = self.snippets + snippet_names = sorted(snippets.keys()) + snippet_path_list = " ".join( + sorted(f'"{path}"' for path in snippets.paths)) + + self.print('''\ +# WARNING. THIS FILE IS AUTO-GENERATED. DO NOT MODIFY! +# +# This file contains build system settings derived from your snippets. +# Its contents are an implementation detail that should not be used outside +# of Zephyr's snippets CMake module. +# +# See the Snippets guide in the Zephyr documentation for more information. +''') + + self.print(f'''\ +{self.section} +# Global information about all snippets. + +# The name of every snippet that was discovered. +set(SNIPPET_NAMES {' '.join(f'"{name}"' for name in snippet_names)}) +# The paths to all the snippet.yml files. One snippet +# can have multiple snippet.yml files. +set(SNIPPET_PATHS {snippet_path_list}) +''') + + for snippet_name in snippet_names: + if snippet_name not in snippets.requested: + continue + self.print_cmake_for(snippets[snippet_name]) + self.print() + + def print_cmake_for(self, snippet: Snippet): + self.print(f'''\ +{self.section} +# Snippet '{snippet.name}' + +# Common variable appends.''') + self.print_appends(snippet.appends, 0) + for board, appends in snippet.board2appends.items(): + self.print_appends_for_board(board, appends) + + def print_appends_for_board(self, board: str, appends: Appends): + if board.startswith('/'): + board_re = board[1:-1] + self.print(f'''\ +# Appends for board regular expression '{board_re}' +if("${{BOARD}}" MATCHES "^{board_re}$")''') + else: + self.print(f'''\ +# Appends for board '{board}' +if("${{BOARD}}" STREQUAL "{board}")''') + self.print_appends(appends, 1) + self.print('endif()') + + def print_appends(self, appends: Appends, indent: int): + space = ' ' * indent + for name, values in appends.items(): + for value in values: + self.print(f'{space}list(APPEND {name} {value})') + + def print(self, *args, **kwargs): + kwargs['file'] = self.out_file + print(*args, **kwargs) + +# Name of the file containing the pykwalify schema for snippet.yml +# files. +SCHEMA_PATH = str(Path(__file__).parent / 'schemas' / 'snippet-schema.yml') +with open(SCHEMA_PATH, 'r') as f: + SNIPPET_SCHEMA = yaml.safe_load(f.read()) + +# The name of the file which contains metadata about the snippets +# being defined in a directory. +SNIPPET_YML = 'snippet.yml' + +# Regular expression for validating snippet names. Snippet names must +# begin with an alphanumeric character, and may contain alphanumeric +# characters or underscores. This is intentionally very restrictive to +# keep things consistent and easy to type and remember. We can relax +# this a bit later if needed. +SNIPPET_NAME_RE = re.compile('[A-Za-z0-9][A-Za-z0-9_-]*') + +# Logger for this module. +LOG = logging.getLogger('snippets') + +def _err(msg): + raise SnippetsError(f'error: {msg}') + +def parse_args(): + parser = argparse.ArgumentParser(description='snippets helper', + allow_abbrev=False) + parser.add_argument('--snippet-root', default=[], action='append', type=Path, + help='''a SNIPPET_ROOT element; may be given + multiple times''') + parser.add_argument('--snippet', dest='snippets', default=[], action='append', + help='''a SNIPPET element; may be given + multiple times''') + parser.add_argument('--cmake-out', type=Path, + help='''file to write cmake output to; include() + this file after calling this script''') + return parser.parse_args() + +def setup_logging(): + # Silence validation errors from pykwalify, which are logged at + # logging.ERROR level. We want to handle those ourselves as + # needed. + logging.getLogger('pykwalify').setLevel(logging.CRITICAL) + logging.basicConfig(level=logging.INFO, + format=' %(name)s: %(message)s') + +def process_snippets(args: argparse.Namespace) -> Snippets: + '''Process snippet.yml files under each *snippet_root* + by recursive search. Return a Snippets object describing + the results of the search. + ''' + # This will contain information about all the snippets + # we discover in each snippet_root element. + snippets = Snippets(requested=args.snippets) + + # Process each path in snippet_root in order, adjusting + # snippets as needed for each one. + for root in args.snippet_root: + process_snippets_in(root, snippets) + + return snippets + +def process_snippets_in(root_dir: Path, snippets: Snippets) -> None: + '''Process snippet.yml files in *root_dir*, + updating *snippets* as needed.''' + + if not root_dir.is_dir(): + LOG.warning(f'SNIPPET_ROOT {root_dir} ' + 'is not a directory; ignoring it') + return + + snippets_dir = root_dir / 'snippets' + if not snippets_dir.is_dir(): + return + + for dirpath, _, filenames in os.walk(snippets_dir): + if SNIPPET_YML not in filenames: + continue + + snippet_yml = Path(dirpath) / SNIPPET_YML + snippet_data = load_snippet_yml(snippet_yml) + name = snippet_data['name'] + if name not in snippets: + snippets[name] = Snippet(name=name) + snippets[name].process_data(snippet_yml, snippet_data) + snippets.paths.add(snippet_yml) + +def load_snippet_yml(snippet_yml: Path) -> dict: + '''Load a snippet.yml file *snippet_yml*, validate the contents + against the schema, and do other basic checks. Return the dict + of the resulting YAML data.''' + + with open(snippet_yml, 'r') as f: + try: + snippet_data = yaml.safe_load(f.read()) + except yaml.scanner.ScannerError: + _err(f'snippets file {snippet_yml} is invalid YAML') + + def pykwalify_err(e): + return f'''\ +invalid {SNIPPET_YML} file: {snippet_yml} +{textwrap.indent(e.msg, ' ')} +''' + + try: + pykwalify.core.Core(source_data=snippet_data, + schema_data=SNIPPET_SCHEMA).validate() + except pykwalify.errors.PyKwalifyException as e: + _err(pykwalify_err(e)) + + name = snippet_data['name'] + if not SNIPPET_NAME_RE.fullmatch(name): + _err(f"snippet file {snippet_yml}: invalid snippet name '{name}'; " + 'snippet names must begin with a letter ' + 'or number, and may only contain letters, numbers, ' + 'dashes (-), and underscores (_)') + + return snippet_data + +def check_for_errors(snippets: Snippets) -> None: + unknown_snippets = sorted(snippet for snippet in snippets.requested + if snippet not in snippets) + if unknown_snippets: + all_snippets = '\n '.join(sorted(snippets)) + _err(f'''\ +snippets not found: {', '.join(unknown_snippets)} + Please choose from among the following snippets: + {all_snippets}''') + +def write_cmake_out(snippets: Snippets, cmake_out: Path) -> None: + '''Write a cmake include file to *cmake_out* which + reflects the information in *snippets*. + + The contents of this file should be considered an implementation + detail and are not meant to be used outside of snippets.cmake.''' + if not cmake_out.parent.exists(): + cmake_out.parent.mkdir() + with open(cmake_out, 'w') as f: + SnippetToCMakePrinter(snippets, f).print_cmake() + +def main(): + args = parse_args() + setup_logging() + try: + snippets = process_snippets(args) + check_for_errors(snippets) + except SnippetsError as e: + LOG.critical(e.msg) + sys.exit(1) + write_cmake_out(snippets, args.cmake_out) + +if __name__ == "__main__": + main()