From b8f6536b29203a9b9d5d4910226d3afedd499f74 Mon Sep 17 00:00:00 2001 From: Zach White Date: Thu, 24 Jun 2021 12:42:23 -0700 Subject: [PATCH] Rework `qmk compile` to bypass `Makefile`. Add new --filter option. --- Makefile | 2 +- lib/python/qmk/cli/clean.py | 6 +- lib/python/qmk/cli/compile.py | 138 +++++++-- lib/python/qmk/cli/flash.py | 4 +- lib/python/qmk/commands.py | 51 ++-- lib/python/qmk/info.py | 513 +--------------------------------- lib/python/qmk/keyboard.py | 11 + lib/python/qmk/keymap.py | 46 ++- lib/python/qmk/metadata.py | 495 ++++++++++++++++++++++++++++++++ 9 files changed, 673 insertions(+), 593 deletions(-) create mode 100644 lib/python/qmk/metadata.py diff --git a/Makefile b/Makefile index e007ae3679..c5b1d6c425 100644 --- a/Makefile +++ b/Makefile @@ -393,7 +393,7 @@ define PARSE_KEYMAP # Specify the variables that we are passing forward to submake MAKE_VARS := KEYBOARD=$$(CURRENT_KB) KEYMAP=$$(CURRENT_KM) REQUIRE_PLATFORM_KEY=$$(REQUIRE_PLATFORM_KEY) QMK_BIN=$$(QMK_BIN) # And the first part of the make command - MAKE_CMD := $$(MAKE) -r -R -C $(ROOT_DIR) -f build_keyboard.mk $$(MAKE_TARGET) + MAKE_CMD := echo $$(MAKE) -r -R -C $(ROOT_DIR) -f build_keyboard.mk $$(MAKE_TARGET) # The message to display MAKE_MSG := $$(MSG_MAKE_KB) # We run the command differently, depending on if we want more output or not diff --git a/lib/python/qmk/cli/clean.py b/lib/python/qmk/cli/clean.py index 72b7ffe810..f0b1a897af 100644 --- a/lib/python/qmk/cli/clean.py +++ b/lib/python/qmk/cli/clean.py @@ -2,7 +2,7 @@ """ from subprocess import DEVNULL -from qmk.commands import create_make_target +from qmk.commands import _find_make from milc import cli @@ -11,4 +11,6 @@ from milc import cli def clean(cli): """Runs `make clean` (or `make distclean` if --all is passed) """ - cli.run(create_make_target('distclean' if cli.args.all else 'clean'), capture_output=False, stdin=DEVNULL) + make_cmd = [_find_make(), 'distclean' if cli.args.all else 'clean'] + + cli.run(make_cmd, capture_output=False, stdin=DEVNULL) diff --git a/lib/python/qmk/cli/compile.py b/lib/python/qmk/cli/compile.py index 7a45e77214..3bc1f9430e 100755 --- a/lib/python/qmk/cli/compile.py +++ b/lib/python/qmk/cli/compile.py @@ -5,20 +5,70 @@ You can compile a keymap already in the repo or using a QMK Configurator export. from subprocess import DEVNULL from argcomplete.completers import FilesCompleter +from dotty_dict import dotty from milc import cli import qmk.path -from qmk.decorators import automagic_keyboard, automagic_keymap +from qmk.decorators import automagic_keyboard, automagic_keymap, lru_cache from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json -from qmk.keyboard import keyboard_completer, keyboard_folder -from qmk.keymap import keymap_completer +from qmk.info import info_json +from qmk.keyboard import keyboard_completer, is_keyboard_target, list_keyboards +from qmk.keymap import keymap_completer, list_keymaps +from qmk.metadata import true_values, false_values + + +@lru_cache() +def _keyboard_list(): + """Returns a list of keyboards matching cli.config.compile.keyboard. + """ + if cli.config.compile.keyboard == 'all': + return list_keyboards() + + elif cli.config.compile.keyboard.startswith('all-'): + return list_keyboards() + + return [cli.config.compile.keyboard] + + +def keyboard_keymap_iter(): + """Iterates over the keyboard/keymap for this command and yields a pairing of each. + """ + for keyboard in _keyboard_list(): + continue_flag = False + if cli.args.filter: + info_data = dotty(info_json(keyboard)) + for filter in cli.args.filter: + if '=' in filter: + key, value = filter.split('=', 1) + if value in true_values: + value = True + elif value in false_values: + value = False + elif value.isdigit(): + value = int(value) + elif '.' in value and value.replace('.').isdigit(): + value = float(value) + + if info_data.get(key) != value: + continue_flag = True + break + + if continue_flag: + continue + + if cli.config.compile.keymap == 'all': + for keymap in list_keymaps(keyboard): + yield keyboard, keymap + else: + yield keyboard, cli.config.compile.keymap @cli.argument('filename', nargs='?', arg_only=True, type=qmk.path.FileType('r'), completer=FilesCompleter('.json'), help='The configurator export to compile') -@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.') +@cli.argument('-kb', '--keyboard', type=is_keyboard_target, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.') @cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.') @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.") @cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs to run.") +@cli.argument('-f', '--filter', arg_only=True, action='append', default=[], help="Filter your list against info.json.") @cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.") @cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.") @cli.subcommand('Compile a QMK Firmware.') @@ -31,47 +81,79 @@ def compile(cli): If a keyboard and keymap are provided this command will build a firmware based on that. """ - if cli.args.clean and not cli.args.filename and not cli.args.dry_run: - command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, 'clean') - cli.run(command, capture_output=False, stdin=DEVNULL) + envs = {'REQUIRE_PLATFORM_KEY': ''} + silent = cli.config.compile.keyboard == 'all' or cli.config.compile.keyboard.startswith('all-') or cli.config.compile.keymap == 'all' - # Build the environment vars - envs = {} + # Setup the environment for env in cli.args.env: if '=' in env: key, value = env.split('=', 1) + if key in envs: + cli.log.warning('Overwriting existing environment variable %s=%s with %s=%s!', key, envs[key], key, value) envs[key] = value else: cli.log.warning('Invalid environment variable: %s', env) - # Determine the compile command - command = None + if cli.config.compile.keyboard.startswith('all-'): + envs['REQUIRE_PLATFORM_KEY'] = cli.config.compile.keyboard[4:] + + # Run clean if necessary + if cli.args.clean and not cli.args.filename and not cli.args.dry_run: + for keyboard, keymap in keyboard_keymap_iter(): + cli.log.info('Cleaning previous build files for keyboard {fg_cyan}%s{fg_reset} keymap {fg_cyan}%s', keyboard, keymap) + _, _, make_cmd = create_make_command(keyboard, keymap, 'clean', cli.config.compile.parallel, silent, **envs) + cli.run(make_cmd, capture_output=False, stdin=DEVNULL) + + # Determine the compile command(s) + commands = None if cli.args.filename: # If a configurator JSON was provided generate a keymap and compile it user_keymap = parse_configurator_json(cli.args.filename) - command = compile_configurator_json(user_keymap, parallel=cli.config.compile.parallel, **envs) + commands = [compile_configurator_json(user_keymap, parallel=cli.config.compile.parallel, **envs)] - else: - if cli.config.compile.keyboard and cli.config.compile.keymap: - # Generate the make command for a specific keyboard/keymap. - command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, parallel=cli.config.compile.parallel, **envs) + elif cli.config.compile.keyboard and cli.config.compile.keymap: + if cli.args.filter: + cli.log.info('Generating the list of keyboards to compile, this may take some time.') - elif not cli.config.compile.keyboard: - cli.log.error('Could not determine keyboard!') - elif not cli.config.compile.keymap: - cli.log.error('Could not determine keymap!') + commands = [create_make_command(keyboard, keymap, parallel=cli.config.compile.parallel, silent=silent, **envs) for keyboard, keymap in keyboard_keymap_iter()] + + elif not cli.config.compile.keyboard: + cli.log.error('Could not determine keyboard!') + + elif not cli.config.compile.keymap: + cli.log.error('Could not determine keymap!') # Compile the firmware, if we're able to - if command: - cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command)) - if not cli.args.dry_run: - cli.echo('\n') - # FIXME(skullydazed/anyone): Remove text=False once milc 1.0.11 has had enough time to be installed everywhere. - compile = cli.run(command, capture_output=False, text=False) - return compile.returncode + if commands: + returncodes = [] + for keyboard, keymap, command in commands: + print() + cli.log.info('Compiling QMK Firmware for keyboard {fg_cyan}%s{fg_reset} and keymap {fg_cyan}%s', keyboard, keymap) + cli.log.debug('Running make command: {fg_blue}%s', ' '.join(command)) + + if not cli.args.dry_run: + compile = cli.run(command, capture_output=False) + returncodes.append(compile.returncode) + if compile.returncode == 0: + cli.log.info('Success!') + else: + cli.log.error('Failed!') + + if any(returncodes): + print() + cli.log.error('Could not compile all targets, look above this message for more details. Failing target(s):') + + for i, returncode in enumerate(returncodes): + if returncode != 0: + keyboard, keymap, command = commands[i] + cli.echo('\tkeyboard: {fg_cyan}%s{fg_reset} keymap: {fg_cyan}%s', keyboard, keymap) + + elif cli.args.filter: + cli.log.error('No keyboards found after applying filter(s)!') + return False else: cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.') - cli.echo('usage: qmk compile [-h] [-b] [-kb KEYBOARD] [-km KEYMAP] [filename]') + cli.print_help() return False diff --git a/lib/python/qmk/cli/flash.py b/lib/python/qmk/cli/flash.py index 1b2932a5b2..e11df63630 100644 --- a/lib/python/qmk/cli/flash.py +++ b/lib/python/qmk/cli/flash.py @@ -11,7 +11,7 @@ from milc import cli import qmk.path from qmk.decorators import automagic_keyboard, automagic_keymap from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json -from qmk.keyboard import keyboard_completer, keyboard_folder +from qmk.keyboard import keyboard_completer, is_keyboard_target def print_bootloader_help(): @@ -36,7 +36,7 @@ def print_bootloader_help(): @cli.argument('-b', '--bootloaders', action='store_true', help='List the available bootloaders.') @cli.argument('-bl', '--bootloader', default='flash', help='The flash command, corresponding to qmk\'s make options of bootloaders.') @cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.') -@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.') +@cli.argument('-kb', '--keyboard', type=is_keyboard_target, completer=keyboard_completer, help='The keyboard to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.') @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.") @cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs to run.") @cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.") diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py index 3a35c11031..e6ae0fc5b1 100644 --- a/lib/python/qmk/commands.py +++ b/lib/python/qmk/commands.py @@ -27,34 +27,7 @@ def _find_make(): return make_cmd -def create_make_target(target, parallel=1, **env_vars): - """Create a make command - - Args: - - target - Usually a make rule, such as 'clean' or 'all'. - - parallel - The number of make jobs to run in parallel - - **env_vars - Environment variables to be passed to make. - - Returns: - - A command that can be run to make the specified keyboard and keymap - """ - env = [] - make_cmd = _find_make() - - for key, value in env_vars.items(): - env.append(f'{key}={value}') - - return [make_cmd, '-j', str(parallel), *env, target] - - -def create_make_command(keyboard, keymap, target=None, parallel=1, **env_vars): +def create_make_command(keyboard, keymap, target=None, parallel=1, silent=False, **env_vars): """Create a make compile command Args: @@ -78,12 +51,26 @@ def create_make_command(keyboard, keymap, target=None, parallel=1, **env_vars): A command that can be run to make the specified keyboard and keymap """ - make_args = [keyboard, keymap] + make_cmd = [_find_make(), '--no-print-directory', '-r', '-R', '-C', './', '-f', 'build_keyboard.mk'] + + env_vars['KEYBOARD'] = keyboard + env_vars['KEYMAP'] = keymap + env_vars['QMK_BIN'] = 'bin/qmk' if 'DEPRECATED_BIN_QMK' in os.environ else 'qmk' + env_vars['VERBOSE'] = 'true' if cli.config.general.verbose else '' + env_vars['SILENT'] = 'true' if silent else 'false' + env_vars['COLOR'] = 'true' if cli.config.general.color else '' + + if parallel > 1: + make_cmd.append('-j') + make_cmd.append(parallel) if target: - make_args.append(target) + make_cmd.append(target) - return create_make_target(':'.join(make_args), parallel, **env_vars) + for key, value in env_vars.items(): + make_cmd.append(f'{key}={value}') + + return keyboard, keymap, make_cmd def get_git_version(repo_dir='.', check_dir='.'): @@ -204,7 +191,7 @@ def compile_configurator_json(user_keymap, bootloader=None, parallel=1, **env_va f'QMK_BIN={"bin/qmk" if "DEPRECATED_BIN_QMK" in os.environ else "qmk"}', ]) - return make_command + return user_keymap['keyboard'], user_keymap['keymap'], make_command def parse_configurator_json(configurator_file): diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py index 47c8bff7a8..9e171af605 100644 --- a/lib/python/qmk/info.py +++ b/lib/python/qmk/info.py @@ -1,22 +1,14 @@ """Functions that help us generate and use info.json files. """ -from glob import glob +from functools import lru_cache from pathlib import Path import jsonschema -from dotty_dict import dotty from milc import cli -from qmk.constants import CHIBIOS_PROCESSORS, LUFA_PROCESSORS, VUSB_PROCESSORS -from qmk.c_parse import find_layouts -from qmk.json_schema import deep_update, json_load, keyboard_validate, keyboard_api_validate -from qmk.keyboard import config_h, rules_mk +from qmk.json_schema import keyboard_api_validate from qmk.keymap import list_keymaps -from qmk.makefile import parse_rules_mk_file -from qmk.math import compute - -true_values = ['1', 'on', 'yes'] -false_values = ['0', 'off', 'no'] +from qmk.metadata import basic_info_json, info_log_error def _valid_community_layout(layout): @@ -25,45 +17,16 @@ def _valid_community_layout(layout): return (Path('layouts/default') / layout).exists() +@lru_cache(maxsize=None) def info_json(keyboard): """Generate the info.json data for a specific keyboard. """ - cur_dir = Path('keyboards') - rules = parse_rules_mk_file(cur_dir / keyboard / 'rules.mk') - if 'DEFAULT_FOLDER' in rules: - keyboard = rules['DEFAULT_FOLDER'] - rules = parse_rules_mk_file(cur_dir / keyboard / 'rules.mk', rules) - - info_data = { - 'keyboard_name': str(keyboard), - 'keyboard_folder': str(keyboard), - 'keymaps': {}, - 'layouts': {}, - 'parse_errors': [], - 'parse_warnings': [], - 'maintainer': 'qmk', - } + info_data = basic_info_json(keyboard) # Populate the list of JSON keymaps for keymap in list_keymaps(keyboard, c=False, fullpath=True): info_data['keymaps'][keymap.name] = {'url': f'https://raw.githubusercontent.com/qmk/qmk_firmware/master/{keymap}/keymap.json'} - # Populate layout data - layouts, aliases = _find_all_layouts(info_data, keyboard) - - if aliases: - info_data['layout_aliases'] = aliases - - for layout_name, layout_json in layouts.items(): - if not layout_name.startswith('LAYOUT_kc'): - layout_json['c_macro'] = True - info_data['layouts'][layout_name] = layout_json - - # Merge in the data from info.json, config.h, and rules.mk - info_data = merge_info_jsons(keyboard, info_data) - info_data = _extract_config_h(info_data) - info_data = _extract_rules_mk(info_data) - # Validate against the jsonschema try: keyboard_api_validate(info_data) @@ -75,477 +38,19 @@ def info_json(keyboard): # Make sure we have at least one layout if not info_data.get('layouts'): - _log_error(info_data, 'No LAYOUTs defined! Need at least one layout defined in the keyboard.h or info.json.') + info_log_error(info_data, 'No LAYOUTs defined! Need at least one layout defined in the keyboard.h or info.json.') # Filter out any non-existing community layouts for layout in info_data.get('community_layouts', []): if not _valid_community_layout(layout): # Ignore layout from future checks info_data['community_layouts'].remove(layout) - _log_error(info_data, 'Claims to support a community layout that does not exist: %s' % (layout)) + info_log_error(info_data, 'Claims to support a community layout that does not exist: %s' % (layout)) # Make sure we supply layout macros for the community layouts we claim to support for layout in info_data.get('community_layouts', []): layout_name = 'LAYOUT_' + layout if layout_name not in info_data.get('layouts', {}) and layout_name not in info_data.get('layout_aliases', {}): - _log_error(info_data, 'Claims to support community layout %s but no %s() macro found' % (layout, layout_name)) + info_log_error(info_data, 'Claims to support community layout %s but no %s() macro found' % (layout, layout_name)) - return info_data - - -def _extract_features(info_data, rules): - """Find all the features enabled in rules.mk. - """ - # Special handling for bootmagic which also supports a "lite" mode. - if rules.get('BOOTMAGIC_ENABLE') == 'lite': - rules['BOOTMAGIC_LITE_ENABLE'] = 'on' - del rules['BOOTMAGIC_ENABLE'] - if rules.get('BOOTMAGIC_ENABLE') == 'full': - rules['BOOTMAGIC_ENABLE'] = 'on' - - # Skip non-boolean features we haven't implemented special handling for - for feature in 'HAPTIC_ENABLE', 'QWIIC_ENABLE': - if rules.get(feature): - del rules[feature] - - # Process the rest of the rules as booleans - for key, value in rules.items(): - if key.endswith('_ENABLE'): - key = '_'.join(key.split('_')[:-1]).lower() - value = True if value.lower() in true_values else False if value.lower() in false_values else value - - if 'config_h_features' not in info_data: - info_data['config_h_features'] = {} - - if 'features' not in info_data: - info_data['features'] = {} - - if key in info_data['features']: - _log_warning(info_data, 'Feature %s is specified in both info.json and rules.mk, the rules.mk value wins.' % (key,)) - - info_data['features'][key] = value - info_data['config_h_features'][key] = value - - return info_data - - -def _pin_name(pin): - """Returns the proper representation for a pin. - """ - pin = pin.strip() - - if not pin: - return None - - elif pin.isdigit(): - return int(pin) - - elif pin == 'NO_PIN': - return None - - elif pin[0] in 'ABCDEFGHIJK' and pin[1].isdigit(): - return pin - - raise ValueError(f'Invalid pin: {pin}') - - -def _extract_pins(pins): - """Returns a list of pins from a comma separated string of pins. - """ - return [_pin_name(pin) for pin in pins.split(',')] - - -def _extract_direct_matrix(info_data, direct_pins): - """ - """ - info_data['matrix_pins'] = {} - direct_pin_array = [] - - while direct_pins[-1] != '}': - direct_pins = direct_pins[:-1] - - for row in direct_pins.split('},{'): - if row.startswith('{'): - row = row[1:] - - if row.endswith('}'): - row = row[:-1] - - direct_pin_array.append([]) - - for pin in row.split(','): - if pin == 'NO_PIN': - pin = None - - direct_pin_array[-1].append(pin) - - return direct_pin_array - - -def _extract_matrix_info(info_data, config_c): - """Populate the matrix information. - """ - row_pins = config_c.get('MATRIX_ROW_PINS', '').replace('{', '').replace('}', '').strip() - col_pins = config_c.get('MATRIX_COL_PINS', '').replace('{', '').replace('}', '').strip() - direct_pins = config_c.get('DIRECT_PINS', '').replace(' ', '')[1:-1] - - if 'MATRIX_ROWS' in config_c and 'MATRIX_COLS' in config_c: - if 'matrix_size' in info_data: - _log_warning(info_data, 'Matrix size is specified in both info.json and config.h, the config.h values win.') - - info_data['matrix_size'] = { - 'cols': compute(config_c.get('MATRIX_COLS', '0')), - 'rows': compute(config_c.get('MATRIX_ROWS', '0')), - } - - if row_pins and col_pins: - if 'matrix_pins' in info_data: - _log_warning(info_data, 'Matrix pins are specified in both info.json and config.h, the config.h values win.') - - info_data['matrix_pins'] = { - 'cols': _extract_pins(col_pins), - 'rows': _extract_pins(row_pins), - } - - if direct_pins: - if 'matrix_pins' in info_data: - _log_warning(info_data, 'Direct pins are specified in both info.json and config.h, the config.h values win.') - - info_data['matrix_pins']['direct'] = _extract_direct_matrix(info_data, direct_pins) - - return info_data - - -def _extract_config_h(info_data): - """Pull some keyboard information from existing config.h files - """ - config_c = config_h(info_data['keyboard_folder']) - - # Pull in data from the json map - dotty_info = dotty(info_data) - info_config_map = json_load(Path('data/mappings/info_config.json')) - - for config_key, info_dict in info_config_map.items(): - info_key = info_dict['info_key'] - key_type = info_dict.get('value_type', 'str') - - try: - if config_key in config_c and info_dict.get('to_json', True): - if dotty_info.get(info_key) and info_dict.get('warn_duplicate', True): - _log_warning(info_data, '%s in config.h is overwriting %s in info.json' % (config_key, info_key)) - - if key_type.startswith('array'): - if '.' in key_type: - key_type, array_type = key_type.split('.', 1) - else: - array_type = None - - config_value = config_c[config_key].replace('{', '').replace('}', '').strip() - - if array_type == 'int': - dotty_info[info_key] = list(map(int, config_value.split(','))) - else: - dotty_info[info_key] = config_value.split(',') - - elif key_type == 'bool': - dotty_info[info_key] = config_c[config_key] in true_values - - elif key_type == 'hex': - dotty_info[info_key] = '0x' + config_c[config_key][2:].upper() - - elif key_type == 'list': - dotty_info[info_key] = config_c[config_key].split() - - elif key_type == 'int': - dotty_info[info_key] = int(config_c[config_key]) - - else: - dotty_info[info_key] = config_c[config_key] - - except Exception as e: - _log_warning(info_data, f'{config_key}->{info_key}: {e}') - - info_data.update(dotty_info) - - # Pull data that easily can't be mapped in json - _extract_matrix_info(info_data, config_c) - - return info_data - - -def _extract_rules_mk(info_data): - """Pull some keyboard information from existing rules.mk files - """ - rules = rules_mk(info_data['keyboard_folder']) - info_data['processor'] = rules.get('MCU', info_data.get('processor', 'atmega32u4')) - - if info_data['processor'] in CHIBIOS_PROCESSORS: - arm_processor_rules(info_data, rules) - - elif info_data['processor'] in LUFA_PROCESSORS + VUSB_PROCESSORS: - avr_processor_rules(info_data, rules) - - else: - cli.log.warning("%s: Unknown MCU: %s" % (info_data['keyboard_folder'], info_data['processor'])) - unknown_processor_rules(info_data, rules) - - # Pull in data from the json map - dotty_info = dotty(info_data) - info_rules_map = json_load(Path('data/mappings/info_rules.json')) - - for rules_key, info_dict in info_rules_map.items(): - info_key = info_dict['info_key'] - key_type = info_dict.get('value_type', 'str') - - try: - if rules_key in rules and info_dict.get('to_json', True): - if dotty_info.get(info_key) and info_dict.get('warn_duplicate', True): - _log_warning(info_data, '%s in rules.mk is overwriting %s in info.json' % (rules_key, info_key)) - - if key_type.startswith('array'): - if '.' in key_type: - key_type, array_type = key_type.split('.', 1) - else: - array_type = None - - rules_value = rules[rules_key].replace('{', '').replace('}', '').strip() - - if array_type == 'int': - dotty_info[info_key] = list(map(int, rules_value.split(','))) - else: - dotty_info[info_key] = rules_value.split(',') - - elif key_type == 'list': - dotty_info[info_key] = rules[rules_key].split() - - elif key_type == 'bool': - dotty_info[info_key] = rules[rules_key] in true_values - - elif key_type == 'hex': - dotty_info[info_key] = '0x' + rules[rules_key][2:].upper() - - elif key_type == 'int': - dotty_info[info_key] = int(rules[rules_key]) - - else: - dotty_info[info_key] = rules[rules_key] - - except Exception as e: - _log_warning(info_data, f'{rules_key}->{info_key}: {e}') - - info_data.update(dotty_info) - - # Merge in config values that can't be easily mapped - _extract_features(info_data, rules) - - return info_data - - -def _merge_layouts(info_data, new_info_data): - """Merge new_info_data into info_data in an intelligent way. - """ - for layout_name, layout_json in new_info_data['layouts'].items(): - if layout_name in info_data['layouts']: - # Pull in layouts we have a macro for - if len(info_data['layouts'][layout_name]['layout']) != len(layout_json['layout']): - msg = '%s: %s: Number of elements in info.json does not match! info.json:%s != %s:%s' - _log_error(info_data, msg % (info_data['keyboard_folder'], layout_name, len(layout_json['layout']), layout_name, len(info_data['layouts'][layout_name]['layout']))) - else: - for i, key in enumerate(info_data['layouts'][layout_name]['layout']): - key.update(layout_json['layout'][i]) - else: - # Pull in layouts that have matrix data - missing_matrix = False - for key in layout_json.get('layout', {}): - if 'matrix' not in key: - missing_matrix = True - - if not missing_matrix: - if layout_name in info_data['layouts']: - # Update an existing layout with new data - for i, key in enumerate(info_data['layouts'][layout_name]['layout']): - key.update(layout_json['layout'][i]) - - else: - # Copy in the new layout wholesale - layout_json['c_macro'] = False - info_data['layouts'][layout_name] = layout_json - - return info_data - - -def _search_keyboard_h(path): - current_path = Path('keyboards/') - aliases = {} - layouts = {} - - for directory in path.parts: - current_path = current_path / directory - keyboard_h = '%s.h' % (directory,) - keyboard_h_path = current_path / keyboard_h - if keyboard_h_path.exists(): - new_layouts, new_aliases = find_layouts(keyboard_h_path) - layouts.update(new_layouts) - - for alias, alias_text in new_aliases.items(): - if alias_text in layouts: - aliases[alias] = alias_text - - return layouts, aliases - - -def _find_all_layouts(info_data, keyboard): - """Looks for layout macros associated with this keyboard. - """ - layouts, aliases = _search_keyboard_h(Path(keyboard)) - - if not layouts: - # If we don't find any layouts from info.json or keyboard.h we widen our search. This is error prone which is why we want to encourage people to follow the standard above. - info_data['parse_warnings'].append('%s: Falling back to searching for KEYMAP/LAYOUT macros.' % (keyboard)) - - for file in glob('keyboards/%s/*.h' % keyboard): - if file.endswith('.h'): - these_layouts, these_aliases = find_layouts(file) - - if these_layouts: - layouts.update(these_layouts) - - for alias, alias_text in these_aliases.items(): - if alias_text in layouts: - aliases[alias] = alias_text - - return layouts, aliases - - -def _log_error(info_data, message): - """Send an error message to both JSON and the log. - """ - info_data['parse_errors'].append(message) - cli.log.error('%s: %s', info_data.get('keyboard_folder', 'Unknown Keyboard!'), message) - - -def _log_warning(info_data, message): - """Send a warning message to both JSON and the log. - """ - info_data['parse_warnings'].append(message) - cli.log.warning('%s: %s', info_data.get('keyboard_folder', 'Unknown Keyboard!'), message) - - -def arm_processor_rules(info_data, rules): - """Setup the default info for an ARM board. - """ - info_data['processor_type'] = 'arm' - info_data['protocol'] = 'ChibiOS' - - if 'bootloader' not in info_data: - if 'STM32' in info_data['processor']: - info_data['bootloader'] = 'stm32-dfu' - else: - info_data['bootloader'] = 'unknown' - - if 'STM32' in info_data['processor']: - info_data['platform'] = 'STM32' - elif 'MCU_SERIES' in rules: - info_data['platform'] = rules['MCU_SERIES'] - elif 'ARM_ATSAM' in rules: - info_data['platform'] = 'ARM_ATSAM' - - return info_data - - -def avr_processor_rules(info_data, rules): - """Setup the default info for an AVR board. - """ - info_data['processor_type'] = 'avr' - info_data['platform'] = rules['ARCH'] if 'ARCH' in rules else 'unknown' - info_data['protocol'] = 'V-USB' if rules.get('MCU') in VUSB_PROCESSORS else 'LUFA' - - if 'bootloader' not in info_data: - info_data['bootloader'] = 'atmel-dfu' - - # FIXME(fauxpark/anyone): Eventually we should detect the protocol by looking at PROTOCOL inherited from mcu_selection.mk: - # info_data['protocol'] = 'V-USB' if rules.get('PROTOCOL') == 'VUSB' else 'LUFA' - - return info_data - - -def unknown_processor_rules(info_data, rules): - """Setup the default keyboard info for unknown boards. - """ - info_data['bootloader'] = 'unknown' - info_data['platform'] = 'unknown' - info_data['processor'] = 'unknown' - info_data['processor_type'] = 'unknown' - info_data['protocol'] = 'unknown' - - return info_data - - -def merge_info_jsons(keyboard, info_data): - """Return a merged copy of all the info.json files for a keyboard. - """ - for info_file in find_info_json(keyboard): - # Load and validate the JSON data - new_info_data = json_load(info_file) - - if not isinstance(new_info_data, dict): - _log_error(info_data, "Invalid file %s, root object should be a dictionary." % (str(info_file),)) - continue - - try: - keyboard_validate(new_info_data) - except jsonschema.ValidationError as e: - json_path = '.'.join([str(p) for p in e.absolute_path]) - cli.log.error('Not including data from file: %s', info_file) - cli.log.error('\t%s: %s', json_path, e.message) - continue - - # Merge layout data in - if 'layout_aliases' in new_info_data: - info_data['layout_aliases'] = {**info_data.get('layout_aliases', {}), **new_info_data['layout_aliases']} - del new_info_data['layout_aliases'] - - for layout_name, layout in new_info_data.get('layouts', {}).items(): - if layout_name in info_data.get('layout_aliases', {}): - _log_warning(info_data, f"info.json uses alias name {layout_name} instead of {info_data['layout_aliases'][layout_name]}") - layout_name = info_data['layout_aliases'][layout_name] - - if layout_name in info_data['layouts']: - for new_key, existing_key in zip(layout['layout'], info_data['layouts'][layout_name]['layout']): - existing_key.update(new_key) - else: - layout['c_macro'] = False - info_data['layouts'][layout_name] = layout - - # Update info_data with the new data - if 'layouts' in new_info_data: - del new_info_data['layouts'] - - deep_update(info_data, new_info_data) - - return info_data - - -def find_info_json(keyboard): - """Finds all the info.json files associated with a keyboard. - """ - # Find the most specific first - base_path = Path('keyboards') - keyboard_path = base_path / keyboard - keyboard_parent = keyboard_path.parent - info_jsons = [keyboard_path / 'info.json'] - - # Add DEFAULT_FOLDER before parents, if present - rules = rules_mk(keyboard) - if 'DEFAULT_FOLDER' in rules: - info_jsons.append(Path(rules['DEFAULT_FOLDER']) / 'info.json') - - # Add in parent folders for least specific - for _ in range(5): - info_jsons.append(keyboard_parent / 'info.json') - if keyboard_parent.parent == base_path: - break - keyboard_parent = keyboard_parent.parent - - # Return a list of the info.json files that actually exist - return [info_json for info_json in info_jsons if info_json.exists()] + return info_data \ No newline at end of file diff --git a/lib/python/qmk/keyboard.py b/lib/python/qmk/keyboard.py index c87ea9050b..77fd9b92d5 100644 --- a/lib/python/qmk/keyboard.py +++ b/lib/python/qmk/keyboard.py @@ -64,6 +64,17 @@ def find_readme(keyboard): return cur_dir / 'readme.md' +def is_keyboard_target(keyboard_target): + """Checks to make sure the supplied keyboard_target is valid. + + This is mainly used by commands that accept --keyboard. + """ + if keyboard_target in ['all', 'all-avr', 'all-chibios', 'all-arm_atsam']: + return keyboard_target + + return keyboard_folder(keyboard_target) + + def keyboard_folder(keyboard): """Returns the actual keyboard folder. diff --git a/lib/python/qmk/keymap.py b/lib/python/qmk/keymap.py index 2d5921e7a8..992b280296 100644 --- a/lib/python/qmk/keymap.py +++ b/lib/python/qmk/keymap.py @@ -14,6 +14,7 @@ from pygments import lex import qmk.path from qmk.keyboard import find_keyboard_from_dir, rules_mk from qmk.errors import CppError +from qmk.metadata import basic_info_json # The `keymap.c` template to use when a keyboard doesn't have its own DEFAULT_KEYMAP_C = """#include QMK_KEYBOARD_H @@ -327,36 +328,33 @@ def list_keymaps(keyboard, c=True, json=True, additional_files=None, fullpath=Fa Returns: a sorted list of valid keymap names. """ - # parse all the rules.mk files for the keyboard - rules = rules_mk(keyboard) + info_data = basic_info_json(keyboard) names = set() - if rules: - keyboards_dir = Path('keyboards') - kb_path = keyboards_dir / keyboard + keyboards_dir = Path('keyboards') + kb_path = keyboards_dir / info_data['keyboard_folder'] - # walk up the directory tree until keyboards_dir - # and collect all directories' name with keymap.c file in it - while kb_path != keyboards_dir: - keymaps_dir = kb_path / "keymaps" + # walk up the directory tree until keyboards_dir + # and collect all directories' name with keymap.c file in it + while kb_path != keyboards_dir: + keymaps_dir = kb_path / "keymaps" - if keymaps_dir.is_dir(): - for keymap in keymaps_dir.iterdir(): - if is_keymap_dir(keymap, c, json, additional_files): - keymap = keymap if fullpath else keymap.name - names.add(keymap) + if keymaps_dir.is_dir(): + for keymap in keymaps_dir.iterdir(): + if is_keymap_dir(keymap, c, json, additional_files): + keymap = keymap if fullpath else keymap.name + names.add(keymap) - kb_path = kb_path.parent + kb_path = kb_path.parent - # if community layouts are supported, get them - if "LAYOUTS" in rules: - for layout in rules["LAYOUTS"].split(): - cl_path = Path('layouts/community') / layout - if cl_path.is_dir(): - for keymap in cl_path.iterdir(): - if is_keymap_dir(keymap, c, json, additional_files): - keymap = keymap if fullpath else keymap.name - names.add(keymap) + # if community layouts are supported, get them + for layout in info_data.get('community_layouts', []): + cl_path = Path('layouts/community') / layout + if cl_path.is_dir(): + for keymap in cl_path.iterdir(): + if is_keymap_dir(keymap, c, json, additional_files): + keymap = keymap if fullpath else keymap.name + names.add(keymap) return sorted(names) diff --git a/lib/python/qmk/metadata.py b/lib/python/qmk/metadata.py new file mode 100644 index 0000000000..a957cb1fd8 --- /dev/null +++ b/lib/python/qmk/metadata.py @@ -0,0 +1,495 @@ +"""Functions that help us generate and use info.json files. +""" +from functools import lru_cache +from glob import glob +from pathlib import Path + +import jsonschema +from dotty_dict import dotty +from milc import cli + +from qmk.constants import CHIBIOS_PROCESSORS, LUFA_PROCESSORS, VUSB_PROCESSORS +from qmk.c_parse import find_layouts +from qmk.json_schema import deep_update, json_load, keyboard_validate +from qmk.keyboard import config_h, rules_mk +from qmk.makefile import parse_rules_mk_file +from qmk.math import compute + +true_values = ['1', 'on', 'yes', 'true'] +false_values = ['0', 'off', 'no', 'false'] + + +@lru_cache(maxsize=None) +def basic_info_json(keyboard): + """Generate a subset of info.json for a specific keyboard. + + This does no validation, and should only be used as needed to avoid loops or when performance is critical. + """ + cur_dir = Path('keyboards') + rules = parse_rules_mk_file(cur_dir / keyboard / 'rules.mk') + + if 'DEFAULT_FOLDER' in rules: + keyboard = rules['DEFAULT_FOLDER'] + rules = parse_rules_mk_file(cur_dir / keyboard / 'rules.mk', rules) + + info_data = { + 'keyboard_name': str(keyboard), + 'keyboard_folder': str(keyboard), + 'keymaps': {}, + 'layouts': {}, + 'parse_errors': [], + 'parse_warnings': [], + 'maintainer': 'qmk', + } + + # Populate layout data + layouts, aliases = _find_all_layouts(info_data, keyboard) + + if aliases: + info_data['layout_aliases'] = aliases + + for layout_name, layout_json in layouts.items(): + if not layout_name.startswith('LAYOUT_kc'): + layout_json['c_macro'] = True + info_data['layouts'][layout_name] = layout_json + + # Merge in the data from info.json, config.h, and rules.mk + info_data = merge_info_jsons(keyboard, info_data) + info_data = _extract_config_h(info_data) + info_data = _extract_rules_mk(info_data) + + return info_data + + +def _extract_features(info_data, rules): + """Find all the features enabled in rules.mk. + """ + # Special handling for bootmagic which also supports a "lite" mode. + if rules.get('BOOTMAGIC_ENABLE') == 'lite': + rules['BOOTMAGIC_LITE_ENABLE'] = 'on' + del rules['BOOTMAGIC_ENABLE'] + if rules.get('BOOTMAGIC_ENABLE') == 'full': + rules['BOOTMAGIC_ENABLE'] = 'on' + + # Skip non-boolean features we haven't implemented special handling for + for feature in 'HAPTIC_ENABLE', 'QWIIC_ENABLE': + if rules.get(feature): + del rules[feature] + + # Process the rest of the rules as booleans + for key, value in rules.items(): + if key.endswith('_ENABLE'): + key = '_'.join(key.split('_')[:-1]).lower() + value = True if value.lower() in true_values else False if value.lower() in false_values else value + + if 'config_h_features' not in info_data: + info_data['config_h_features'] = {} + + if 'features' not in info_data: + info_data['features'] = {} + + if key in info_data['features']: + info_log_warning(info_data, 'Feature %s is specified in both info.json and rules.mk, the rules.mk value wins.' % (key,)) + + info_data['features'][key] = value + info_data['config_h_features'][key] = value + + return info_data + + +def _pin_name(pin): + """Returns the proper representation for a pin. + """ + pin = pin.strip() + + if not pin: + return None + + elif pin.isdigit(): + return int(pin) + + elif pin == 'NO_PIN': + return None + + elif pin[0] in 'ABCDEFGHIJK' and pin[1].isdigit(): + return pin + + raise ValueError(f'Invalid pin: {pin}') + + +def _extract_pins(pins): + """Returns a list of pins from a comma separated string of pins. + """ + return [_pin_name(pin) for pin in pins.split(',')] + + +def _extract_direct_matrix(info_data, direct_pins): + """ + """ + info_data['matrix_pins'] = {} + direct_pin_array = [] + + while direct_pins[-1] != '}': + direct_pins = direct_pins[:-1] + + for row in direct_pins.split('},{'): + if row.startswith('{'): + row = row[1:] + + if row.endswith('}'): + row = row[:-1] + + direct_pin_array.append([]) + + for pin in row.split(','): + if pin == 'NO_PIN': + pin = None + + direct_pin_array[-1].append(pin) + + return direct_pin_array + + +def _extract_matrix_info(info_data, config_c): + """Populate the matrix information. + """ + row_pins = config_c.get('MATRIX_ROW_PINS', '').replace('{', '').replace('}', '').strip() + col_pins = config_c.get('MATRIX_COL_PINS', '').replace('{', '').replace('}', '').strip() + direct_pins = config_c.get('DIRECT_PINS', '').replace(' ', '')[1:-1] + + if 'MATRIX_ROWS' in config_c and 'MATRIX_COLS' in config_c: + if 'matrix_size' in info_data: + info_log_warning(info_data, 'Matrix size is specified in both info.json and config.h, the config.h values win.') + + info_data['matrix_size'] = { + 'cols': compute(config_c.get('MATRIX_COLS', '0')), + 'rows': compute(config_c.get('MATRIX_ROWS', '0')), + } + + if row_pins and col_pins: + if 'matrix_pins' in info_data: + info_log_warning(info_data, 'Matrix pins are specified in both info.json and config.h, the config.h values win.') + + info_data['matrix_pins'] = { + 'cols': _extract_pins(col_pins), + 'rows': _extract_pins(row_pins), + } + + if direct_pins: + if 'matrix_pins' in info_data: + info_log_warning(info_data, 'Direct pins are specified in both info.json and config.h, the config.h values win.') + + info_data['matrix_pins']['direct'] = _extract_direct_matrix(info_data, direct_pins) + + return info_data + + +def _extract_config_h(info_data): + """Pull some keyboard information from existing config.h files + """ + config_c = config_h(info_data['keyboard_folder']) + + # Pull in data from the json map + dotty_info = dotty(info_data) + info_config_map = json_load(Path('data/mappings/info_config.json')) + + for config_key, info_dict in info_config_map.items(): + info_key = info_dict['info_key'] + key_type = info_dict.get('value_type', 'str') + + try: + if config_key in config_c and info_dict.get('to_json', True): + if dotty_info.get(info_key) and info_dict.get('warn_duplicate', True): + info_log_warning(info_data, '%s in config.h is overwriting %s in info.json' % (config_key, info_key)) + + if key_type.startswith('array'): + if '.' in key_type: + key_type, array_type = key_type.split('.', 1) + else: + array_type = None + + config_value = config_c[config_key].replace('{', '').replace('}', '').strip() + + if array_type == 'int': + dotty_info[info_key] = list(map(int, config_value.split(','))) + else: + dotty_info[info_key] = config_value.split(',') + + elif key_type == 'bool': + dotty_info[info_key] = config_c[config_key] in true_values + + elif key_type == 'hex': + dotty_info[info_key] = '0x' + config_c[config_key][2:].upper() + + elif key_type == 'list': + dotty_info[info_key] = config_c[config_key].split() + + elif key_type == 'int': + dotty_info[info_key] = int(config_c[config_key]) + + else: + dotty_info[info_key] = config_c[config_key] + + except Exception as e: + info_log_warning(info_data, f'{config_key}->{info_key}: {e}') + + info_data.update(dotty_info) + + # Pull data that easily can't be mapped in json + _extract_matrix_info(info_data, config_c) + + return info_data + + +def _extract_rules_mk(info_data): + """Pull some keyboard information from existing rules.mk files + """ + rules = rules_mk(info_data['keyboard_folder']) + info_data['processor'] = rules.get('MCU', info_data.get('processor', 'atmega32u4')) + + if info_data['processor'] in CHIBIOS_PROCESSORS: + arm_processor_rules(info_data, rules) + + elif info_data['processor'] in LUFA_PROCESSORS + VUSB_PROCESSORS: + avr_processor_rules(info_data, rules) + + else: + cli.log.warning("%s: Unknown MCU: %s" % (info_data['keyboard_folder'], info_data['processor'])) + unknown_processor_rules(info_data, rules) + + # Pull in data from the json map + dotty_info = dotty(info_data) + info_rules_map = json_load(Path('data/mappings/info_rules.json')) + + for rules_key, info_dict in info_rules_map.items(): + info_key = info_dict['info_key'] + key_type = info_dict.get('value_type', 'str') + + try: + if rules_key in rules and info_dict.get('to_json', True): + if dotty_info.get(info_key) and info_dict.get('warn_duplicate', True): + info_log_warning(info_data, '%s in rules.mk is overwriting %s in info.json' % (rules_key, info_key)) + + if key_type.startswith('array'): + if '.' in key_type: + key_type, array_type = key_type.split('.', 1) + else: + array_type = None + + rules_value = rules[rules_key].replace('{', '').replace('}', '').strip() + + if array_type == 'int': + dotty_info[info_key] = list(map(int, rules_value.split(','))) + else: + dotty_info[info_key] = rules_value.split(',') + + elif key_type == 'list': + dotty_info[info_key] = rules[rules_key].split() + + elif key_type == 'bool': + dotty_info[info_key] = rules[rules_key] in true_values + + elif key_type == 'hex': + dotty_info[info_key] = '0x' + rules[rules_key][2:].upper() + + elif key_type == 'int': + dotty_info[info_key] = int(rules[rules_key]) + + else: + dotty_info[info_key] = rules[rules_key] + + except Exception as e: + info_log_warning(info_data, f'{rules_key}->{info_key}: {e}') + + info_data.update(dotty_info) + + # Merge in config values that can't be easily mapped + _extract_features(info_data, rules) + + return info_data + + +def _search_keyboard_h(path): + current_path = Path('keyboards/') + aliases = {} + layouts = {} + + for directory in path.parts: + current_path = current_path / directory + keyboard_h = '%s.h' % (directory,) + keyboard_h_path = current_path / keyboard_h + if keyboard_h_path.exists(): + new_layouts, new_aliases = find_layouts(keyboard_h_path) + layouts.update(new_layouts) + + for alias, alias_text in new_aliases.items(): + if alias_text in layouts: + aliases[alias] = alias_text + + return layouts, aliases + + +def _find_all_layouts(info_data, keyboard): + """Looks for layout macros associated with this keyboard. + """ + found_layouts = False + layouts, aliases = _search_keyboard_h(Path(keyboard)) + + if layouts: + found_layouts = True + else: + for layout in info_data['layouts'].values(): + if 'matrix' in layout['layout']: + found_layouts = True + break + + if not found_layouts: + # If we don't find any layouts from info.json or keyboard.h we widen our search. This is error prone which is why we want to encourage people to follow the standard. + info_data['parse_warnings'].append('%s: Falling back to searching for KEYMAP/LAYOUT macros.' % (keyboard)) + + for file in glob('keyboards/%s/*.h' % keyboard): + if file.endswith('.h'): + these_layouts, these_aliases = find_layouts(file) + + if these_layouts: + layouts.update(these_layouts) + + for alias, alias_text in these_aliases.items(): + if alias_text in layouts: + aliases[alias] = alias_text + + return layouts, aliases + + +def info_log_error(info_data, message): + """Send an error message to both JSON and the log. + """ + info_data['parse_errors'].append(message) + cli.log.error('%s: %s', info_data.get('keyboard_folder', 'Unknown Keyboard!'), message) + + +def info_log_warning(info_data, message): + """Send a warning message to both JSON and the log. + """ + info_data['parse_warnings'].append(message) + cli.log.warning('%s: %s', info_data.get('keyboard_folder', 'Unknown Keyboard!'), message) + + +def arm_processor_rules(info_data, rules): + """Setup the default info for an ARM board. + """ + info_data['processor_type'] = 'arm' + info_data['protocol'] = 'ChibiOS' + + if 'bootloader' not in info_data: + if 'STM32' in info_data['processor']: + info_data['bootloader'] = 'stm32-dfu' + else: + info_data['bootloader'] = 'unknown' + + if 'STM32' in info_data['processor']: + info_data['platform'] = 'STM32' + elif 'MCU_SERIES' in rules: + info_data['platform'] = rules['MCU_SERIES'] + elif 'ARM_ATSAM' in rules: + info_data['platform'] = 'ARM_ATSAM' + + return info_data + + +def avr_processor_rules(info_data, rules): + """Setup the default info for an AVR board. + """ + info_data['processor_type'] = 'avr' + info_data['platform'] = rules['ARCH'] if 'ARCH' in rules else 'unknown' + info_data['protocol'] = 'V-USB' if rules.get('MCU') in VUSB_PROCESSORS else 'LUFA' + + if 'bootloader' not in info_data: + info_data['bootloader'] = 'atmel-dfu' + + # FIXME(fauxpark/anyone): Eventually we should detect the protocol by looking at PROTOCOL inherited from mcu_selection.mk: + # info_data['protocol'] = 'V-USB' if rules.get('PROTOCOL') == 'VUSB' else 'LUFA' + + return info_data + + +def unknown_processor_rules(info_data, rules): + """Setup the default keyboard info for unknown boards. + """ + info_data['bootloader'] = 'unknown' + info_data['platform'] = 'unknown' + info_data['processor'] = 'unknown' + info_data['processor_type'] = 'unknown' + info_data['protocol'] = 'unknown' + + return info_data + + +def merge_info_jsons(keyboard, info_data): + """Return a merged copy of all the info.json files for a keyboard. + """ + for info_file in find_info_json(keyboard): + # Load and validate the JSON data + new_info_data = json_load(info_file) + + if not isinstance(new_info_data, dict): + info_log_error(info_data, "Invalid file %s, root object should be a dictionary." % (str(info_file),)) + continue + + try: + keyboard_validate(new_info_data) + except jsonschema.ValidationError as e: + json_path = '.'.join([str(p) for p in e.absolute_path]) + cli.log.error('Not including data from file: %s', info_file) + cli.log.error('\t%s: %s', json_path, e.message) + continue + + # Merge layout data in + if 'layout_aliases' in new_info_data: + info_data['layout_aliases'] = {**info_data.get('layout_aliases', {}), **new_info_data['layout_aliases']} + del new_info_data['layout_aliases'] + + for layout_name, layout in new_info_data.get('layouts', {}).items(): + if layout_name in info_data.get('layout_aliases', {}): + info_log_warning(info_data, f"info.json uses alias name {layout_name} instead of {info_data['layout_aliases'][layout_name]}") + layout_name = info_data['layout_aliases'][layout_name] + + if layout_name in info_data['layouts']: + for new_key, existing_key in zip(layout['layout'], info_data['layouts'][layout_name]['layout']): + existing_key.update(new_key) + else: + layout['c_macro'] = False + info_data['layouts'][layout_name] = layout + + # Update info_data with the new data + if 'layouts' in new_info_data: + del new_info_data['layouts'] + + deep_update(info_data, new_info_data) + + return info_data + + +def find_info_json(keyboard): + """Finds all the info.json files associated with a keyboard. + """ + # Find the most specific first + base_path = Path('keyboards') + keyboard_path = base_path / keyboard + keyboard_parent = keyboard_path.parent + info_jsons = [keyboard_path / 'info.json'] + + # Add DEFAULT_FOLDER before parents, if present + rules = rules_mk(keyboard) + if 'DEFAULT_FOLDER' in rules: + info_jsons.append(Path(rules['DEFAULT_FOLDER']) / 'info.json') + + # Add in parent folders for least specific + for _ in range(5): + info_jsons.append(keyboard_parent / 'info.json') + if keyboard_parent.parent == base_path: + break + keyboard_parent = keyboard_parent.parent + + # Return a list of the info.json files that actually exist + return [info_json for info_json in info_jsons if info_json.exists()]