Rework qmk compile to bypass Makefile. Add new --filter option.

Makefile

@@ -393,7 +393,7 @@ define PARSE_KEYMAP
     # Specify the variables that we are passing forward to submake
     MAKE_VARS := KEYBOARD=$$(CURRENT_KB) KEYMAP=$$(CURRENT_KM) REQUIRE_PLATFORM_KEY=$$(REQUIRE_PLATFORM_KEY) QMK_BIN=$$(QMK_BIN)
     # And the first part of the make command
-    MAKE_CMD := $$(MAKE) -r -R -C $(ROOT_DIR) -f build_keyboard.mk $$(MAKE_TARGET)
+    MAKE_CMD := echo $$(MAKE) -r -R -C $(ROOT_DIR) -f build_keyboard.mk $$(MAKE_TARGET)
     # The message to display
     MAKE_MSG := $$(MSG_MAKE_KB)
     # We run the command differently, depending on if we want more output or not

lib/python/qmk/cli/clean.py

@@ -2,7 +2,7 @@
 """
 from subprocess import DEVNULL
 
-from qmk.commands import create_make_target
+from qmk.commands import _find_make
 from milc import cli
 
 
@@ -11,4 +11,6 @@ from milc import cli
 def clean(cli):
     """Runs `make clean` (or `make distclean` if --all is passed)
     """
-    cli.run(create_make_target('distclean' if cli.args.all else 'clean'), capture_output=False, stdin=DEVNULL)
+    make_cmd = [_find_make(), 'distclean' if cli.args.all else 'clean']
+
+    cli.run(make_cmd, capture_output=False, stdin=DEVNULL)

lib/python/qmk/cli/compile.py

@@ -5,20 +5,70 @@ You can compile a keymap already in the repo or using a QMK Configurator export.
 from subprocess import DEVNULL
 
 from argcomplete.completers import FilesCompleter
+from dotty_dict import dotty
 from milc import cli
 
 import qmk.path
-from qmk.decorators import automagic_keyboard, automagic_keymap
+from qmk.decorators import automagic_keyboard, automagic_keymap, lru_cache
 from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json
-from qmk.keyboard import keyboard_completer, keyboard_folder
+from qmk.info import info_json
-from qmk.keymap import keymap_completer
+from qmk.keyboard import keyboard_completer, is_keyboard_target, list_keyboards
+from qmk.keymap import keymap_completer, list_keymaps
+from qmk.metadata import true_values, false_values
+
+
+@lru_cache()
+def _keyboard_list():
+    """Returns a list of keyboards matching cli.config.compile.keyboard.
+    """
+    if cli.config.compile.keyboard == 'all':
+        return list_keyboards()
+
+    elif cli.config.compile.keyboard.startswith('all-'):
+        return list_keyboards()
+
+    return [cli.config.compile.keyboard]
+
+
+def keyboard_keymap_iter():
+    """Iterates over the keyboard/keymap for this command and yields a pairing of each.
+    """
+    for keyboard in _keyboard_list():
+        continue_flag = False
+        if cli.args.filter:
+            info_data = dotty(info_json(keyboard))
+            for filter in cli.args.filter:
+                if '=' in filter:
+                    key, value = filter.split('=', 1)
+                    if value in true_values:
+                        value = True
+                    elif value in false_values:
+                        value = False
+                    elif value.isdigit():
+                        value = int(value)
+                    elif '.' in value and value.replace('.').isdigit():
+                        value = float(value)
+
+                    if info_data.get(key) != value:
+                        continue_flag = True
+                        break
+
+            if continue_flag:
+                continue
+
+        if cli.config.compile.keymap == 'all':
+            for keymap in list_keymaps(keyboard):
+                yield keyboard, keymap
+        else:
+            yield keyboard, cli.config.compile.keymap
 
 
 @cli.argument('filename', nargs='?', arg_only=True, type=qmk.path.FileType('r'), completer=FilesCompleter('.json'), help='The configurator export to compile')
-@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.')
+@cli.argument('-kb', '--keyboard', type=is_keyboard_target, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.')
 @cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.')
 @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.")
 @cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs to run.")
+@cli.argument('-f', '--filter', arg_only=True, action='append', default=[], help="Filter your list against info.json.")
 @cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.")
 @cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.")
 @cli.subcommand('Compile a QMK Firmware.')
@@ -31,47 +81,79 @@ def compile(cli):
 
     If a keyboard and keymap are provided this command will build a firmware based on that.
     """
-    if cli.args.clean and not cli.args.filename and not cli.args.dry_run:
+    envs = {'REQUIRE_PLATFORM_KEY': ''}
-        command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, 'clean')
+    silent = cli.config.compile.keyboard == 'all' or cli.config.compile.keyboard.startswith('all-') or cli.config.compile.keymap == 'all'
-        cli.run(command, capture_output=False, stdin=DEVNULL)
 
-    # Build the environment vars
+    # Setup the environment
-    envs = {}
     for env in cli.args.env:
         if '=' in env:
             key, value = env.split('=', 1)
+            if key in envs:
+                cli.log.warning('Overwriting existing environment variable %s=%s with %s=%s!', key, envs[key], key, value)
             envs[key] = value
         else:
             cli.log.warning('Invalid environment variable: %s', env)
 
-    # Determine the compile command
+    if cli.config.compile.keyboard.startswith('all-'):
-    command = None
+        envs['REQUIRE_PLATFORM_KEY'] = cli.config.compile.keyboard[4:]
+
+    # Run clean if necessary
+    if cli.args.clean and not cli.args.filename and not cli.args.dry_run:
+        for keyboard, keymap in keyboard_keymap_iter():
+            cli.log.info('Cleaning previous build files for keyboard {fg_cyan}%s{fg_reset} keymap {fg_cyan}%s', keyboard, keymap)
+            _, _, make_cmd = create_make_command(keyboard, keymap, 'clean', cli.config.compile.parallel, silent, **envs)
+            cli.run(make_cmd, capture_output=False, stdin=DEVNULL)
+
+    # Determine the compile command(s)
+    commands = None
 
     if cli.args.filename:
         # If a configurator JSON was provided generate a keymap and compile it
         user_keymap = parse_configurator_json(cli.args.filename)
-        command = compile_configurator_json(user_keymap, parallel=cli.config.compile.parallel, **envs)
+        commands = [compile_configurator_json(user_keymap, parallel=cli.config.compile.parallel, **envs)]
 
-    else:
+    elif cli.config.compile.keyboard and cli.config.compile.keymap:
-        if cli.config.compile.keyboard and cli.config.compile.keymap:
+        if cli.args.filter:
-            # Generate the make command for a specific keyboard/keymap.
+            cli.log.info('Generating the list of keyboards to compile, this may take some time.')
-            command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, parallel=cli.config.compile.parallel, **envs)
 
-        elif not cli.config.compile.keyboard:
+        commands = [create_make_command(keyboard, keymap, parallel=cli.config.compile.parallel, silent=silent, **envs) for keyboard, keymap in keyboard_keymap_iter()]
-            cli.log.error('Could not determine keyboard!')
+
-        elif not cli.config.compile.keymap:
+    elif not cli.config.compile.keyboard:
-            cli.log.error('Could not determine keymap!')
+        cli.log.error('Could not determine keyboard!')
+
+    elif not cli.config.compile.keymap:
+        cli.log.error('Could not determine keymap!')
 
     # Compile the firmware, if we're able to
-    if command:
+    if commands:
-        cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command))
+        returncodes = []
-        if not cli.args.dry_run:
+        for keyboard, keymap, command in commands:
-            cli.echo('\n')
+            print()
-            # FIXME(skullydazed/anyone): Remove text=False once milc 1.0.11 has had enough time to be installed everywhere.
+            cli.log.info('Compiling QMK Firmware for keyboard {fg_cyan}%s{fg_reset} and keymap {fg_cyan}%s', keyboard, keymap)
-            compile = cli.run(command, capture_output=False, text=False)
+            cli.log.debug('Running make command: {fg_blue}%s', ' '.join(command))
-            return compile.returncode
+
+            if not cli.args.dry_run:
+                compile = cli.run(command, capture_output=False)
+                returncodes.append(compile.returncode)
+                if compile.returncode == 0:
+                    cli.log.info('Success!')
+                else:
+                    cli.log.error('Failed!')
+
+        if any(returncodes):
+            print()
+            cli.log.error('Could not compile all targets, look above this message for more details. Failing target(s):')
+
+            for i, returncode in enumerate(returncodes):
+                if returncode != 0:
+                    keyboard, keymap, command = commands[i]
+                    cli.echo('\tkeyboard: {fg_cyan}%s{fg_reset} keymap: {fg_cyan}%s', keyboard, keymap)
+
+    elif cli.args.filter:
+        cli.log.error('No keyboards found after applying filter(s)!')
+        return False
 
     else:
         cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')
-        cli.echo('usage: qmk compile [-h] [-b] [-kb KEYBOARD] [-km KEYMAP] [filename]')
+        cli.print_help()
         return False

lib/python/qmk/cli/flash.py

@@ -11,7 +11,7 @@ from milc import cli
 import qmk.path
 from qmk.decorators import automagic_keyboard, automagic_keymap
 from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json
-from qmk.keyboard import keyboard_completer, keyboard_folder
+from qmk.keyboard import keyboard_completer, is_keyboard_target
 
 
 def print_bootloader_help():
@@ -36,7 +36,7 @@ def print_bootloader_help():
 @cli.argument('-b', '--bootloaders', action='store_true', help='List the available bootloaders.')
 @cli.argument('-bl', '--bootloader', default='flash', help='The flash command, corresponding to qmk\'s make options of bootloaders.')
 @cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.')
-@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.')
+@cli.argument('-kb', '--keyboard', type=is_keyboard_target, completer=keyboard_completer, help='The keyboard to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.')
 @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.")
 @cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs to run.")
 @cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.")

lib/python/qmk/commands.py

@@ -27,34 +27,7 @@ def _find_make():
     return make_cmd
 
 
-def create_make_target(target, parallel=1, **env_vars):
+def create_make_command(keyboard, keymap, target=None, parallel=1, silent=False, **env_vars):
-    """Create a make command
-
-    Args:
-
-        target
-            Usually a make rule, such as 'clean' or 'all'.
-
-        parallel
-            The number of make jobs to run in parallel
-
-        **env_vars
-            Environment variables to be passed to make.
-
-    Returns:
-
-        A command that can be run to make the specified keyboard and keymap
-    """
-    env = []
-    make_cmd = _find_make()
-
-    for key, value in env_vars.items():
-        env.append(f'{key}={value}')
-
-    return [make_cmd, '-j', str(parallel), *env, target]
-
-
-def create_make_command(keyboard, keymap, target=None, parallel=1, **env_vars):
     """Create a make compile command
 
     Args:
@@ -78,12 +51,26 @@ def create_make_command(keyboard, keymap, target=None, parallel=1, **env_vars):
 
         A command that can be run to make the specified keyboard and keymap
     """
-    make_args = [keyboard, keymap]
+    make_cmd = [_find_make(), '--no-print-directory', '-r', '-R', '-C', './', '-f', 'build_keyboard.mk']
+
+    env_vars['KEYBOARD'] = keyboard
+    env_vars['KEYMAP'] = keymap
+    env_vars['QMK_BIN'] = 'bin/qmk' if 'DEPRECATED_BIN_QMK' in os.environ else 'qmk'
+    env_vars['VERBOSE'] = 'true' if cli.config.general.verbose else ''
+    env_vars['SILENT'] = 'true' if silent else 'false'
+    env_vars['COLOR'] = 'true' if cli.config.general.color else ''
+
+    if parallel > 1:
+        make_cmd.append('-j')
+        make_cmd.append(parallel)
 
     if target:
-        make_args.append(target)
+        make_cmd.append(target)
 
-    return create_make_target(':'.join(make_args), parallel, **env_vars)
+    for key, value in env_vars.items():
+        make_cmd.append(f'{key}={value}')
+
+    return keyboard, keymap, make_cmd
 
 
 def get_git_version(repo_dir='.', check_dir='.'):
@@ -204,7 +191,7 @@ def compile_configurator_json(user_keymap, bootloader=None, parallel=1, **env_va
         f'QMK_BIN={"bin/qmk" if "DEPRECATED_BIN_QMK" in os.environ else "qmk"}',
     ])
 
-    return make_command
+    return user_keymap['keyboard'], user_keymap['keymap'], make_command
 
 
 def parse_configurator_json(configurator_file):

lib/python/qmk/info.py

@@ -1,22 +1,14 @@
 """Functions that help us generate and use info.json files.
 """
-from glob import glob
+from functools import lru_cache
 from pathlib import Path
 
 import jsonschema
-from dotty_dict import dotty
 from milc import cli
 
-from qmk.constants import CHIBIOS_PROCESSORS, LUFA_PROCESSORS, VUSB_PROCESSORS
+from qmk.json_schema import keyboard_api_validate
-from qmk.c_parse import find_layouts
-from qmk.json_schema import deep_update, json_load, keyboard_validate, keyboard_api_validate
-from qmk.keyboard import config_h, rules_mk
 from qmk.keymap import list_keymaps
-from qmk.makefile import parse_rules_mk_file
+from qmk.metadata import basic_info_json, info_log_error
-from qmk.math import compute
-
-true_values = ['1', 'on', 'yes']
-false_values = ['0', 'off', 'no']
 
 
 def _valid_community_layout(layout):
@@ -25,45 +17,16 @@ def _valid_community_layout(layout):
     return (Path('layouts/default') / layout).exists()
 
 
+@lru_cache(maxsize=None)
 def info_json(keyboard):
     """Generate the info.json data for a specific keyboard.
     """
-    cur_dir = Path('keyboards')
+    info_data = basic_info_json(keyboard)
-    rules = parse_rules_mk_file(cur_dir / keyboard / 'rules.mk')
-    if 'DEFAULT_FOLDER' in rules:
-        keyboard = rules['DEFAULT_FOLDER']
-        rules = parse_rules_mk_file(cur_dir / keyboard / 'rules.mk', rules)
-
-    info_data = {
-        'keyboard_name': str(keyboard),
-        'keyboard_folder': str(keyboard),
-        'keymaps': {},
-        'layouts': {},
-        'parse_errors': [],
-        'parse_warnings': [],
-        'maintainer': 'qmk',
-    }
 
     # Populate the list of JSON keymaps
     for keymap in list_keymaps(keyboard, c=False, fullpath=True):
         info_data['keymaps'][keymap.name] = {'url': f'https://raw.githubusercontent.com/qmk/qmk_firmware/master/{keymap}/keymap.json'}
 
-    # Populate layout data
-    layouts, aliases = _find_all_layouts(info_data, keyboard)
-
-    if aliases:
-        info_data['layout_aliases'] = aliases
-
-    for layout_name, layout_json in layouts.items():
-        if not layout_name.startswith('LAYOUT_kc'):
-            layout_json['c_macro'] = True
-            info_data['layouts'][layout_name] = layout_json
-
-    # Merge in the data from info.json, config.h, and rules.mk
-    info_data = merge_info_jsons(keyboard, info_data)
-    info_data = _extract_config_h(info_data)
-    info_data = _extract_rules_mk(info_data)
-
     # Validate against the jsonschema
     try:
         keyboard_api_validate(info_data)
@@ -75,477 +38,19 @@ def info_json(keyboard):
 
     # Make sure we have at least one layout
     if not info_data.get('layouts'):
-        _log_error(info_data, 'No LAYOUTs defined! Need at least one layout defined in the keyboard.h or info.json.')
+        info_log_error(info_data, 'No LAYOUTs defined! Need at least one layout defined in the keyboard.h or info.json.')
 
     # Filter out any non-existing community layouts
     for layout in info_data.get('community_layouts', []):
         if not _valid_community_layout(layout):
             # Ignore layout from future checks
             info_data['community_layouts'].remove(layout)
-            _log_error(info_data, 'Claims to support a community layout that does not exist: %s' % (layout))
+            info_log_error(info_data, 'Claims to support a community layout that does not exist: %s' % (layout))
 
     # Make sure we supply layout macros for the community layouts we claim to support
     for layout in info_data.get('community_layouts', []):
         layout_name = 'LAYOUT_' + layout
         if layout_name not in info_data.get('layouts', {}) and layout_name not in info_data.get('layout_aliases', {}):
-            _log_error(info_data, 'Claims to support community layout %s but no %s() macro found' % (layout, layout_name))
+            info_log_error(info_data, 'Claims to support community layout %s but no %s() macro found' % (layout, layout_name))
 
-    return info_data
+    return info_data
-
-
-def _extract_features(info_data, rules):
-    """Find all the features enabled in rules.mk.
-    """
-    # Special handling for bootmagic which also supports a "lite" mode.
-    if rules.get('BOOTMAGIC_ENABLE') == 'lite':
-        rules['BOOTMAGIC_LITE_ENABLE'] = 'on'
-        del rules['BOOTMAGIC_ENABLE']
-    if rules.get('BOOTMAGIC_ENABLE') == 'full':
-        rules['BOOTMAGIC_ENABLE'] = 'on'
-
-    # Skip non-boolean features we haven't implemented special handling for
-    for feature in 'HAPTIC_ENABLE', 'QWIIC_ENABLE':
-        if rules.get(feature):
-            del rules[feature]
-
-    # Process the rest of the rules as booleans
-    for key, value in rules.items():
-        if key.endswith('_ENABLE'):
-            key = '_'.join(key.split('_')[:-1]).lower()
-            value = True if value.lower() in true_values else False if value.lower() in false_values else value
-
-            if 'config_h_features' not in info_data:
-                info_data['config_h_features'] = {}
-
-            if 'features' not in info_data:
-                info_data['features'] = {}
-
-            if key in info_data['features']:
-                _log_warning(info_data, 'Feature %s is specified in both info.json and rules.mk, the rules.mk value wins.' % (key,))
-
-            info_data['features'][key] = value
-            info_data['config_h_features'][key] = value
-
-    return info_data
-
-
-def _pin_name(pin):
-    """Returns the proper representation for a pin.
-    """
-    pin = pin.strip()
-
-    if not pin:
-        return None
-
-    elif pin.isdigit():
-        return int(pin)
-
-    elif pin == 'NO_PIN':
-        return None
-
-    elif pin[0] in 'ABCDEFGHIJK' and pin[1].isdigit():
-        return pin
-
-    raise ValueError(f'Invalid pin: {pin}')
-
-
-def _extract_pins(pins):
-    """Returns a list of pins from a comma separated string of pins.
-    """
-    return [_pin_name(pin) for pin in pins.split(',')]
-
-
-def _extract_direct_matrix(info_data, direct_pins):
-    """
-    """
-    info_data['matrix_pins'] = {}
-    direct_pin_array = []
-
-    while direct_pins[-1] != '}':
-        direct_pins = direct_pins[:-1]
-
-    for row in direct_pins.split('},{'):
-        if row.startswith('{'):
-            row = row[1:]
-
-        if row.endswith('}'):
-            row = row[:-1]
-
-        direct_pin_array.append([])
-
-        for pin in row.split(','):
-            if pin == 'NO_PIN':
-                pin = None
-
-            direct_pin_array[-1].append(pin)
-
-    return direct_pin_array
-
-
-def _extract_matrix_info(info_data, config_c):
-    """Populate the matrix information.
-    """
-    row_pins = config_c.get('MATRIX_ROW_PINS', '').replace('{', '').replace('}', '').strip()
-    col_pins = config_c.get('MATRIX_COL_PINS', '').replace('{', '').replace('}', '').strip()
-    direct_pins = config_c.get('DIRECT_PINS', '').replace(' ', '')[1:-1]
-
-    if 'MATRIX_ROWS' in config_c and 'MATRIX_COLS' in config_c:
-        if 'matrix_size' in info_data:
-            _log_warning(info_data, 'Matrix size is specified in both info.json and config.h, the config.h values win.')
-
-        info_data['matrix_size'] = {
-            'cols': compute(config_c.get('MATRIX_COLS', '0')),
-            'rows': compute(config_c.get('MATRIX_ROWS', '0')),
-        }
-
-    if row_pins and col_pins:
-        if 'matrix_pins' in info_data:
-            _log_warning(info_data, 'Matrix pins are specified in both info.json and config.h, the config.h values win.')
-
-        info_data['matrix_pins'] = {
-            'cols': _extract_pins(col_pins),
-            'rows': _extract_pins(row_pins),
-        }
-
-    if direct_pins:
-        if 'matrix_pins' in info_data:
-            _log_warning(info_data, 'Direct pins are specified in both info.json and config.h, the config.h values win.')
-
-        info_data['matrix_pins']['direct'] = _extract_direct_matrix(info_data, direct_pins)
-
-    return info_data
-
-
-def _extract_config_h(info_data):
-    """Pull some keyboard information from existing config.h files
-    """
-    config_c = config_h(info_data['keyboard_folder'])
-
-    # Pull in data from the json map
-    dotty_info = dotty(info_data)
-    info_config_map = json_load(Path('data/mappings/info_config.json'))
-
-    for config_key, info_dict in info_config_map.items():
-        info_key = info_dict['info_key']
-        key_type = info_dict.get('value_type', 'str')
-
-        try:
-            if config_key in config_c and info_dict.get('to_json', True):
-                if dotty_info.get(info_key) and info_dict.get('warn_duplicate', True):
-                    _log_warning(info_data, '%s in config.h is overwriting %s in info.json' % (config_key, info_key))
-
-                if key_type.startswith('array'):
-                    if '.' in key_type:
-                        key_type, array_type = key_type.split('.', 1)
-                    else:
-                        array_type = None
-
-                    config_value = config_c[config_key].replace('{', '').replace('}', '').strip()
-
-                    if array_type == 'int':
-                        dotty_info[info_key] = list(map(int, config_value.split(',')))
-                    else:
-                        dotty_info[info_key] = config_value.split(',')
-
-                elif key_type == 'bool':
-                    dotty_info[info_key] = config_c[config_key] in true_values
-
-                elif key_type == 'hex':
-                    dotty_info[info_key] = '0x' + config_c[config_key][2:].upper()
-
-                elif key_type == 'list':
-                    dotty_info[info_key] = config_c[config_key].split()
-
-                elif key_type == 'int':
-                    dotty_info[info_key] = int(config_c[config_key])
-
-                else:
-                    dotty_info[info_key] = config_c[config_key]
-
-        except Exception as e:
-            _log_warning(info_data, f'{config_key}->{info_key}: {e}')
-
-    info_data.update(dotty_info)
-
-    # Pull data that easily can't be mapped in json
-    _extract_matrix_info(info_data, config_c)
-
-    return info_data
-
-
-def _extract_rules_mk(info_data):
-    """Pull some keyboard information from existing rules.mk files
-    """
-    rules = rules_mk(info_data['keyboard_folder'])
-    info_data['processor'] = rules.get('MCU', info_data.get('processor', 'atmega32u4'))
-
-    if info_data['processor'] in CHIBIOS_PROCESSORS:
-        arm_processor_rules(info_data, rules)
-
-    elif info_data['processor'] in LUFA_PROCESSORS + VUSB_PROCESSORS:
-        avr_processor_rules(info_data, rules)
-
-    else:
-        cli.log.warning("%s: Unknown MCU: %s" % (info_data['keyboard_folder'], info_data['processor']))
-        unknown_processor_rules(info_data, rules)
-
-    # Pull in data from the json map
-    dotty_info = dotty(info_data)
-    info_rules_map = json_load(Path('data/mappings/info_rules.json'))
-
-    for rules_key, info_dict in info_rules_map.items():
-        info_key = info_dict['info_key']
-        key_type = info_dict.get('value_type', 'str')
-
-        try:
-            if rules_key in rules and info_dict.get('to_json', True):
-                if dotty_info.get(info_key) and info_dict.get('warn_duplicate', True):
-                    _log_warning(info_data, '%s in rules.mk is overwriting %s in info.json' % (rules_key, info_key))
-
-                if key_type.startswith('array'):
-                    if '.' in key_type:
-                        key_type, array_type = key_type.split('.', 1)
-                    else:
-                        array_type = None
-
-                    rules_value = rules[rules_key].replace('{', '').replace('}', '').strip()
-
-                    if array_type == 'int':
-                        dotty_info[info_key] = list(map(int, rules_value.split(',')))
-                    else:
-                        dotty_info[info_key] = rules_value.split(',')
-
-                elif key_type == 'list':
-                    dotty_info[info_key] = rules[rules_key].split()
-
-                elif key_type == 'bool':
-                    dotty_info[info_key] = rules[rules_key] in true_values
-
-                elif key_type == 'hex':
-                    dotty_info[info_key] = '0x' + rules[rules_key][2:].upper()
-
-                elif key_type == 'int':
-                    dotty_info[info_key] = int(rules[rules_key])
-
-                else:
-                    dotty_info[info_key] = rules[rules_key]
-
-        except Exception as e:
-            _log_warning(info_data, f'{rules_key}->{info_key}: {e}')
-
-    info_data.update(dotty_info)
-
-    # Merge in config values that can't be easily mapped
-    _extract_features(info_data, rules)
-
-    return info_data
-
-
-def _merge_layouts(info_data, new_info_data):
-    """Merge new_info_data into info_data in an intelligent way.
-    """
-    for layout_name, layout_json in new_info_data['layouts'].items():
-        if layout_name in info_data['layouts']:
-            # Pull in layouts we have a macro for
-            if len(info_data['layouts'][layout_name]['layout']) != len(layout_json['layout']):
-                msg = '%s: %s: Number of elements in info.json does not match! info.json:%s != %s:%s'
-                _log_error(info_data, msg % (info_data['keyboard_folder'], layout_name, len(layout_json['layout']), layout_name, len(info_data['layouts'][layout_name]['layout'])))
-            else:
-                for i, key in enumerate(info_data['layouts'][layout_name]['layout']):
-                    key.update(layout_json['layout'][i])
-        else:
-            # Pull in layouts that have matrix data
-            missing_matrix = False
-            for key in layout_json.get('layout', {}):
-                if 'matrix' not in key:
-                    missing_matrix = True
-
-            if not missing_matrix:
-                if layout_name in info_data['layouts']:
-                    # Update an existing layout with new data
-                    for i, key in enumerate(info_data['layouts'][layout_name]['layout']):
-                        key.update(layout_json['layout'][i])
-
-                else:
-                    # Copy in the new layout wholesale
-                    layout_json['c_macro'] = False
-                    info_data['layouts'][layout_name] = layout_json
-
-    return info_data
-
-
-def _search_keyboard_h(path):
-    current_path = Path('keyboards/')
-    aliases = {}
-    layouts = {}
-
-    for directory in path.parts:
-        current_path = current_path / directory
-        keyboard_h = '%s.h' % (directory,)
-        keyboard_h_path = current_path / keyboard_h
-        if keyboard_h_path.exists():
-            new_layouts, new_aliases = find_layouts(keyboard_h_path)
-            layouts.update(new_layouts)
-
-            for alias, alias_text in new_aliases.items():
-                if alias_text in layouts:
-                    aliases[alias] = alias_text
-
-    return layouts, aliases
-
-
-def _find_all_layouts(info_data, keyboard):
-    """Looks for layout macros associated with this keyboard.
-    """
-    layouts, aliases = _search_keyboard_h(Path(keyboard))
-
-    if not layouts:
-        # If we don't find any layouts from info.json or keyboard.h we widen our search. This is error prone which is why we want to encourage people to follow the standard above.
-        info_data['parse_warnings'].append('%s: Falling back to searching for KEYMAP/LAYOUT macros.' % (keyboard))
-
-        for file in glob('keyboards/%s/*.h' % keyboard):
-            if file.endswith('.h'):
-                these_layouts, these_aliases = find_layouts(file)
-
-                if these_layouts:
-                    layouts.update(these_layouts)
-
-                for alias, alias_text in these_aliases.items():
-                    if alias_text in layouts:
-                        aliases[alias] = alias_text
-
-    return layouts, aliases
-
-
-def _log_error(info_data, message):
-    """Send an error message to both JSON and the log.
-    """
-    info_data['parse_errors'].append(message)
-    cli.log.error('%s: %s', info_data.get('keyboard_folder', 'Unknown Keyboard!'), message)
-
-
-def _log_warning(info_data, message):
-    """Send a warning message to both JSON and the log.
-    """
-    info_data['parse_warnings'].append(message)
-    cli.log.warning('%s: %s', info_data.get('keyboard_folder', 'Unknown Keyboard!'), message)
-
-
-def arm_processor_rules(info_data, rules):
-    """Setup the default info for an ARM board.
-    """
-    info_data['processor_type'] = 'arm'
-    info_data['protocol'] = 'ChibiOS'
-
-    if 'bootloader' not in info_data:
-        if 'STM32' in info_data['processor']:
-            info_data['bootloader'] = 'stm32-dfu'
-        else:
-            info_data['bootloader'] = 'unknown'
-
-    if 'STM32' in info_data['processor']:
-        info_data['platform'] = 'STM32'
-    elif 'MCU_SERIES' in rules:
-        info_data['platform'] = rules['MCU_SERIES']
-    elif 'ARM_ATSAM' in rules:
-        info_data['platform'] = 'ARM_ATSAM'
-
-    return info_data
-
-
-def avr_processor_rules(info_data, rules):
-    """Setup the default info for an AVR board.
-    """
-    info_data['processor_type'] = 'avr'
-    info_data['platform'] = rules['ARCH'] if 'ARCH' in rules else 'unknown'
-    info_data['protocol'] = 'V-USB' if rules.get('MCU') in VUSB_PROCESSORS else 'LUFA'
-
-    if 'bootloader' not in info_data:
-        info_data['bootloader'] = 'atmel-dfu'
-
-    # FIXME(fauxpark/anyone): Eventually we should detect the protocol by looking at PROTOCOL inherited from mcu_selection.mk:
-    # info_data['protocol'] = 'V-USB' if rules.get('PROTOCOL') == 'VUSB' else 'LUFA'
-
-    return info_data
-
-
-def unknown_processor_rules(info_data, rules):
-    """Setup the default keyboard info for unknown boards.
-    """
-    info_data['bootloader'] = 'unknown'
-    info_data['platform'] = 'unknown'
-    info_data['processor'] = 'unknown'
-    info_data['processor_type'] = 'unknown'
-    info_data['protocol'] = 'unknown'
-
-    return info_data
-
-
-def merge_info_jsons(keyboard, info_data):
-    """Return a merged copy of all the info.json files for a keyboard.
-    """
-    for info_file in find_info_json(keyboard):
-        # Load and validate the JSON data
-        new_info_data = json_load(info_file)
-
-        if not isinstance(new_info_data, dict):
-            _log_error(info_data, "Invalid file %s, root object should be a dictionary." % (str(info_file),))
-            continue
-
-        try:
-            keyboard_validate(new_info_data)
-        except jsonschema.ValidationError as e:
-            json_path = '.'.join([str(p) for p in e.absolute_path])
-            cli.log.error('Not including data from file: %s', info_file)
-            cli.log.error('\t%s: %s', json_path, e.message)
-            continue
-
-        # Merge layout data in
-        if 'layout_aliases' in new_info_data:
-            info_data['layout_aliases'] = {**info_data.get('layout_aliases', {}), **new_info_data['layout_aliases']}
-            del new_info_data['layout_aliases']
-
-        for layout_name, layout in new_info_data.get('layouts', {}).items():
-            if layout_name in info_data.get('layout_aliases', {}):
-                _log_warning(info_data, f"info.json uses alias name {layout_name} instead of {info_data['layout_aliases'][layout_name]}")
-                layout_name = info_data['layout_aliases'][layout_name]
-
-            if layout_name in info_data['layouts']:
-                for new_key, existing_key in zip(layout['layout'], info_data['layouts'][layout_name]['layout']):
-                    existing_key.update(new_key)
-            else:
-                layout['c_macro'] = False
-                info_data['layouts'][layout_name] = layout
-
-        # Update info_data with the new data
-        if 'layouts' in new_info_data:
-            del new_info_data['layouts']
-
-        deep_update(info_data, new_info_data)
-
-    return info_data
-
-
-def find_info_json(keyboard):
-    """Finds all the info.json files associated with a keyboard.
-    """
-    # Find the most specific first
-    base_path = Path('keyboards')
-    keyboard_path = base_path / keyboard
-    keyboard_parent = keyboard_path.parent
-    info_jsons = [keyboard_path / 'info.json']
-
-    # Add DEFAULT_FOLDER before parents, if present
-    rules = rules_mk(keyboard)
-    if 'DEFAULT_FOLDER' in rules:
-        info_jsons.append(Path(rules['DEFAULT_FOLDER']) / 'info.json')
-
-    # Add in parent folders for least specific
-    for _ in range(5):
-        info_jsons.append(keyboard_parent / 'info.json')
-        if keyboard_parent.parent == base_path:
-            break
-        keyboard_parent = keyboard_parent.parent
-
-    # Return a list of the info.json files that actually exist
-    return [info_json for info_json in info_jsons if info_json.exists()]

lib/python/qmk/keyboard.py

@@ -64,6 +64,17 @@ def find_readme(keyboard):
     return cur_dir / 'readme.md'
 
 
+def is_keyboard_target(keyboard_target):
+    """Checks to make sure the supplied keyboard_target is valid.
+
+    This is mainly used by commands that accept --keyboard.
+    """
+    if keyboard_target in ['all', 'all-avr', 'all-chibios', 'all-arm_atsam']:
+        return keyboard_target
+
+    return keyboard_folder(keyboard_target)
+
+
 def keyboard_folder(keyboard):
     """Returns the actual keyboard folder.
 

lib/python/qmk/keymap.py

@@ -14,6 +14,7 @@ from pygments import lex
 import qmk.path
 from qmk.keyboard import find_keyboard_from_dir, rules_mk
 from qmk.errors import CppError
+from qmk.metadata import basic_info_json
 
 # The `keymap.c` template to use when a keyboard doesn't have its own
 DEFAULT_KEYMAP_C = """#include QMK_KEYBOARD_H
@@ -327,36 +328,33 @@ def list_keymaps(keyboard, c=True, json=True, additional_files=None, fullpath=Fa
     Returns:
         a sorted list of valid keymap names.
     """
-    # parse all the rules.mk files for the keyboard
+    info_data = basic_info_json(keyboard)
-    rules = rules_mk(keyboard)
     names = set()
 
-    if rules:
+    keyboards_dir = Path('keyboards')
-        keyboards_dir = Path('keyboards')
+    kb_path = keyboards_dir / info_data['keyboard_folder']
-        kb_path = keyboards_dir / keyboard
 
-        # walk up the directory tree until keyboards_dir
+    # walk up the directory tree until keyboards_dir
-        # and collect all directories' name with keymap.c file in it
+    # and collect all directories' name with keymap.c file in it
-        while kb_path != keyboards_dir:
+    while kb_path != keyboards_dir:
-            keymaps_dir = kb_path / "keymaps"
+        keymaps_dir = kb_path / "keymaps"
 
-            if keymaps_dir.is_dir():
+        if keymaps_dir.is_dir():
-                for keymap in keymaps_dir.iterdir():
+            for keymap in keymaps_dir.iterdir():
-                    if is_keymap_dir(keymap, c, json, additional_files):
+                if is_keymap_dir(keymap, c, json, additional_files):
-                        keymap = keymap if fullpath else keymap.name
+                    keymap = keymap if fullpath else keymap.name
-                        names.add(keymap)
+                    names.add(keymap)
 
-            kb_path = kb_path.parent
+        kb_path = kb_path.parent
 
-        # if community layouts are supported, get them
+    # if community layouts are supported, get them
-        if "LAYOUTS" in rules:
+    for layout in info_data.get('community_layouts', []):
-            for layout in rules["LAYOUTS"].split():
+        cl_path = Path('layouts/community') / layout
-                cl_path = Path('layouts/community') / layout
+        if cl_path.is_dir():
-                if cl_path.is_dir():
+            for keymap in cl_path.iterdir():
-                    for keymap in cl_path.iterdir():
+                if is_keymap_dir(keymap, c, json, additional_files):
-                        if is_keymap_dir(keymap, c, json, additional_files):
+                    keymap = keymap if fullpath else keymap.name
-                            keymap = keymap if fullpath else keymap.name
+                    names.add(keymap)
-                            names.add(keymap)
 
     return sorted(names)
 

lib/python/qmk/metadata.py

@@ -0,0 +1,495 @@
+"""Functions that help us generate and use info.json files.
+"""
+from functools import lru_cache
+from glob import glob
+from pathlib import Path
+
+import jsonschema
+from dotty_dict import dotty
+from milc import cli
+
+from qmk.constants import CHIBIOS_PROCESSORS, LUFA_PROCESSORS, VUSB_PROCESSORS
+from qmk.c_parse import find_layouts
+from qmk.json_schema import deep_update, json_load, keyboard_validate
+from qmk.keyboard import config_h, rules_mk
+from qmk.makefile import parse_rules_mk_file
+from qmk.math import compute
+
+true_values = ['1', 'on', 'yes', 'true']
+false_values = ['0', 'off', 'no', 'false']
+
+
+@lru_cache(maxsize=None)
+def basic_info_json(keyboard):
+    """Generate a subset of info.json for a specific keyboard.
+
+    This does no validation, and should only be used as needed to avoid loops or when performance is critical.
+    """
+    cur_dir = Path('keyboards')
+    rules = parse_rules_mk_file(cur_dir / keyboard / 'rules.mk')
+
+    if 'DEFAULT_FOLDER' in rules:
+        keyboard = rules['DEFAULT_FOLDER']
+        rules = parse_rules_mk_file(cur_dir / keyboard / 'rules.mk', rules)
+
+    info_data = {
+        'keyboard_name': str(keyboard),
+        'keyboard_folder': str(keyboard),
+        'keymaps': {},
+        'layouts': {},
+        'parse_errors': [],
+        'parse_warnings': [],
+        'maintainer': 'qmk',
+    }
+
+    # Populate layout data
+    layouts, aliases = _find_all_layouts(info_data, keyboard)
+
+    if aliases:
+        info_data['layout_aliases'] = aliases
+
+    for layout_name, layout_json in layouts.items():
+        if not layout_name.startswith('LAYOUT_kc'):
+            layout_json['c_macro'] = True
+            info_data['layouts'][layout_name] = layout_json
+
+    # Merge in the data from info.json, config.h, and rules.mk
+    info_data = merge_info_jsons(keyboard, info_data)
+    info_data = _extract_config_h(info_data)
+    info_data = _extract_rules_mk(info_data)
+
+    return info_data
+
+
+def _extract_features(info_data, rules):
+    """Find all the features enabled in rules.mk.
+    """
+    # Special handling for bootmagic which also supports a "lite" mode.
+    if rules.get('BOOTMAGIC_ENABLE') == 'lite':
+        rules['BOOTMAGIC_LITE_ENABLE'] = 'on'
+        del rules['BOOTMAGIC_ENABLE']
+    if rules.get('BOOTMAGIC_ENABLE') == 'full':
+        rules['BOOTMAGIC_ENABLE'] = 'on'
+
+    # Skip non-boolean features we haven't implemented special handling for
+    for feature in 'HAPTIC_ENABLE', 'QWIIC_ENABLE':
+        if rules.get(feature):
+            del rules[feature]
+
+    # Process the rest of the rules as booleans
+    for key, value in rules.items():
+        if key.endswith('_ENABLE'):
+            key = '_'.join(key.split('_')[:-1]).lower()
+            value = True if value.lower() in true_values else False if value.lower() in false_values else value
+
+            if 'config_h_features' not in info_data:
+                info_data['config_h_features'] = {}
+
+            if 'features' not in info_data:
+                info_data['features'] = {}
+
+            if key in info_data['features']:
+                info_log_warning(info_data, 'Feature %s is specified in both info.json and rules.mk, the rules.mk value wins.' % (key,))
+
+            info_data['features'][key] = value
+            info_data['config_h_features'][key] = value
+
+    return info_data
+
+
+def _pin_name(pin):
+    """Returns the proper representation for a pin.
+    """
+    pin = pin.strip()
+
+    if not pin:
+        return None
+
+    elif pin.isdigit():
+        return int(pin)
+
+    elif pin == 'NO_PIN':
+        return None
+
+    elif pin[0] in 'ABCDEFGHIJK' and pin[1].isdigit():
+        return pin
+
+    raise ValueError(f'Invalid pin: {pin}')
+
+
+def _extract_pins(pins):
+    """Returns a list of pins from a comma separated string of pins.
+    """
+    return [_pin_name(pin) for pin in pins.split(',')]
+
+
+def _extract_direct_matrix(info_data, direct_pins):
+    """
+    """
+    info_data['matrix_pins'] = {}
+    direct_pin_array = []
+
+    while direct_pins[-1] != '}':
+        direct_pins = direct_pins[:-1]
+
+    for row in direct_pins.split('},{'):
+        if row.startswith('{'):
+            row = row[1:]
+
+        if row.endswith('}'):
+            row = row[:-1]
+
+        direct_pin_array.append([])
+
+        for pin in row.split(','):
+            if pin == 'NO_PIN':
+                pin = None
+
+            direct_pin_array[-1].append(pin)
+
+    return direct_pin_array
+
+
+def _extract_matrix_info(info_data, config_c):
+    """Populate the matrix information.
+    """
+    row_pins = config_c.get('MATRIX_ROW_PINS', '').replace('{', '').replace('}', '').strip()
+    col_pins = config_c.get('MATRIX_COL_PINS', '').replace('{', '').replace('}', '').strip()
+    direct_pins = config_c.get('DIRECT_PINS', '').replace(' ', '')[1:-1]
+
+    if 'MATRIX_ROWS' in config_c and 'MATRIX_COLS' in config_c:
+        if 'matrix_size' in info_data:
+            info_log_warning(info_data, 'Matrix size is specified in both info.json and config.h, the config.h values win.')
+
+        info_data['matrix_size'] = {
+            'cols': compute(config_c.get('MATRIX_COLS', '0')),
+            'rows': compute(config_c.get('MATRIX_ROWS', '0')),
+        }
+
+    if row_pins and col_pins:
+        if 'matrix_pins' in info_data:
+            info_log_warning(info_data, 'Matrix pins are specified in both info.json and config.h, the config.h values win.')
+
+        info_data['matrix_pins'] = {
+            'cols': _extract_pins(col_pins),
+            'rows': _extract_pins(row_pins),
+        }
+
+    if direct_pins:
+        if 'matrix_pins' in info_data:
+            info_log_warning(info_data, 'Direct pins are specified in both info.json and config.h, the config.h values win.')
+
+        info_data['matrix_pins']['direct'] = _extract_direct_matrix(info_data, direct_pins)
+
+    return info_data
+
+
+def _extract_config_h(info_data):
+    """Pull some keyboard information from existing config.h files
+    """
+    config_c = config_h(info_data['keyboard_folder'])
+
+    # Pull in data from the json map
+    dotty_info = dotty(info_data)
+    info_config_map = json_load(Path('data/mappings/info_config.json'))
+
+    for config_key, info_dict in info_config_map.items():
+        info_key = info_dict['info_key']
+        key_type = info_dict.get('value_type', 'str')
+
+        try:
+            if config_key in config_c and info_dict.get('to_json', True):
+                if dotty_info.get(info_key) and info_dict.get('warn_duplicate', True):
+                    info_log_warning(info_data, '%s in config.h is overwriting %s in info.json' % (config_key, info_key))
+
+                if key_type.startswith('array'):
+                    if '.' in key_type:
+                        key_type, array_type = key_type.split('.', 1)
+                    else:
+                        array_type = None
+
+                    config_value = config_c[config_key].replace('{', '').replace('}', '').strip()
+
+                    if array_type == 'int':
+                        dotty_info[info_key] = list(map(int, config_value.split(',')))
+                    else:
+                        dotty_info[info_key] = config_value.split(',')
+
+                elif key_type == 'bool':
+                    dotty_info[info_key] = config_c[config_key] in true_values
+
+                elif key_type == 'hex':
+                    dotty_info[info_key] = '0x' + config_c[config_key][2:].upper()
+
+                elif key_type == 'list':
+                    dotty_info[info_key] = config_c[config_key].split()
+
+                elif key_type == 'int':
+                    dotty_info[info_key] = int(config_c[config_key])
+
+                else:
+                    dotty_info[info_key] = config_c[config_key]
+
+        except Exception as e:
+            info_log_warning(info_data, f'{config_key}->{info_key}: {e}')
+
+    info_data.update(dotty_info)
+
+    # Pull data that easily can't be mapped in json
+    _extract_matrix_info(info_data, config_c)
+
+    return info_data
+
+
+def _extract_rules_mk(info_data):
+    """Pull some keyboard information from existing rules.mk files
+    """
+    rules = rules_mk(info_data['keyboard_folder'])
+    info_data['processor'] = rules.get('MCU', info_data.get('processor', 'atmega32u4'))
+
+    if info_data['processor'] in CHIBIOS_PROCESSORS:
+        arm_processor_rules(info_data, rules)
+
+    elif info_data['processor'] in LUFA_PROCESSORS + VUSB_PROCESSORS:
+        avr_processor_rules(info_data, rules)
+
+    else:
+        cli.log.warning("%s: Unknown MCU: %s" % (info_data['keyboard_folder'], info_data['processor']))
+        unknown_processor_rules(info_data, rules)
+
+    # Pull in data from the json map
+    dotty_info = dotty(info_data)
+    info_rules_map = json_load(Path('data/mappings/info_rules.json'))
+
+    for rules_key, info_dict in info_rules_map.items():
+        info_key = info_dict['info_key']
+        key_type = info_dict.get('value_type', 'str')
+
+        try:
+            if rules_key in rules and info_dict.get('to_json', True):
+                if dotty_info.get(info_key) and info_dict.get('warn_duplicate', True):
+                    info_log_warning(info_data, '%s in rules.mk is overwriting %s in info.json' % (rules_key, info_key))
+
+                if key_type.startswith('array'):
+                    if '.' in key_type:
+                        key_type, array_type = key_type.split('.', 1)
+                    else:
+                        array_type = None
+
+                    rules_value = rules[rules_key].replace('{', '').replace('}', '').strip()
+
+                    if array_type == 'int':
+                        dotty_info[info_key] = list(map(int, rules_value.split(',')))
+                    else:
+                        dotty_info[info_key] = rules_value.split(',')
+
+                elif key_type == 'list':
+                    dotty_info[info_key] = rules[rules_key].split()
+
+                elif key_type == 'bool':
+                    dotty_info[info_key] = rules[rules_key] in true_values
+
+                elif key_type == 'hex':
+                    dotty_info[info_key] = '0x' + rules[rules_key][2:].upper()
+
+                elif key_type == 'int':
+                    dotty_info[info_key] = int(rules[rules_key])
+
+                else:
+                    dotty_info[info_key] = rules[rules_key]
+
+        except Exception as e:
+            info_log_warning(info_data, f'{rules_key}->{info_key}: {e}')
+
+    info_data.update(dotty_info)
+
+    # Merge in config values that can't be easily mapped
+    _extract_features(info_data, rules)
+
+    return info_data
+
+
+def _search_keyboard_h(path):
+    current_path = Path('keyboards/')
+    aliases = {}
+    layouts = {}
+
+    for directory in path.parts:
+        current_path = current_path / directory
+        keyboard_h = '%s.h' % (directory,)
+        keyboard_h_path = current_path / keyboard_h
+        if keyboard_h_path.exists():
+            new_layouts, new_aliases = find_layouts(keyboard_h_path)
+            layouts.update(new_layouts)
+
+            for alias, alias_text in new_aliases.items():
+                if alias_text in layouts:
+                    aliases[alias] = alias_text
+
+    return layouts, aliases
+
+
+def _find_all_layouts(info_data, keyboard):
+    """Looks for layout macros associated with this keyboard.
+    """
+    found_layouts = False
+    layouts, aliases = _search_keyboard_h(Path(keyboard))
+
+    if layouts:
+        found_layouts = True
+    else:
+        for layout in info_data['layouts'].values():
+            if 'matrix' in layout['layout']:
+                found_layouts = True
+                break
+
+    if not found_layouts:
+        # If we don't find any layouts from info.json or keyboard.h we widen our search. This is error prone which is why we want to encourage people to follow the standard.
+        info_data['parse_warnings'].append('%s: Falling back to searching for KEYMAP/LAYOUT macros.' % (keyboard))
+
+        for file in glob('keyboards/%s/*.h' % keyboard):
+            if file.endswith('.h'):
+                these_layouts, these_aliases = find_layouts(file)
+
+                if these_layouts:
+                    layouts.update(these_layouts)
+
+                for alias, alias_text in these_aliases.items():
+                    if alias_text in layouts:
+                        aliases[alias] = alias_text
+
+    return layouts, aliases
+
+
+def info_log_error(info_data, message):
+    """Send an error message to both JSON and the log.
+    """
+    info_data['parse_errors'].append(message)
+    cli.log.error('%s: %s', info_data.get('keyboard_folder', 'Unknown Keyboard!'), message)
+
+
+def info_log_warning(info_data, message):
+    """Send a warning message to both JSON and the log.
+    """
+    info_data['parse_warnings'].append(message)
+    cli.log.warning('%s: %s', info_data.get('keyboard_folder', 'Unknown Keyboard!'), message)
+
+
+def arm_processor_rules(info_data, rules):
+    """Setup the default info for an ARM board.
+    """
+    info_data['processor_type'] = 'arm'
+    info_data['protocol'] = 'ChibiOS'
+
+    if 'bootloader' not in info_data:
+        if 'STM32' in info_data['processor']:
+            info_data['bootloader'] = 'stm32-dfu'
+        else:
+            info_data['bootloader'] = 'unknown'
+
+    if 'STM32' in info_data['processor']:
+        info_data['platform'] = 'STM32'
+    elif 'MCU_SERIES' in rules:
+        info_data['platform'] = rules['MCU_SERIES']
+    elif 'ARM_ATSAM' in rules:
+        info_data['platform'] = 'ARM_ATSAM'
+
+    return info_data
+
+
+def avr_processor_rules(info_data, rules):
+    """Setup the default info for an AVR board.
+    """
+    info_data['processor_type'] = 'avr'
+    info_data['platform'] = rules['ARCH'] if 'ARCH' in rules else 'unknown'
+    info_data['protocol'] = 'V-USB' if rules.get('MCU') in VUSB_PROCESSORS else 'LUFA'
+
+    if 'bootloader' not in info_data:
+        info_data['bootloader'] = 'atmel-dfu'
+
+    # FIXME(fauxpark/anyone): Eventually we should detect the protocol by looking at PROTOCOL inherited from mcu_selection.mk:
+    # info_data['protocol'] = 'V-USB' if rules.get('PROTOCOL') == 'VUSB' else 'LUFA'
+
+    return info_data
+
+
+def unknown_processor_rules(info_data, rules):
+    """Setup the default keyboard info for unknown boards.
+    """
+    info_data['bootloader'] = 'unknown'
+    info_data['platform'] = 'unknown'
+    info_data['processor'] = 'unknown'
+    info_data['processor_type'] = 'unknown'
+    info_data['protocol'] = 'unknown'
+
+    return info_data
+
+
+def merge_info_jsons(keyboard, info_data):
+    """Return a merged copy of all the info.json files for a keyboard.
+    """
+    for info_file in find_info_json(keyboard):
+        # Load and validate the JSON data
+        new_info_data = json_load(info_file)
+
+        if not isinstance(new_info_data, dict):
+            info_log_error(info_data, "Invalid file %s, root object should be a dictionary." % (str(info_file),))
+            continue
+
+        try:
+            keyboard_validate(new_info_data)
+        except jsonschema.ValidationError as e:
+            json_path = '.'.join([str(p) for p in e.absolute_path])
+            cli.log.error('Not including data from file: %s', info_file)
+            cli.log.error('\t%s: %s', json_path, e.message)
+            continue
+
+        # Merge layout data in
+        if 'layout_aliases' in new_info_data:
+            info_data['layout_aliases'] = {**info_data.get('layout_aliases', {}), **new_info_data['layout_aliases']}
+            del new_info_data['layout_aliases']
+
+        for layout_name, layout in new_info_data.get('layouts', {}).items():
+            if layout_name in info_data.get('layout_aliases', {}):
+                info_log_warning(info_data, f"info.json uses alias name {layout_name} instead of {info_data['layout_aliases'][layout_name]}")
+                layout_name = info_data['layout_aliases'][layout_name]
+
+            if layout_name in info_data['layouts']:
+                for new_key, existing_key in zip(layout['layout'], info_data['layouts'][layout_name]['layout']):
+                    existing_key.update(new_key)
+            else:
+                layout['c_macro'] = False
+                info_data['layouts'][layout_name] = layout
+
+        # Update info_data with the new data
+        if 'layouts' in new_info_data:
+            del new_info_data['layouts']
+
+        deep_update(info_data, new_info_data)
+
+    return info_data
+
+
+def find_info_json(keyboard):
+    """Finds all the info.json files associated with a keyboard.
+    """
+    # Find the most specific first
+    base_path = Path('keyboards')
+    keyboard_path = base_path / keyboard
+    keyboard_parent = keyboard_path.parent
+    info_jsons = [keyboard_path / 'info.json']
+
+    # Add DEFAULT_FOLDER before parents, if present
+    rules = rules_mk(keyboard)
+    if 'DEFAULT_FOLDER' in rules:
+        info_jsons.append(Path(rules['DEFAULT_FOLDER']) / 'info.json')
+
+    # Add in parent folders for least specific
+    for _ in range(5):
+        info_jsons.append(keyboard_parent / 'info.json')
+        if keyboard_parent.parent == base_path:
+            break
+        keyboard_parent = keyboard_parent.parent
+
+    # Return a list of the info.json files that actually exist
+    return [info_json for info_json in info_jsons if info_json.exists()]