Rework qmk compile to bypass Makefile. Add new --filter option.
This commit is contained in:
Zach White
parent
596c4a1f87
commit
50fdb2a52c
9 changed files with 666 additions and 111 deletions
2
Makefile
2
Makefile
|
|
@ -398,7 +398,7 @@ define PARSE_KEYMAP
|
|||
# Specify the variables that we are passing forward to submake
|
||||
MAKE_VARS := KEYBOARD=$$(CURRENT_KB) KEYMAP=$$(CURRENT_KM) REQUIRE_PLATFORM_KEY=$$(REQUIRE_PLATFORM_KEY) QMK_BIN=$$(QMK_BIN)
|
||||
# And the first part of the make command
|
||||
MAKE_CMD := $$(MAKE) -r -R -C $(ROOT_DIR) -f build_keyboard.mk $$(MAKE_TARGET)
|
||||
MAKE_CMD := echo $$(MAKE) -r -R -C $(ROOT_DIR) -f build_keyboard.mk $$(MAKE_TARGET)
|
||||
# The message to display
|
||||
MAKE_MSG := $$(MSG_MAKE_KB)
|
||||
# We run the command differently, depending on if we want more output or not
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
"""
|
||||
from subprocess import DEVNULL
|
||||
|
||||
from qmk.commands import create_make_target
|
||||
from qmk.commands import _find_make
|
||||
from milc import cli
|
||||
|
||||
|
||||
|
|
@ -11,4 +11,6 @@ from milc import cli
|
|||
def clean(cli):
|
||||
"""Runs `make clean` (or `make distclean` if --all is passed)
|
||||
"""
|
||||
cli.run(create_make_target('distclean' if cli.args.all else 'clean'), capture_output=False, stdin=DEVNULL)
|
||||
make_cmd = [_find_make(), 'distclean' if cli.args.all else 'clean']
|
||||
|
||||
cli.run(make_cmd, capture_output=False, stdin=DEVNULL)
|
||||
|
|
|
|||
|
|
@ -5,20 +5,70 @@ You can compile a keymap already in the repo or using a QMK Configurator export.
|
|||
from subprocess import DEVNULL
|
||||
|
||||
from argcomplete.completers import FilesCompleter
|
||||
from dotty_dict import dotty
|
||||
from milc import cli
|
||||
|
||||
import qmk.path
|
||||
from qmk.decorators import automagic_keyboard, automagic_keymap
|
||||
from qmk.decorators import automagic_keyboard, automagic_keymap, lru_cache
|
||||
from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json
|
||||
from qmk.keyboard import keyboard_completer, keyboard_folder
|
||||
from qmk.keymap import keymap_completer
|
||||
from qmk.info import info_json
|
||||
from qmk.keyboard import keyboard_completer, is_keyboard_target, list_keyboards
|
||||
from qmk.keymap import keymap_completer, list_keymaps
|
||||
from qmk.metadata import true_values, false_values
|
||||
|
||||
|
||||
@lru_cache()
|
||||
def _keyboard_list():
|
||||
"""Returns a list of keyboards matching cli.config.compile.keyboard.
|
||||
"""
|
||||
if cli.config.compile.keyboard == 'all':
|
||||
return list_keyboards()
|
||||
|
||||
elif cli.config.compile.keyboard.startswith('all-'):
|
||||
return list_keyboards()
|
||||
|
||||
return [cli.config.compile.keyboard]
|
||||
|
||||
|
||||
def keyboard_keymap_iter():
|
||||
"""Iterates over the keyboard/keymap for this command and yields a pairing of each.
|
||||
"""
|
||||
for keyboard in _keyboard_list():
|
||||
continue_flag = False
|
||||
if cli.args.filter:
|
||||
info_data = dotty(info_json(keyboard))
|
||||
for filter in cli.args.filter:
|
||||
if '=' in filter:
|
||||
key, value = filter.split('=', 1)
|
||||
if value in true_values:
|
||||
value = True
|
||||
elif value in false_values:
|
||||
value = False
|
||||
elif value.isdigit():
|
||||
value = int(value)
|
||||
elif '.' in value and value.replace('.').isdigit():
|
||||
value = float(value)
|
||||
|
||||
if info_data.get(key) != value:
|
||||
continue_flag = True
|
||||
break
|
||||
|
||||
if continue_flag:
|
||||
continue
|
||||
|
||||
if cli.config.compile.keymap == 'all':
|
||||
for keymap in list_keymaps(keyboard):
|
||||
yield keyboard, keymap
|
||||
else:
|
||||
yield keyboard, cli.config.compile.keymap
|
||||
|
||||
|
||||
@cli.argument('filename', nargs='?', arg_only=True, type=qmk.path.FileType('r'), completer=FilesCompleter('.json'), help='The configurator export to compile')
|
||||
@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.')
|
||||
@cli.argument('-kb', '--keyboard', type=is_keyboard_target, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.')
|
||||
@cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.')
|
||||
@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.")
|
||||
@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.")
|
||||
@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs to run.")
|
||||
@cli.argument('-f', '--filter', arg_only=True, action='append', default=[], help="Filter your list against info.json.")
|
||||
@cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.")
|
||||
@cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.")
|
||||
@cli.subcommand('Compile a QMK Firmware.')
|
||||
|
|
@ -31,47 +81,79 @@ def compile(cli):
|
|||
|
||||
If a keyboard and keymap are provided this command will build a firmware based on that.
|
||||
"""
|
||||
if cli.args.clean and not cli.args.filename and not cli.args.dry_run:
|
||||
command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, 'clean')
|
||||
cli.run(command, capture_output=False, stdin=DEVNULL)
|
||||
envs = {'REQUIRE_PLATFORM_KEY': ''}
|
||||
silent = cli.config.compile.keyboard == 'all' or cli.config.compile.keyboard.startswith('all-') or cli.config.compile.keymap == 'all'
|
||||
|
||||
# Build the environment vars
|
||||
envs = {}
|
||||
# Setup the environment
|
||||
for env in cli.args.env:
|
||||
if '=' in env:
|
||||
key, value = env.split('=', 1)
|
||||
if key in envs:
|
||||
cli.log.warning('Overwriting existing environment variable %s=%s with %s=%s!', key, envs[key], key, value)
|
||||
envs[key] = value
|
||||
else:
|
||||
cli.log.warning('Invalid environment variable: %s', env)
|
||||
|
||||
# Determine the compile command
|
||||
command = None
|
||||
if cli.config.compile.keyboard.startswith('all-'):
|
||||
envs['REQUIRE_PLATFORM_KEY'] = cli.config.compile.keyboard[4:]
|
||||
|
||||
# Run clean if necessary
|
||||
if cli.args.clean and not cli.args.filename and not cli.args.dry_run:
|
||||
for keyboard, keymap in keyboard_keymap_iter():
|
||||
cli.log.info('Cleaning previous build files for keyboard {fg_cyan}%s{fg_reset} keymap {fg_cyan}%s', keyboard, keymap)
|
||||
_, _, make_cmd = create_make_command(keyboard, keymap, 'clean', cli.config.compile.parallel, silent, **envs)
|
||||
cli.run(make_cmd, capture_output=False, stdin=DEVNULL)
|
||||
|
||||
# Determine the compile command(s)
|
||||
commands = None
|
||||
|
||||
if cli.args.filename:
|
||||
# If a configurator JSON was provided generate a keymap and compile it
|
||||
user_keymap = parse_configurator_json(cli.args.filename)
|
||||
command = compile_configurator_json(user_keymap, parallel=cli.config.compile.parallel, **envs)
|
||||
commands = [compile_configurator_json(user_keymap, parallel=cli.config.compile.parallel, **envs)]
|
||||
|
||||
else:
|
||||
if cli.config.compile.keyboard and cli.config.compile.keymap:
|
||||
# Generate the make command for a specific keyboard/keymap.
|
||||
command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, parallel=cli.config.compile.parallel, **envs)
|
||||
elif cli.config.compile.keyboard and cli.config.compile.keymap:
|
||||
if cli.args.filter:
|
||||
cli.log.info('Generating the list of keyboards to compile, this may take some time.')
|
||||
|
||||
elif not cli.config.compile.keyboard:
|
||||
cli.log.error('Could not determine keyboard!')
|
||||
elif not cli.config.compile.keymap:
|
||||
cli.log.error('Could not determine keymap!')
|
||||
commands = [create_make_command(keyboard, keymap, parallel=cli.config.compile.parallel, silent=silent, **envs) for keyboard, keymap in keyboard_keymap_iter()]
|
||||
|
||||
elif not cli.config.compile.keyboard:
|
||||
cli.log.error('Could not determine keyboard!')
|
||||
|
||||
elif not cli.config.compile.keymap:
|
||||
cli.log.error('Could not determine keymap!')
|
||||
|
||||
# Compile the firmware, if we're able to
|
||||
if command:
|
||||
cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command))
|
||||
if not cli.args.dry_run:
|
||||
cli.echo('\n')
|
||||
# FIXME(skullydazed/anyone): Remove text=False once milc 1.0.11 has had enough time to be installed everywhere.
|
||||
compile = cli.run(command, capture_output=False, text=False)
|
||||
return compile.returncode
|
||||
if commands:
|
||||
returncodes = []
|
||||
for keyboard, keymap, command in commands:
|
||||
print()
|
||||
cli.log.info('Building firmware for {fg_cyan}%s{fg_reset} with keymap {fg_cyan}%s', keyboard, keymap)
|
||||
cli.log.debug('Running make command: {fg_blue}%s', ' '.join(command))
|
||||
|
||||
if not cli.args.dry_run:
|
||||
compile = cli.run(command, capture_output=False)
|
||||
returncodes.append(compile.returncode)
|
||||
if compile.returncode == 0:
|
||||
cli.log.info('Success!')
|
||||
else:
|
||||
cli.log.error('Failed!')
|
||||
|
||||
if any(returncodes):
|
||||
print()
|
||||
cli.log.error('Could not compile all targets, look above this message for more details. Failing target(s):')
|
||||
|
||||
for i, returncode in enumerate(returncodes):
|
||||
if returncode != 0:
|
||||
keyboard, keymap, command = commands[i]
|
||||
cli.echo('\tkeyboard: {fg_cyan}%s{fg_reset} keymap: {fg_cyan}%s', keyboard, keymap)
|
||||
|
||||
elif cli.args.filter:
|
||||
cli.log.error('No keyboards found after applying filter(s)!')
|
||||
return False
|
||||
|
||||
else:
|
||||
cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')
|
||||
cli.echo('usage: qmk compile [-h] [-b] [-kb KEYBOARD] [-km KEYMAP] [filename]')
|
||||
cli.print_help()
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from milc import cli
|
|||
import qmk.path
|
||||
from qmk.decorators import automagic_keyboard, automagic_keymap
|
||||
from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json
|
||||
from qmk.keyboard import keyboard_completer, keyboard_folder
|
||||
from qmk.keyboard import keyboard_completer, is_keyboard_target
|
||||
|
||||
|
||||
def print_bootloader_help():
|
||||
|
|
@ -36,7 +36,7 @@ def print_bootloader_help():
|
|||
@cli.argument('-b', '--bootloaders', action='store_true', help='List the available bootloaders.')
|
||||
@cli.argument('-bl', '--bootloader', default='flash', help='The flash command, corresponding to qmk\'s make options of bootloaders.')
|
||||
@cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.')
|
||||
@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.')
|
||||
@cli.argument('-kb', '--keyboard', type=is_keyboard_target, completer=keyboard_completer, help='The keyboard to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.')
|
||||
@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.")
|
||||
@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.")
|
||||
@cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.")
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@ def create_make_target(target, parallel=1, **env_vars):
|
|||
return [make_cmd, *get_make_parallel_args(parallel), *env, target]
|
||||
|
||||
|
||||
def create_make_command(keyboard, keymap, target=None, parallel=1, **env_vars):
|
||||
def create_make_command(keyboard, keymap, target=None, parallel=1, silent=False, **env_vars):
|
||||
"""Create a make compile command
|
||||
|
||||
Args:
|
||||
|
|
@ -79,12 +79,26 @@ def create_make_command(keyboard, keymap, target=None, parallel=1, **env_vars):
|
|||
|
||||
A command that can be run to make the specified keyboard and keymap
|
||||
"""
|
||||
make_args = [keyboard, keymap]
|
||||
make_cmd = [_find_make(), '--no-print-directory', '-r', '-R', '-C', './', '-f', 'build_keyboard.mk']
|
||||
|
||||
env_vars['KEYBOARD'] = keyboard
|
||||
env_vars['KEYMAP'] = keymap
|
||||
env_vars['QMK_BIN'] = 'bin/qmk' if 'DEPRECATED_BIN_QMK' in os.environ else 'qmk'
|
||||
env_vars['VERBOSE'] = 'true' if cli.config.general.verbose else ''
|
||||
env_vars['SILENT'] = 'true' if silent else 'false'
|
||||
env_vars['COLOR'] = 'true' if cli.config.general.color else ''
|
||||
|
||||
if parallel > 1:
|
||||
make_cmd.append('-j')
|
||||
make_cmd.append(parallel)
|
||||
|
||||
if target:
|
||||
make_args.append(target)
|
||||
make_cmd.append(target)
|
||||
|
||||
return create_make_target(':'.join(make_args), parallel, **env_vars)
|
||||
for key, value in env_vars.items():
|
||||
make_cmd.append(f'{key}={value}')
|
||||
|
||||
return keyboard, keymap, make_cmd
|
||||
|
||||
|
||||
def get_git_version(current_time, repo_dir='.', check_dir='.'):
|
||||
|
|
@ -236,7 +250,7 @@ def compile_configurator_json(user_keymap, bootloader=None, parallel=1, **env_va
|
|||
'QMK_BIN="qmk"',
|
||||
])
|
||||
|
||||
return make_command
|
||||
return user_keymap['keyboard'], user_keymap['keymap'], make_command
|
||||
|
||||
|
||||
def parse_configurator_json(configurator_file):
|
||||
|
|
|
|||
|
|
@ -1,22 +1,15 @@
|
|||
"""Functions that help us generate and use info.json files.
|
||||
"""
|
||||
from glob import glob
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
|
||||
import jsonschema
|
||||
from dotty_dict import dotty
|
||||
from milc import cli
|
||||
|
||||
from qmk.constants import CHIBIOS_PROCESSORS, LUFA_PROCESSORS, VUSB_PROCESSORS
|
||||
from qmk.c_parse import find_layouts
|
||||
from qmk.json_schema import deep_update, json_load, validate
|
||||
from qmk.keyboard import config_h, rules_mk
|
||||
from qmk.json_schema import validate
|
||||
from qmk.keymap import list_keymaps
|
||||
from qmk.makefile import parse_rules_mk_file
|
||||
from qmk.math import compute
|
||||
|
||||
true_values = ['1', 'on', 'yes']
|
||||
false_values = ['0', 'off', 'no']
|
||||
from qmk.metadata import basic_info_json, info_log_error
|
||||
|
||||
|
||||
def _valid_community_layout(layout):
|
||||
|
|
@ -25,24 +18,11 @@ def _valid_community_layout(layout):
|
|||
return (Path('layouts/default') / layout).exists()
|
||||
|
||||
|
||||
@lru_cache(maxsize=None)
|
||||
def info_json(keyboard):
|
||||
"""Generate the info.json data for a specific keyboard.
|
||||
"""
|
||||
cur_dir = Path('keyboards')
|
||||
rules = parse_rules_mk_file(cur_dir / keyboard / 'rules.mk')
|
||||
if 'DEFAULT_FOLDER' in rules:
|
||||
keyboard = rules['DEFAULT_FOLDER']
|
||||
rules = parse_rules_mk_file(cur_dir / keyboard / 'rules.mk', rules)
|
||||
|
||||
info_data = {
|
||||
'keyboard_name': str(keyboard),
|
||||
'keyboard_folder': str(keyboard),
|
||||
'keymaps': {},
|
||||
'layouts': {},
|
||||
'parse_errors': [],
|
||||
'parse_warnings': [],
|
||||
'maintainer': 'qmk',
|
||||
}
|
||||
info_data = basic_info_json(keyboard)
|
||||
|
||||
# Populate the list of JSON keymaps
|
||||
for keymap in list_keymaps(keyboard, c=False, fullpath=True):
|
||||
|
|
@ -81,20 +61,20 @@ def info_json(keyboard):
|
|||
_find_missing_layouts(info_data, keyboard)
|
||||
|
||||
if not info_data.get('layouts'):
|
||||
_log_error(info_data, 'No LAYOUTs defined! Need at least one layout defined in the keyboard.h or info.json.')
|
||||
info_log_error(info_data, 'No LAYOUTs defined! Need at least one layout defined in the keyboard.h or info.json.')
|
||||
|
||||
# Filter out any non-existing community layouts
|
||||
for layout in info_data.get('community_layouts', []):
|
||||
if not _valid_community_layout(layout):
|
||||
# Ignore layout from future checks
|
||||
info_data['community_layouts'].remove(layout)
|
||||
_log_error(info_data, 'Claims to support a community layout that does not exist: %s' % (layout))
|
||||
info_log_error(info_data, 'Claims to support a community layout that does not exist: %s' % (layout))
|
||||
|
||||
# Make sure we supply layout macros for the community layouts we claim to support
|
||||
for layout in info_data.get('community_layouts', []):
|
||||
layout_name = 'LAYOUT_' + layout
|
||||
if layout_name not in info_data.get('layouts', {}) and layout_name not in info_data.get('layout_aliases', {}):
|
||||
_log_error(info_data, 'Claims to support community layout %s but no %s() macro found' % (layout, layout_name))
|
||||
info_log_error(info_data, 'Claims to support community layout %s but no %s() macro found' % (layout, layout_name))
|
||||
|
||||
# Check that the reported matrix size is consistent with the actual matrix size
|
||||
_check_matrix(info_data)
|
||||
|
|
@ -544,11 +524,11 @@ def _check_matrix(info_data):
|
|||
|
||||
if col_count != actual_col_count and col_count != (actual_col_count / 2):
|
||||
# FIXME: once we can we should detect if split is enabled to do the actual_col_count/2 check.
|
||||
_log_error(info_data, f'MATRIX_COLS is inconsistent with the size of MATRIX_COL_PINS: {col_count} != {actual_col_count}')
|
||||
info_log_error(info_data, f'MATRIX_COLS is inconsistent with the size of MATRIX_COL_PINS: {col_count} != {actual_col_count}')
|
||||
|
||||
if row_count != actual_row_count and row_count != (actual_row_count / 2):
|
||||
# FIXME: once we can we should detect if split is enabled to do the actual_row_count/2 check.
|
||||
_log_error(info_data, f'MATRIX_ROWS is inconsistent with the size of MATRIX_ROW_PINS: {row_count} != {actual_row_count}')
|
||||
info_log_error(info_data, f'MATRIX_ROWS is inconsistent with the size of MATRIX_ROW_PINS: {row_count} != {actual_row_count}')
|
||||
|
||||
|
||||
def _search_keyboard_h(keyboard):
|
||||
|
|
@ -596,20 +576,6 @@ def _find_missing_layouts(info_data, keyboard):
|
|||
info_data['layout_aliases'][alias] = alias_text
|
||||
|
||||
|
||||
def _log_error(info_data, message):
|
||||
"""Send an error message to both JSON and the log.
|
||||
"""
|
||||
info_data['parse_errors'].append(message)
|
||||
cli.log.error('%s: %s', info_data.get('keyboard_folder', 'Unknown Keyboard!'), message)
|
||||
|
||||
|
||||
def _log_warning(info_data, message):
|
||||
"""Send a warning message to both JSON and the log.
|
||||
"""
|
||||
info_data['parse_warnings'].append(message)
|
||||
cli.log.warning('%s: %s', info_data.get('keyboard_folder', 'Unknown Keyboard!'), message)
|
||||
|
||||
|
||||
def arm_processor_rules(info_data, rules):
|
||||
"""Setup the default info for an ARM board.
|
||||
"""
|
||||
|
|
@ -668,7 +634,7 @@ def merge_info_jsons(keyboard, info_data):
|
|||
new_info_data = json_load(info_file)
|
||||
|
||||
if not isinstance(new_info_data, dict):
|
||||
_log_error(info_data, "Invalid file %s, root object should be a dictionary." % (str(info_file),))
|
||||
info_log_error(info_data, "Invalid file %s, root object should be a dictionary." % (str(info_file),))
|
||||
continue
|
||||
|
||||
try:
|
||||
|
|
@ -692,7 +658,7 @@ def merge_info_jsons(keyboard, info_data):
|
|||
if layout_name in info_data['layouts']:
|
||||
if len(info_data['layouts'][layout_name]['layout']) != len(layout['layout']):
|
||||
msg = '%s: %s: Number of elements in info.json does not match! info.json:%s != %s:%s'
|
||||
_log_error(info_data, msg % (info_data['keyboard_folder'], layout_name, len(layout['layout']), layout_name, len(info_data['layouts'][layout_name]['layout'])))
|
||||
info_log_error(info_data, msg % (info_data['keyboard_folder'], layout_name, len(layout['layout']), layout_name, len(info_data['layouts'][layout_name]['layout'])))
|
||||
else:
|
||||
for new_key, existing_key in zip(layout['layout'], info_data['layouts'][layout_name]['layout']):
|
||||
existing_key.update(new_key)
|
||||
|
|
@ -730,5 +696,4 @@ def find_info_json(keyboard):
|
|||
break
|
||||
keyboard_parent = keyboard_parent.parent
|
||||
|
||||
# Return a list of the info.json files that actually exist
|
||||
return [info_json for info_json in info_jsons if info_json.exists()]
|
||||
return info_data
|
||||
|
|
|
|||
|
|
@ -64,6 +64,17 @@ def find_readme(keyboard):
|
|||
return cur_dir / 'readme.md'
|
||||
|
||||
|
||||
def is_keyboard_target(keyboard_target):
|
||||
"""Checks to make sure the supplied keyboard_target is valid.
|
||||
|
||||
This is mainly used by commands that accept --keyboard.
|
||||
"""
|
||||
if keyboard_target in ['all', 'all-avr', 'all-chibios', 'all-arm_atsam']:
|
||||
return keyboard_target
|
||||
|
||||
return keyboard_folder(keyboard_target)
|
||||
|
||||
|
||||
def keyboard_folder(keyboard):
|
||||
"""Returns the actual keyboard folder.
|
||||
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ from pygments import lex
|
|||
import qmk.path
|
||||
from qmk.keyboard import find_keyboard_from_dir, rules_mk
|
||||
from qmk.errors import CppError
|
||||
from qmk.metadata import basic_info_json
|
||||
|
||||
# The `keymap.c` template to use when a keyboard doesn't have its own
|
||||
DEFAULT_KEYMAP_C = """#include QMK_KEYBOARD_H
|
||||
|
|
@ -327,36 +328,33 @@ def list_keymaps(keyboard, c=True, json=True, additional_files=None, fullpath=Fa
|
|||
Returns:
|
||||
a sorted list of valid keymap names.
|
||||
"""
|
||||
# parse all the rules.mk files for the keyboard
|
||||
rules = rules_mk(keyboard)
|
||||
info_data = basic_info_json(keyboard)
|
||||
names = set()
|
||||
|
||||
if rules:
|
||||
keyboards_dir = Path('keyboards')
|
||||
kb_path = keyboards_dir / keyboard
|
||||
keyboards_dir = Path('keyboards')
|
||||
kb_path = keyboards_dir / info_data['keyboard_folder']
|
||||
|
||||
# walk up the directory tree until keyboards_dir
|
||||
# and collect all directories' name with keymap.c file in it
|
||||
while kb_path != keyboards_dir:
|
||||
keymaps_dir = kb_path / "keymaps"
|
||||
# walk up the directory tree until keyboards_dir
|
||||
# and collect all directories' name with keymap.c file in it
|
||||
while kb_path != keyboards_dir:
|
||||
keymaps_dir = kb_path / "keymaps"
|
||||
|
||||
if keymaps_dir.is_dir():
|
||||
for keymap in keymaps_dir.iterdir():
|
||||
if is_keymap_dir(keymap, c, json, additional_files):
|
||||
keymap = keymap if fullpath else keymap.name
|
||||
names.add(keymap)
|
||||
if keymaps_dir.is_dir():
|
||||
for keymap in keymaps_dir.iterdir():
|
||||
if is_keymap_dir(keymap, c, json, additional_files):
|
||||
keymap = keymap if fullpath else keymap.name
|
||||
names.add(keymap)
|
||||
|
||||
kb_path = kb_path.parent
|
||||
kb_path = kb_path.parent
|
||||
|
||||
# if community layouts are supported, get them
|
||||
if "LAYOUTS" in rules:
|
||||
for layout in rules["LAYOUTS"].split():
|
||||
cl_path = Path('layouts/community') / layout
|
||||
if cl_path.is_dir():
|
||||
for keymap in cl_path.iterdir():
|
||||
if is_keymap_dir(keymap, c, json, additional_files):
|
||||
keymap = keymap if fullpath else keymap.name
|
||||
names.add(keymap)
|
||||
# if community layouts are supported, get them
|
||||
for layout in info_data.get('community_layouts', []):
|
||||
cl_path = Path('layouts/community') / layout
|
||||
if cl_path.is_dir():
|
||||
for keymap in cl_path.iterdir():
|
||||
if is_keymap_dir(keymap, c, json, additional_files):
|
||||
keymap = keymap if fullpath else keymap.name
|
||||
names.add(keymap)
|
||||
|
||||
return sorted(names)
|
||||
|
||||
|
|
|
|||
483
lib/python/qmk/metadata.py
Normal file
483
lib/python/qmk/metadata.py
Normal file
|
|
@ -0,0 +1,483 @@
|
|||
"""Functions that help us generate and use info.json files.
|
||||
"""
|
||||
from functools import lru_cache
|
||||
from glob import glob
|
||||
from pathlib import Path
|
||||
|
||||
import jsonschema
|
||||
from dotty_dict import dotty
|
||||
from milc import cli
|
||||
|
||||
from qmk.constants import CHIBIOS_PROCESSORS, LUFA_PROCESSORS, VUSB_PROCESSORS
|
||||
from qmk.c_parse import find_layouts
|
||||
from qmk.json_schema import deep_update, json_load, validate
|
||||
from qmk.keyboard import config_h, rules_mk
|
||||
from qmk.makefile import parse_rules_mk_file
|
||||
from qmk.math import compute
|
||||
|
||||
true_values = ['1', 'on', 'yes', 'true']
|
||||
false_values = ['0', 'off', 'no', 'false']
|
||||
|
||||
|
||||
@lru_cache(maxsize=None)
|
||||
def basic_info_json(keyboard):
|
||||
"""Generate a subset of info.json for a specific keyboard.
|
||||
|
||||
This does no validation, and should only be used as needed to avoid loops or when performance is critical.
|
||||
"""
|
||||
cur_dir = Path('keyboards')
|
||||
rules = parse_rules_mk_file(cur_dir / keyboard / 'rules.mk')
|
||||
|
||||
if 'DEFAULT_FOLDER' in rules:
|
||||
keyboard = rules['DEFAULT_FOLDER']
|
||||
rules = parse_rules_mk_file(cur_dir / keyboard / 'rules.mk', rules)
|
||||
|
||||
info_data = {
|
||||
'keyboard_name': str(keyboard),
|
||||
'keyboard_folder': str(keyboard),
|
||||
'keymaps': {},
|
||||
'layouts': {},
|
||||
'parse_errors': [],
|
||||
'parse_warnings': [],
|
||||
'maintainer': 'qmk',
|
||||
}
|
||||
|
||||
# Populate layout data
|
||||
layouts, aliases = _find_all_layouts(info_data, keyboard)
|
||||
|
||||
if aliases:
|
||||
info_data['layout_aliases'] = aliases
|
||||
|
||||
for layout_name, layout_json in layouts.items():
|
||||
if not layout_name.startswith('LAYOUT_kc'):
|
||||
layout_json['c_macro'] = True
|
||||
info_data['layouts'][layout_name] = layout_json
|
||||
|
||||
# Merge in the data from info.json, config.h, and rules.mk
|
||||
info_data = merge_info_jsons(keyboard, info_data)
|
||||
info_data = _extract_config_h(info_data)
|
||||
info_data = _extract_rules_mk(info_data)
|
||||
|
||||
return info_data
|
||||
|
||||
|
||||
def _extract_features(info_data, rules):
|
||||
"""Find all the features enabled in rules.mk.
|
||||
"""
|
||||
# Special handling for bootmagic which also supports a "lite" mode.
|
||||
if rules.get('BOOTMAGIC_ENABLE') == 'lite':
|
||||
rules['BOOTMAGIC_LITE_ENABLE'] = 'on'
|
||||
del rules['BOOTMAGIC_ENABLE']
|
||||
if rules.get('BOOTMAGIC_ENABLE') == 'full':
|
||||
rules['BOOTMAGIC_ENABLE'] = 'on'
|
||||
|
||||
# Skip non-boolean features we haven't implemented special handling for
|
||||
for feature in 'HAPTIC_ENABLE', 'QWIIC_ENABLE':
|
||||
if rules.get(feature):
|
||||
del rules[feature]
|
||||
|
||||
# Process the rest of the rules as booleans
|
||||
for key, value in rules.items():
|
||||
if key.endswith('_ENABLE'):
|
||||
key = '_'.join(key.split('_')[:-1]).lower()
|
||||
value = True if value.lower() in true_values else False if value.lower() in false_values else value
|
||||
|
||||
if 'config_h_features' not in info_data:
|
||||
info_data['config_h_features'] = {}
|
||||
|
||||
if 'features' not in info_data:
|
||||
info_data['features'] = {}
|
||||
|
||||
if key in info_data['features']:
|
||||
info_log_warning(info_data, 'Feature %s is specified in both info.json and rules.mk, the rules.mk value wins.' % (key,))
|
||||
|
||||
info_data['features'][key] = value
|
||||
info_data['config_h_features'][key] = value
|
||||
|
||||
return info_data
|
||||
|
||||
|
||||
def _pin_name(pin):
|
||||
"""Returns the proper representation for a pin.
|
||||
"""
|
||||
pin = pin.strip()
|
||||
|
||||
if not pin:
|
||||
return None
|
||||
|
||||
elif pin.isdigit():
|
||||
return int(pin)
|
||||
|
||||
elif pin == 'NO_PIN':
|
||||
return None
|
||||
|
||||
return pin
|
||||
|
||||
|
||||
def _extract_pins(pins):
|
||||
"""Returns a list of pins from a comma separated string of pins.
|
||||
"""
|
||||
return [_pin_name(pin) for pin in pins.split(',')]
|
||||
|
||||
|
||||
def _extract_direct_matrix(info_data, direct_pins):
|
||||
"""
|
||||
"""
|
||||
info_data['matrix_pins'] = {}
|
||||
direct_pin_array = []
|
||||
|
||||
while direct_pins[-1] != '}':
|
||||
direct_pins = direct_pins[:-1]
|
||||
|
||||
for row in direct_pins.split('},{'):
|
||||
if row.startswith('{'):
|
||||
row = row[1:]
|
||||
|
||||
if row.endswith('}'):
|
||||
row = row[:-1]
|
||||
|
||||
direct_pin_array.append([])
|
||||
|
||||
for pin in row.split(','):
|
||||
if pin == 'NO_PIN':
|
||||
pin = None
|
||||
|
||||
direct_pin_array[-1].append(pin)
|
||||
|
||||
return direct_pin_array
|
||||
|
||||
|
||||
def _extract_matrix_info(info_data, config_c):
|
||||
"""Populate the matrix information.
|
||||
"""
|
||||
row_pins = config_c.get('MATRIX_ROW_PINS', '').replace('{', '').replace('}', '').strip()
|
||||
col_pins = config_c.get('MATRIX_COL_PINS', '').replace('{', '').replace('}', '').strip()
|
||||
direct_pins = config_c.get('DIRECT_PINS', '').replace(' ', '')[1:-1]
|
||||
|
||||
if 'MATRIX_ROWS' in config_c and 'MATRIX_COLS' in config_c:
|
||||
if 'matrix_size' in info_data:
|
||||
info_log_warning(info_data, 'Matrix size is specified in both info.json and config.h, the config.h values win.')
|
||||
|
||||
info_data['matrix_size'] = {
|
||||
'cols': compute(config_c.get('MATRIX_COLS', '0')),
|
||||
'rows': compute(config_c.get('MATRIX_ROWS', '0')),
|
||||
}
|
||||
|
||||
if row_pins and col_pins:
|
||||
if 'matrix_pins' in info_data:
|
||||
info_log_warning(info_data, 'Matrix pins are specified in both info.json and config.h, the config.h values win.')
|
||||
|
||||
info_data['matrix_pins'] = {
|
||||
'cols': _extract_pins(col_pins),
|
||||
'rows': _extract_pins(row_pins),
|
||||
}
|
||||
|
||||
if direct_pins:
|
||||
if 'matrix_pins' in info_data:
|
||||
info_log_warning(info_data, 'Direct pins are specified in both info.json and config.h, the config.h values win.')
|
||||
|
||||
info_data['matrix_pins']['direct'] = _extract_direct_matrix(info_data, direct_pins)
|
||||
|
||||
return info_data
|
||||
|
||||
|
||||
def _extract_config_h(info_data):
|
||||
"""Pull some keyboard information from existing config.h files
|
||||
"""
|
||||
config_c = config_h(info_data['keyboard_folder'])
|
||||
|
||||
# Pull in data from the json map
|
||||
dotty_info = dotty(info_data)
|
||||
info_config_map = json_load(Path('data/mappings/info_config.json'))
|
||||
|
||||
for config_key, info_dict in info_config_map.items():
|
||||
info_key = info_dict['info_key']
|
||||
key_type = info_dict.get('value_type', 'str')
|
||||
|
||||
try:
|
||||
if config_key in config_c and info_dict.get('to_json', True):
|
||||
if dotty_info.get(info_key) and info_dict.get('warn_duplicate', True):
|
||||
info_log_warning(info_data, '%s in config.h is overwriting %s in info.json' % (config_key, info_key))
|
||||
|
||||
if key_type.startswith('array'):
|
||||
if '.' in key_type:
|
||||
key_type, array_type = key_type.split('.', 1)
|
||||
else:
|
||||
array_type = None
|
||||
|
||||
config_value = config_c[config_key].replace('{', '').replace('}', '').strip()
|
||||
|
||||
if array_type == 'int':
|
||||
dotty_info[info_key] = list(map(int, config_value.split(',')))
|
||||
else:
|
||||
dotty_info[info_key] = config_value.split(',')
|
||||
|
||||
elif key_type == 'bool':
|
||||
dotty_info[info_key] = config_c[config_key] in true_values
|
||||
|
||||
elif key_type == 'hex':
|
||||
dotty_info[info_key] = '0x' + config_c[config_key][2:].upper()
|
||||
|
||||
elif key_type == 'list':
|
||||
dotty_info[info_key] = config_c[config_key].split()
|
||||
|
||||
elif key_type == 'int':
|
||||
dotty_info[info_key] = int(config_c[config_key])
|
||||
|
||||
else:
|
||||
dotty_info[info_key] = config_c[config_key]
|
||||
|
||||
except Exception as e:
|
||||
info_log_warning(info_data, f'{config_key}->{info_key}: {e}')
|
||||
|
||||
info_data.update(dotty_info)
|
||||
|
||||
# Pull data that easily can't be mapped in json
|
||||
_extract_matrix_info(info_data, config_c)
|
||||
|
||||
return info_data
|
||||
|
||||
|
||||
def _extract_rules_mk(info_data):
|
||||
"""Pull some keyboard information from existing rules.mk files
|
||||
"""
|
||||
rules = rules_mk(info_data['keyboard_folder'])
|
||||
info_data['processor'] = rules.get('MCU', info_data.get('processor', 'atmega32u4'))
|
||||
|
||||
if info_data['processor'] in CHIBIOS_PROCESSORS:
|
||||
arm_processor_rules(info_data, rules)
|
||||
|
||||
elif info_data['processor'] in LUFA_PROCESSORS + VUSB_PROCESSORS:
|
||||
avr_processor_rules(info_data, rules)
|
||||
|
||||
else:
|
||||
cli.log.warning("%s: Unknown MCU: %s" % (info_data['keyboard_folder'], info_data['processor']))
|
||||
unknown_processor_rules(info_data, rules)
|
||||
|
||||
# Pull in data from the json map
|
||||
dotty_info = dotty(info_data)
|
||||
info_rules_map = json_load(Path('data/mappings/info_rules.json'))
|
||||
|
||||
for rules_key, info_dict in info_rules_map.items():
|
||||
info_key = info_dict['info_key']
|
||||
key_type = info_dict.get('value_type', 'str')
|
||||
|
||||
try:
|
||||
if rules_key in rules and info_dict.get('to_json', True):
|
||||
if dotty_info.get(info_key) and info_dict.get('warn_duplicate', True):
|
||||
info_log_warning(info_data, '%s in rules.mk is overwriting %s in info.json' % (rules_key, info_key))
|
||||
|
||||
if key_type.startswith('array'):
|
||||
if '.' in key_type:
|
||||
key_type, array_type = key_type.split('.', 1)
|
||||
else:
|
||||
array_type = None
|
||||
|
||||
rules_value = rules[rules_key].replace('{', '').replace('}', '').strip()
|
||||
|
||||
if array_type == 'int':
|
||||
dotty_info[info_key] = list(map(int, rules_value.split(',')))
|
||||
else:
|
||||
dotty_info[info_key] = rules_value.split(',')
|
||||
|
||||
elif key_type == 'list':
|
||||
dotty_info[info_key] = rules[rules_key].split()
|
||||
|
||||
elif key_type == 'bool':
|
||||
dotty_info[info_key] = rules[rules_key] in true_values
|
||||
|
||||
elif key_type == 'hex':
|
||||
dotty_info[info_key] = '0x' + rules[rules_key][2:].upper()
|
||||
|
||||
elif key_type == 'int':
|
||||
dotty_info[info_key] = int(rules[rules_key])
|
||||
|
||||
else:
|
||||
dotty_info[info_key] = rules[rules_key]
|
||||
|
||||
except Exception as e:
|
||||
info_log_warning(info_data, f'{rules_key}->{info_key}: {e}')
|
||||
|
||||
info_data.update(dotty_info)
|
||||
|
||||
# Merge in config values that can't be easily mapped
|
||||
_extract_features(info_data, rules)
|
||||
|
||||
return info_data
|
||||
|
||||
|
||||
def _search_keyboard_h(path):
|
||||
current_path = Path('keyboards/')
|
||||
aliases = {}
|
||||
layouts = {}
|
||||
|
||||
for directory in path.parts:
|
||||
current_path = current_path / directory
|
||||
keyboard_h = '%s.h' % (directory,)
|
||||
keyboard_h_path = current_path / keyboard_h
|
||||
if keyboard_h_path.exists():
|
||||
new_layouts, new_aliases = find_layouts(keyboard_h_path)
|
||||
layouts.update(new_layouts)
|
||||
|
||||
for alias, alias_text in new_aliases.items():
|
||||
if alias_text in layouts:
|
||||
aliases[alias] = alias_text
|
||||
|
||||
return layouts, aliases
|
||||
|
||||
|
||||
def _find_all_layouts(info_data, keyboard):
|
||||
"""Looks for layout macros associated with this keyboard.
|
||||
"""
|
||||
layouts, aliases = _search_keyboard_h(Path(keyboard))
|
||||
|
||||
if not layouts:
|
||||
# If we don't find any layouts from info.json or keyboard.h we widen our search. This is error prone which is why we want to encourage people to follow the standard above.
|
||||
info_data['parse_warnings'].append('%s: Falling back to searching for KEYMAP/LAYOUT macros.' % (keyboard))
|
||||
|
||||
for file in glob('keyboards/%s/*.h' % keyboard):
|
||||
if file.endswith('.h'):
|
||||
these_layouts, these_aliases = find_layouts(file)
|
||||
|
||||
if these_layouts:
|
||||
layouts.update(these_layouts)
|
||||
|
||||
for alias, alias_text in these_aliases.items():
|
||||
if alias_text in layouts:
|
||||
aliases[alias] = alias_text
|
||||
|
||||
return layouts, aliases
|
||||
|
||||
|
||||
def info_log_error(info_data, message):
|
||||
"""Send an error message to both JSON and the log.
|
||||
"""
|
||||
info_data['parse_errors'].append(message)
|
||||
cli.log.error('%s: %s', info_data.get('keyboard_folder', 'Unknown Keyboard!'), message)
|
||||
|
||||
|
||||
def info_log_warning(info_data, message):
|
||||
"""Send a warning message to both JSON and the log.
|
||||
"""
|
||||
info_data['parse_warnings'].append(message)
|
||||
cli.log.warning('%s: %s', info_data.get('keyboard_folder', 'Unknown Keyboard!'), message)
|
||||
|
||||
|
||||
def arm_processor_rules(info_data, rules):
|
||||
"""Setup the default info for an ARM board.
|
||||
"""
|
||||
info_data['processor_type'] = 'arm'
|
||||
info_data['protocol'] = 'ChibiOS'
|
||||
|
||||
if 'bootloader' not in info_data:
|
||||
if 'STM32' in info_data['processor']:
|
||||
info_data['bootloader'] = 'stm32-dfu'
|
||||
else:
|
||||
info_data['bootloader'] = 'unknown'
|
||||
|
||||
if 'STM32' in info_data['processor']:
|
||||
info_data['platform'] = 'STM32'
|
||||
elif 'MCU_SERIES' in rules:
|
||||
info_data['platform'] = rules['MCU_SERIES']
|
||||
elif 'ARM_ATSAM' in rules:
|
||||
info_data['platform'] = 'ARM_ATSAM'
|
||||
|
||||
return info_data
|
||||
|
||||
|
||||
def avr_processor_rules(info_data, rules):
|
||||
"""Setup the default info for an AVR board.
|
||||
"""
|
||||
info_data['processor_type'] = 'avr'
|
||||
info_data['platform'] = rules['ARCH'] if 'ARCH' in rules else 'unknown'
|
||||
info_data['protocol'] = 'V-USB' if rules.get('MCU') in VUSB_PROCESSORS else 'LUFA'
|
||||
|
||||
if 'bootloader' not in info_data:
|
||||
info_data['bootloader'] = 'atmel-dfu'
|
||||
|
||||
# FIXME(fauxpark/anyone): Eventually we should detect the protocol by looking at PROTOCOL inherited from mcu_selection.mk:
|
||||
# info_data['protocol'] = 'V-USB' if rules.get('PROTOCOL') == 'VUSB' else 'LUFA'
|
||||
|
||||
return info_data
|
||||
|
||||
|
||||
def unknown_processor_rules(info_data, rules):
|
||||
"""Setup the default keyboard info for unknown boards.
|
||||
"""
|
||||
info_data['bootloader'] = 'unknown'
|
||||
info_data['platform'] = 'unknown'
|
||||
info_data['processor'] = 'unknown'
|
||||
info_data['processor_type'] = 'unknown'
|
||||
info_data['protocol'] = 'unknown'
|
||||
|
||||
return info_data
|
||||
|
||||
|
||||
def merge_info_jsons(keyboard, info_data):
|
||||
"""Return a merged copy of all the info.json files for a keyboard.
|
||||
"""
|
||||
for info_file in find_info_json(keyboard):
|
||||
# Load and validate the JSON data
|
||||
new_info_data = json_load(info_file)
|
||||
|
||||
if not isinstance(new_info_data, dict):
|
||||
info_log_error(info_data, "Invalid file %s, root object should be a dictionary." % (str(info_file),))
|
||||
continue
|
||||
|
||||
try:
|
||||
validate(new_info_data, 'qmk.keyboard.v1')
|
||||
except jsonschema.ValidationError as e:
|
||||
json_path = '.'.join([str(p) for p in e.absolute_path])
|
||||
cli.log.error('Not including data from file: %s', info_file)
|
||||
cli.log.error('\t%s: %s', json_path, e.message)
|
||||
continue
|
||||
|
||||
# Merge layout data in
|
||||
if 'layout_aliases' in new_info_data:
|
||||
info_data['layout_aliases'] = {**info_data.get('layout_aliases', {}), **new_info_data['layout_aliases']}
|
||||
del new_info_data['layout_aliases']
|
||||
|
||||
for layout_name, layout in new_info_data.get('layouts', {}).items():
|
||||
if layout_name in info_data.get('layout_aliases', {}):
|
||||
info_log_warning(info_data, f"info.json uses alias name {layout_name} instead of {info_data['layout_aliases'][layout_name]}")
|
||||
layout_name = info_data['layout_aliases'][layout_name]
|
||||
|
||||
if layout_name in info_data['layouts']:
|
||||
for new_key, existing_key in zip(layout['layout'], info_data['layouts'][layout_name]['layout']):
|
||||
existing_key.update(new_key)
|
||||
else:
|
||||
layout['c_macro'] = False
|
||||
info_data['layouts'][layout_name] = layout
|
||||
|
||||
# Update info_data with the new data
|
||||
if 'layouts' in new_info_data:
|
||||
del new_info_data['layouts']
|
||||
|
||||
deep_update(info_data, new_info_data)
|
||||
|
||||
return info_data
|
||||
|
||||
|
||||
def find_info_json(keyboard):
|
||||
"""Finds all the info.json files associated with a keyboard.
|
||||
"""
|
||||
# Find the most specific first
|
||||
base_path = Path('keyboards')
|
||||
keyboard_path = base_path / keyboard
|
||||
keyboard_parent = keyboard_path.parent
|
||||
info_jsons = [keyboard_path / 'info.json']
|
||||
|
||||
# Add DEFAULT_FOLDER before parents, if present
|
||||
rules = rules_mk(keyboard)
|
||||
if 'DEFAULT_FOLDER' in rules:
|
||||
info_jsons.append(Path(rules['DEFAULT_FOLDER']) / 'info.json')
|
||||
|
||||
# Add in parent folders for least specific
|
||||
for _ in range(5):
|
||||
info_jsons.append(keyboard_parent / 'info.json')
|
||||
if keyboard_parent.parent == base_path:
|
||||
break
|
||||
keyboard_parent = keyboard_parent.parent
|
||||
|
||||
# Return a list of the info.json files that actually exist
|
||||
return [info_json for info_json in info_jsons if info_json.exists()]
|
||||
Loading…
Reference in a new issue