CLI refactoring for common build target APIs (#22221)

lib/python/qmk/build_targets.py

@@ -0,0 +1,211 @@
+# Copyright 2023 Nick Brassel (@tzarc)
+# SPDX-License-Identifier: GPL-2.0-or-later
+import json
+import shutil
+from typing import List, Union
+from pathlib import Path
+from dotty_dict import dotty, Dotty
+from milc import cli
+from qmk.constants import QMK_FIRMWARE, INTERMEDIATE_OUTPUT_PREFIX
+from qmk.commands import find_make, get_make_parallel_args, parse_configurator_json
+from qmk.keyboard import keyboard_folder
+from qmk.info import keymap_json
+from qmk.cli.generate.compilation_database import write_compilation_database
+
+
+class BuildTarget:
+    def __init__(self, keyboard: str, keymap: str, json: Union[dict, Dotty] = None):
+        self._keyboard = keyboard_folder(keyboard)
+        self._keyboard_safe = self._keyboard.replace('/', '_')
+        self._keymap = keymap
+        self._parallel = 1
+        self._clean = False
+        self._compiledb = False
+        self._target = f'{self._keyboard_safe}_{self.keymap}'
+        self._intermediate_output = Path(f'{INTERMEDIATE_OUTPUT_PREFIX}{self._target}')
+        self._generated_files_path = self._intermediate_output / 'src'
+        self._json = json.to_dict() if isinstance(json, Dotty) else json
+
+    def __str__(self):
+        return f'{self.keyboard}:{self.keymap}'
+
+    def __repr__(self):
+        return f'BuildTarget(keyboard={self.keyboard}, keymap={self.keymap})'
+
+    def configure(self, parallel: int = None, clean: bool = None, compiledb: bool = None) -> None:
+        if parallel is not None:
+            self._parallel = parallel
+        if clean is not None:
+            self._clean = clean
+        if compiledb is not None:
+            self._compiledb = compiledb
+
+    @property
+    def keyboard(self) -> str:
+        return self._keyboard
+
+    @property
+    def keymap(self) -> str:
+        return self._keymap
+
+    @property
+    def json(self) -> dict:
+        if not self._json:
+            self._load_json()
+        if not self._json:
+            return {}
+        return self._json
+
+    @property
+    def dotty(self) -> Dotty:
+        return dotty(self.json)
+
+    def _common_make_args(self, dry_run: bool = False, build_target: str = None):
+        compile_args = [
+            find_make(),
+            *get_make_parallel_args(self._parallel),
+            '-r',
+            '-R',
+            '-f',
+            'builddefs/build_keyboard.mk',
+        ]
+
+        if not cli.config.general.verbose:
+            compile_args.append('-s')
+
+        verbose = 'true' if cli.config.general.verbose else 'false'
+        color = 'true' if cli.config.general.color else 'false'
+
+        if dry_run:
+            compile_args.append('-n')
+
+        if build_target:
+            compile_args.append(build_target)
+
+        compile_args.extend([
+            f'KEYBOARD={self.keyboard}',
+            f'KEYMAP={self.keymap}',
+            f'KEYBOARD_FILESAFE={self._keyboard_safe}',
+            f'TARGET={self._target}',
+            f'INTERMEDIATE_OUTPUT={self._intermediate_output}',
+            f'VERBOSE={verbose}',
+            f'COLOR={color}',
+            'SILENT=false',
+            'QMK_BIN="qmk"',
+        ])
+
+        return compile_args
+
+    def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None:
+        raise NotImplementedError("prepare_build() not implemented in base class")
+
+    def compile_command(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]:
+        raise NotImplementedError("compile_command() not implemented in base class")
+
+    def generate_compilation_database(self, build_target: str = None, skip_clean: bool = False, **env_vars) -> None:
+        self.prepare_build(build_target=build_target, **env_vars)
+        command = self.compile_command(build_target=build_target, dry_run=True, **env_vars)
+        write_compilation_database(command=command, output_path=QMK_FIRMWARE / 'compile_commands.json', skip_clean=skip_clean, **env_vars)
+
+    def compile(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None:
+        if self._clean or self._compiledb:
+            command = [find_make(), "clean"]
+            if dry_run:
+                command.append('-n')
+            cli.log.info('Cleaning with {fg_cyan}%s', ' '.join(command))
+            cli.run(command, capture_output=False)
+
+        if self._compiledb and not dry_run:
+            self.generate_compilation_database(build_target=build_target, skip_clean=True, **env_vars)
+
+        self.prepare_build(build_target=build_target, dry_run=dry_run, **env_vars)
+        command = self.compile_command(build_target=build_target, **env_vars)
+        cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command))
+        if not dry_run:
+            cli.echo('\n')
+            ret = cli.run(command, capture_output=False)
+            if ret.returncode:
+                return ret.returncode
+
+
+class KeyboardKeymapBuildTarget(BuildTarget):
+    def __init__(self, keyboard: str, keymap: str, json: dict = None):
+        super().__init__(keyboard=keyboard, keymap=keymap, json=json)
+
+    def __repr__(self):
+        return f'KeyboardKeymapTarget(keyboard={self.keyboard}, keymap={self.keymap})'
+
+    def _load_json(self):
+        self._json = keymap_json(self.keyboard, self.keymap)
+
+    def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None:
+        pass
+
+    def compile_command(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]:
+        compile_args = self._common_make_args(dry_run=dry_run, build_target=build_target)
+
+        for key, value in env_vars.items():
+            compile_args.append(f'{key}={value}')
+
+        return compile_args
+
+
+class JsonKeymapBuildTarget(BuildTarget):
+    def __init__(self, json_path):
+        if isinstance(json_path, Path):
+            self.json_path = json_path
+        else:
+            self.json_path = None
+
+        json = parse_configurator_json(json_path)  # Will load from stdin if provided
+
+        # In case the user passes a keymap.json from a keymap directory directly to the CLI.
+        # e.g.: qmk compile - < keyboards/clueboard/california/keymaps/default/keymap.json
+        json["keymap"] = json.get("keymap", "default_json")
+
+        super().__init__(keyboard=json['keyboard'], keymap=json['keymap'], json=json)
+
+        self._keymap_json = self._generated_files_path / 'keymap.json'
+
+    def __repr__(self):
+        return f'JsonKeymapTarget(keyboard={self.keyboard}, keymap={self.keymap}, path={self.json_path})'
+
+    def _load_json(self):
+        pass  # Already loaded in constructor
+
+    def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None:
+        if self._clean:
+            if self._intermediate_output.exists():
+                shutil.rmtree(self._intermediate_output)
+
+        # begin with making the deepest folder in the tree
+        self._generated_files_path.mkdir(exist_ok=True, parents=True)
+
+        # Compare minified to ensure consistent comparison
+        new_content = json.dumps(self.json, separators=(',', ':'))
+        if self._keymap_json.exists():
+            old_content = json.dumps(json.loads(self._keymap_json.read_text(encoding='utf-8')), separators=(',', ':'))
+            if old_content == new_content:
+                new_content = None
+
+        # Write the keymap.json file if different so timestamps are only updated
+        # if the content changes -- running `make` won't treat it as modified.
+        if new_content:
+            self._keymap_json.write_text(new_content, encoding='utf-8')
+
+    def compile_command(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]:
+        compile_args = self._common_make_args(dry_run=dry_run, build_target=build_target)
+        compile_args.extend([
+            f'MAIN_KEYMAP_PATH_1={self._intermediate_output}',
+            f'MAIN_KEYMAP_PATH_2={self._intermediate_output}',
+            f'MAIN_KEYMAP_PATH_3={self._intermediate_output}',
+            f'MAIN_KEYMAP_PATH_4={self._intermediate_output}',
+            f'MAIN_KEYMAP_PATH_5={self._intermediate_output}',
+            f'KEYMAP_JSON={self._keymap_json}',
+            f'KEYMAP_PATH={self._generated_files_path}',
+        ])
+
+        for key, value in env_vars.items():
+            compile_args.append(f'{key}={value}')
+
+        return compile_args

lib/python/qmk/cli/clean.py

@@ -2,7 +2,7 @@
 """
 from subprocess import DEVNULL
 
-from qmk.commands import create_make_target
+from qmk.commands import find_make
 from milc import cli
 
 
@@ -11,4 +11,4 @@ from milc import cli
 def clean(cli):
     """Runs `make clean` (or `make distclean` if --all is passed)
     """
-    cli.run(create_make_target('distclean' if cli.args.all else 'clean'), capture_output=False, stdin=DEVNULL)
+    cli.run([find_make(), 'distclean' if cli.args.all else 'clean'], capture_output=False, stdin=DEVNULL)

lib/python/qmk/cli/compile.py

@@ -7,22 +7,11 @@ from argcomplete.completers import FilesCompleter
 from milc import cli
 
 import qmk.path
-from qmk.constants import QMK_FIRMWARE
 from qmk.decorators import automagic_keyboard, automagic_keymap
-from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json, build_environment
+from qmk.commands import build_environment
 from qmk.keyboard import keyboard_completer, keyboard_folder_or_all, is_all_keyboards
 from qmk.keymap import keymap_completer, locate_keymap
-from qmk.cli.generate.compilation_database import write_compilation_database
+from qmk.build_targets import KeyboardKeymapBuildTarget, JsonKeymapBuildTarget
-
-
-def _is_keymap_target(keyboard, keymap):
-    if keymap == 'all':
-        return True
-
-    if locate_keymap(keyboard, keymap):
-        return True
-
-    return False
 
 
 @cli.argument('filename', nargs='?', arg_only=True, type=qmk.path.FileType('r'), completer=FilesCompleter('.json'), help='The configurator export to compile')
@@ -32,6 +21,7 @@ def _is_keymap_target(keyboard, keymap):
 @cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.")
 @cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.")
 @cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.")
+@cli.argument('-t', '--target', type=str, default=None, help="Intended alternative build target, such as `production` in `make planck/rev4:default:production`.")
 @cli.argument('--compiledb', arg_only=True, action='store_true', help="Generates the clang compile_commands.json file during build. Implies --clean.")
 @cli.subcommand('Compile a QMK Firmware.')
 @automagic_keyboard
@@ -53,47 +43,27 @@ def compile(cli):
     # Build the environment vars
     envs = build_environment(cli.args.env)
 
-    # Determine the compile command
+    # Handler for the build target
-    commands = []
+    target = None
-
-    current_keyboard = None
-    current_keymap = None
 
     if cli.args.filename:
-        # If a configurator JSON was provided generate a keymap and compile it
+        # if we were given a filename, assume we have a json build target
-        user_keymap = parse_configurator_json(cli.args.filename)
+        target = JsonKeymapBuildTarget(cli.args.filename)
-        commands = [compile_configurator_json(user_keymap, parallel=cli.config.compile.parallel, clean=cli.args.clean, **envs)]
 
     elif cli.config.compile.keyboard and cli.config.compile.keymap:
-        # Generate the make command for a specific keyboard/keymap.
+        # if we got a keyboard and keymap, attempt to find it
-        if not _is_keymap_target(cli.config.compile.keyboard, cli.config.compile.keymap):
+        if not locate_keymap(cli.config.compile.keyboard, cli.config.compile.keymap):
             cli.log.error('Invalid keymap argument.')
             cli.print_help()
             return False
 
-        if cli.args.clean:
+        # If we got here, then we have a valid keyboard and keymap for a build target
-            commands.append(create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, 'clean', **envs))
+        target = KeyboardKeymapBuildTarget(cli.config.compile.keyboard, cli.config.compile.keymap)
-        commands.append(create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, parallel=cli.config.compile.parallel, **envs))
 
-        current_keyboard = cli.config.compile.keyboard
+    if not target:
-        current_keymap = cli.config.compile.keymap
-
-    if not commands:
         cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')
         cli.print_help()
         return False
 
-    if cli.args.compiledb:
+    target.configure(parallel=cli.config.compile.parallel, clean=cli.args.clean, compiledb=cli.args.compiledb)
-        if current_keyboard is None or current_keymap is None:
+    target.compile(cli.args.target, dry_run=cli.args.dry_run, **envs)
-            cli.log.error('You must supply both `--keyboard` and `--keymap` or be in a directory with a keymap to generate a compile_commands.json file.')
-            cli.print_help()
-            return False
-        write_compilation_database(current_keyboard, current_keymap, QMK_FIRMWARE / 'compile_commands.json')
-
-    cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(commands[-1]))
-    if not cli.args.dry_run:
-        cli.echo('\n')
-        for command in commands:
-            ret = cli.run(command, capture_output=False)
-            if ret.returncode:
-                return ret.returncode

lib/python/qmk/cli/find.py

@@ -19,13 +19,9 @@ from qmk.search import search_keymap_targets
 def find(cli):
     """Search through all keyboards and keymaps for a given search criteria.
     """
+    targets = search_keymap_targets([('all', cli.config.find.keymap)], cli.args.filter)
+    for target in sorted(targets, key=lambda t: (t.keyboard, t.keymap)):
+        print(f'{target}')
 
-    if len(cli.args.filter) == 0 and len(cli.args.print) > 0:
+        for key in cli.args.print:
-        cli.log.warning('No filters supplied -- keymaps not parsed, unable to print requested values.')
+            print(f'    {key}={target.dotty.get(key, None)}')
-
-    targets = search_keymap_targets([('all', cli.config.find.keymap)], cli.args.filter, cli.args.print)
-    for keyboard, keymap, print_vals in targets:
-        print(f'{keyboard}:{keymap}')
-
-        for key, val in print_vals:
-            print(f'    {key}={val}')

lib/python/qmk/cli/flash.py

@@ -4,25 +4,17 @@ You can compile a keymap already in the repo or using a QMK Configurator export.
 A bootloader must be specified.
 """
 from argcomplete.completers import FilesCompleter
+from pathlib import Path
 
 from milc import cli
 
 import qmk.path
 from qmk.decorators import automagic_keyboard, automagic_keymap
-from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json, build_environment
+from qmk.commands import build_environment
 from qmk.keyboard import keyboard_completer, keyboard_folder
 from qmk.keymap import keymap_completer, locate_keymap
 from qmk.flashers import flasher
-
+from qmk.build_targets import KeyboardKeymapBuildTarget, JsonKeymapBuildTarget
-
-def _is_keymap_target(keyboard, keymap):
-    if keymap == 'all':
-        return True
-
-    if locate_keymap(keyboard, keymap):
-        return True
-
-    return False
 
 
 def _list_bootloaders():
@@ -89,7 +81,7 @@ def flash(cli):
 
     If bootloader is omitted the make system will use the configured bootloader for that keyboard.
     """
-    if cli.args.filename and cli.args.filename.suffix in ['.bin', '.hex', '.uf2']:
+    if cli.args.filename and isinstance(cli.args.filename, Path) and cli.args.filename.suffix in ['.bin', '.hex', '.uf2']:
         return _flash_binary(cli.args.filename, cli.args.mcu)
 
     if cli.args.bootloaders:
@@ -98,34 +90,27 @@ def flash(cli):
     # Build the environment vars
     envs = build_environment(cli.args.env)
 
-    # Determine the compile command
+    # Handler for the build target
-    commands = []
+    target = None
 
     if cli.args.filename:
-        # If a configurator JSON was provided generate a keymap and compile it
+        # if we were given a filename, assume we have a json build target
-        user_keymap = parse_configurator_json(cli.args.filename)
+        target = JsonKeymapBuildTarget(cli.args.filename)
-        commands = [compile_configurator_json(user_keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, clean=cli.args.clean, **envs)]
 
     elif cli.config.flash.keyboard and cli.config.flash.keymap:
-        # Generate the make command for a specific keyboard/keymap.
+        # if we got a keyboard and keymap, attempt to find it
-        if not _is_keymap_target(cli.config.flash.keyboard, cli.config.flash.keymap):
+        if not locate_keymap(cli.config.flash.keyboard, cli.config.flash.keymap):
             cli.log.error('Invalid keymap argument.')
             cli.print_help()
             return False
 
-        if cli.args.clean:
+        # If we got here, then we have a valid keyboard and keymap for a build target
-            commands.append(create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, 'clean', **envs))
+        target = KeyboardKeymapBuildTarget(cli.config.flash.keyboard, cli.config.flash.keymap)
-        commands.append(create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, **envs))
 
-    if not commands:
+    if not target:
         cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')
         cli.print_help()
         return False
 
-    cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(commands[-1]))
+    target.configure(parallel=cli.config.flash.parallel, clean=cli.args.clean)
-    if not cli.args.dry_run:
+    target.compile(cli.args.bootloader, dry_run=cli.args.dry_run, **envs)
-        cli.echo('\n')
-        for command in commands:
-            ret = cli.run(command, capture_output=False)
-            if ret.returncode:
-                return ret.returncode

lib/python/qmk/cli/generate/compilation_database.py

@@ -12,7 +12,7 @@ from typing import Dict, Iterator, List, Union
 
 from milc import cli, MILC
 
-from qmk.commands import create_make_command
+from qmk.commands import find_make
 from qmk.constants import QMK_FIRMWARE
 from qmk.decorators import automagic_keyboard, automagic_keymap
 from qmk.keyboard import keyboard_completer, keyboard_folder
@@ -76,9 +76,12 @@ def parse_make_n(f: Iterator[str]) -> List[Dict[str, str]]:
     return records
 
 
-def write_compilation_database(keyboard: str, keymap: str, output_path: Path) -> bool:
+def write_compilation_database(keyboard: str = None, keymap: str = None, output_path: Path = QMK_FIRMWARE / 'compile_commands.json', skip_clean: bool = False, command: List[str] = None, **env_vars) -> bool:
     # Generate the make command for a specific keyboard/keymap.
-    command = create_make_command(keyboard, keymap, dry_run=True)
+    if not command:
+        from qmk.build_targets import KeyboardKeymapBuildTarget  # Lazy load due to circular references
+        target = KeyboardKeymapBuildTarget(keyboard, keymap)
+        command = target.compile_command(dry_run=True, **env_vars)
 
     if not command:
         cli.log.error('You must supply both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')
@@ -90,9 +93,10 @@ def write_compilation_database(keyboard: str, keymap: str, output_path: Path) ->
     env.pop("MAKEFLAGS", None)
 
     # re-use same executable as the main make invocation (might be gmake)
-    clean_command = [command[0], 'clean']
+    if not skip_clean:
-    cli.log.info('Making clean with {fg_cyan}%s', ' '.join(clean_command))
+        clean_command = [find_make(), "clean"]
-    cli.run(clean_command, capture_output=False, check=True, env=env)
+        cli.log.info('Making clean with {fg_cyan}%s', ' '.join(clean_command))
+        cli.run(clean_command, capture_output=False, check=True, env=env)
 
     cli.log.info('Gathering build instructions from {fg_cyan}%s', ' '.join(command))
 

lib/python/qmk/cli/mass_compile.py

@@ -3,26 +3,28 @@
 This will compile everything in parallel, for testing purposes.
 """
 import os
+from typing import List
 from pathlib import Path
 from subprocess import DEVNULL
 from milc import cli
 
 from qmk.constants import QMK_FIRMWARE
-from qmk.commands import _find_make, get_make_parallel_args
+from qmk.commands import find_make, get_make_parallel_args, build_environment
 from qmk.search import search_keymap_targets, search_make_targets
+from qmk.build_targets import BuildTarget, JsonKeymapBuildTarget
 
 
-def mass_compile_targets(targets, clean, dry_run, no_temp, parallel, env):
+def mass_compile_targets(targets: List[BuildTarget], clean: bool, dry_run: bool, no_temp: bool, parallel: int, **env):
     if len(targets) == 0:
         return
 
-    make_cmd = _find_make()
+    make_cmd = find_make()
     builddir = Path(QMK_FIRMWARE) / '.build'
     makefile = builddir / 'parallel_kb_builds.mk'
 
     if dry_run:
         cli.log.info('Compilation targets:')
-        for target in sorted(targets):
+        for target in sorted(targets, key=lambda t: (t.keyboard, t.keymap)):
             cli.log.info(f"{{fg_cyan}}qmk compile -kb {target[0]} -km {target[1]}{{fg_reset}}")
     else:
         if clean:
@@ -30,9 +32,13 @@ def mass_compile_targets(targets, clean, dry_run, no_temp, parallel, env):
 
         builddir.mkdir(parents=True, exist_ok=True)
         with open(makefile, "w") as f:
-            for target in sorted(targets):
+            for target in sorted(targets, key=lambda t: (t.keyboard, t.keymap)):
-                keyboard_name = target[0]
+                keyboard_name = target.keyboard
-                keymap_name = target[1]
+                keymap_name = target.keymap
+                target.configure(parallel=1)  # We ignore parallelism on a per-build basis as we defer to the parent make invocation
+                target.prepare_build(**env)  # If we've got json targets, allow them to write out any extra info to .build before we kick off `make`
+                command = target.compile_command(**env)
+                command[0] = '+@$(MAKE)'  # Override the make so that we can use jobserver to handle parallelism
                 keyboard_safe = keyboard_name.replace('/', '_')
                 build_log = f"{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}.{keymap_name}"
                 failed_log = f"{QMK_FIRMWARE}/.build/failed.log.{os.getpid()}.{keyboard_safe}.{keymap_name}"
@@ -43,7 +49,7 @@ all: {keyboard_safe}_{keymap_name}_binary
 {keyboard_safe}_{keymap_name}_binary:
 	@rm -f "{build_log}" || true
 	@echo "Compiling QMK Firmware for target: '{keyboard_name}:{keymap_name}'..." >>"{build_log}"
-	+@$(MAKE) -C "{QMK_FIRMWARE}" -f "{QMK_FIRMWARE}/builddefs/build_keyboard.mk" KEYBOARD="{keyboard_name}" KEYMAP="{keymap_name}" COLOR=true SILENT=false {' '.join(env)} \\
+	{' '.join(command)} \\
 		>>"{build_log}" 2>&1 \\
 		|| cp "{build_log}" "{failed_log}"
 	@{{ grep '\[ERRORS\]' "{build_log}" >/dev/null 2>&1 && printf "Build %-64s \e[1;31m[ERRORS]\e[0m\\n" "{keyboard_name}:{keymap_name}" ; }} \\
@@ -95,8 +101,11 @@ def mass_compile(cli):
     """Compile QMK Firmware against all keyboards.
     """
     if len(cli.args.builds) > 0:
-        targets = search_make_targets(cli.args.builds, cli.args.filter)
+        json_like_targets = list([Path(p) for p in filter(lambda e: Path(e).exists() and Path(e).suffix == '.json', cli.args.builds)])
+        make_like_targets = list(filter(lambda e: Path(e) not in json_like_targets, cli.args.builds))
+        targets = search_make_targets(make_like_targets)
+        targets.extend([JsonKeymapBuildTarget(e) for e in json_like_targets])
     else:
         targets = search_keymap_targets([('all', cli.config.mass_compile.keymap)], cli.args.filter)
 
-    return mass_compile_targets(targets, cli.args.clean, cli.args.dry_run, cli.config.mass_compile.no_temp, cli.config.mass_compile.parallel, cli.args.env)
+    return mass_compile_targets(targets, cli.args.clean, cli.args.dry_run, cli.config.mass_compile.no_temp, cli.config.mass_compile.parallel, **build_environment(cli.args.env))

lib/python/qmk/commands.py

@@ -2,18 +2,16 @@
 """
 import os
 import sys
-import json
 import shutil
 from pathlib import Path
 
 from milc import cli
 import jsonschema
 
-from qmk.constants import INTERMEDIATE_OUTPUT_PREFIX
 from qmk.json_schema import json_load, validate
 
 
-def _find_make():
+def find_make():
     """Returns the correct make command for this environment.
     """
     make_cmd = os.environ.get('MAKE')
@@ -24,74 +22,6 @@ def _find_make():
     return make_cmd
 
 
-def create_make_target(target, dry_run=False, parallel=1, **env_vars):
-    """Create a make command
-
-    Args:
-
-        target
-            Usually a make rule, such as 'clean' or 'all'.
-
-        dry_run
-            make -n -- don't actually build
-
-        parallel
-            The number of make jobs to run in parallel
-
-        **env_vars
-            Environment variables to be passed to make.
-
-    Returns:
-
-        A command that can be run to make the specified keyboard and keymap
-    """
-    env = []
-    make_cmd = _find_make()
-
-    for key, value in env_vars.items():
-        env.append(f'{key}={value}')
-
-    if cli.config.general.verbose:
-        env.append('VERBOSE=true')
-
-    return [make_cmd, *(['-n'] if dry_run else []), *get_make_parallel_args(parallel), *env, target]
-
-
-def create_make_command(keyboard, keymap, target=None, dry_run=False, parallel=1, **env_vars):
-    """Create a make compile command
-
-    Args:
-
-        keyboard
-            The path of the keyboard, for example 'plank'
-
-        keymap
-            The name of the keymap, for example 'algernon'
-
-        target
-            Usually a bootloader.
-
-        dry_run
-            make -n -- don't actually build
-
-        parallel
-            The number of make jobs to run in parallel
-
-        **env_vars
-            Environment variables to be passed to make.
-
-    Returns:
-
-        A command that can be run to make the specified keyboard and keymap
-    """
-    make_args = [keyboard, keymap]
-
-    if target:
-        make_args.append(target)
-
-    return create_make_target(':'.join(make_args), dry_run=dry_run, parallel=parallel, **env_vars)
-
-
 def get_make_parallel_args(parallel=1):
     """Returns the arguments for running the specified number of parallel jobs.
     """
@@ -100,7 +30,7 @@ def get_make_parallel_args(parallel=1):
     if int(parallel) <= 0:
         # 0 or -1 means -j without argument (unlimited jobs)
         parallel_args.append('--jobs')
-    else:
+    elif int(parallel) > 1:
         parallel_args.append('--jobs=' + str(parallel))
 
     if int(parallel) != 1:
@@ -110,96 +40,6 @@ def get_make_parallel_args(parallel=1):
     return parallel_args
 
 
-def compile_configurator_json(user_keymap, bootloader=None, parallel=1, clean=False, **env_vars):
-    """Convert a configurator export JSON file into a C file and then compile it.
-
-    Args:
-
-        user_keymap
-            A deserialized keymap export
-
-        bootloader
-            A bootloader to flash
-
-        parallel
-            The number of make jobs to run in parallel
-
-    Returns:
-
-        A command to run to compile and flash the C file.
-    """
-    # In case the user passes a keymap.json from a keymap directory directly to the CLI.
-    # e.g.: qmk compile - < keyboards/clueboard/california/keymaps/default/keymap.json
-    user_keymap["keymap"] = user_keymap.get("keymap", "default_json")
-
-    keyboard_filesafe = user_keymap['keyboard'].replace('/', '_')
-    target = f'{keyboard_filesafe}_{user_keymap["keymap"]}'
-    intermediate_output = Path(f'{INTERMEDIATE_OUTPUT_PREFIX}{keyboard_filesafe}_{user_keymap["keymap"]}')
-    keymap_dir = intermediate_output / 'src'
-    keymap_json = keymap_dir / 'keymap.json'
-
-    if clean:
-        if intermediate_output.exists():
-            shutil.rmtree(intermediate_output)
-
-    # begin with making the deepest folder in the tree
-    keymap_dir.mkdir(exist_ok=True, parents=True)
-
-    # Compare minified to ensure consistent comparison
-    new_content = json.dumps(user_keymap, separators=(',', ':'))
-    if keymap_json.exists():
-        old_content = json.dumps(json.loads(keymap_json.read_text(encoding='utf-8')), separators=(',', ':'))
-        if old_content == new_content:
-            new_content = None
-
-    # Write the keymap.json file if different
-    if new_content:
-        keymap_json.write_text(new_content, encoding='utf-8')
-
-    # Return a command that can be run to make the keymap and flash if given
-    verbose = 'true' if cli.config.general.verbose else 'false'
-    color = 'true' if cli.config.general.color else 'false'
-    make_command = [_find_make()]
-
-    if not cli.config.general.verbose:
-        make_command.append('-s')
-
-    make_command.extend([
-        *get_make_parallel_args(parallel),
-        '-r',
-        '-R',
-        '-f',
-        'builddefs/build_keyboard.mk',
-    ])
-
-    if bootloader:
-        make_command.append(bootloader)
-
-    make_command.extend([
-        f'KEYBOARD={user_keymap["keyboard"]}',
-        f'KEYMAP={user_keymap["keymap"]}',
-        f'KEYBOARD_FILESAFE={keyboard_filesafe}',
-        f'TARGET={target}',
-        f'INTERMEDIATE_OUTPUT={intermediate_output}',
-        f'MAIN_KEYMAP_PATH_1={intermediate_output}',
-        f'MAIN_KEYMAP_PATH_2={intermediate_output}',
-        f'MAIN_KEYMAP_PATH_3={intermediate_output}',
-        f'MAIN_KEYMAP_PATH_4={intermediate_output}',
-        f'MAIN_KEYMAP_PATH_5={intermediate_output}',
-        f'KEYMAP_JSON={keymap_json}',
-        f'KEYMAP_PATH={keymap_dir}',
-        f'VERBOSE={verbose}',
-        f'COLOR={color}',
-        'SILENT=false',
-        'QMK_BIN="qmk"',
-    ])
-
-    for key, value in env_vars.items():
-        make_command.append(f'{key}={value}')
-
-    return make_command
-
-
 def parse_configurator_json(configurator_file):
     """Open and parse a configurator json export
     """

lib/python/qmk/keyboard.py

@@ -98,11 +98,7 @@ def keyboard_folder(keyboard):
         if keyboard == last_keyboard:
             break
 
-    rules_mk_file = Path(base_path, keyboard, 'rules.mk')
+    keyboard = resolve_keyboard(keyboard)
-
-    if rules_mk_file.exists():
-        rules_mk = parse_rules_mk_file(rules_mk_file)
-        keyboard = rules_mk.get('DEFAULT_FOLDER', keyboard)
 
     if not qmk.path.is_keyboard(keyboard):
         raise ValueError(f'Invalid keyboard: {keyboard}')

lib/python/qmk/search.py

@@ -11,8 +11,9 @@ from milc import cli
 
 from qmk.util import parallel_map
 from qmk.info import keymap_json
-import qmk.keyboard
+from qmk.keyboard import list_keyboards, keyboard_folder
-import qmk.keymap
+from qmk.keymap import list_keymaps, locate_keymap
+from qmk.build_targets import KeyboardKeymapBuildTarget, BuildTarget
 
 
 def _set_log_level(level):
@@ -36,15 +37,15 @@ def _all_keymaps(keyboard):
     """Returns a list of tuples of (keyboard, keymap) for all keymaps for the given keyboard.
     """
     with ignore_logging():
-        keyboard = qmk.keyboard.resolve_keyboard(keyboard)
+        keyboard = keyboard_folder(keyboard)
-        return [(keyboard, keymap) for keymap in qmk.keymap.list_keymaps(keyboard)]
+        return [(keyboard, keymap) for keymap in list_keymaps(keyboard)]
 
 
 def _keymap_exists(keyboard, keymap):
     """Returns the keyboard name if the keyboard+keymap combination exists, otherwise None.
     """
     with ignore_logging():
-        return keyboard if qmk.keymap.locate_keymap(keyboard, keymap) is not None else None
+        return keyboard if locate_keymap(keyboard, keymap) is not None else None
 
 
 def _load_keymap_info(kb_km):
@@ -75,7 +76,7 @@ def _expand_keymap_target(keyboard: str, keymap: str, all_keyboards: List[str] =
     Caters for 'all' in either keyboard or keymap, or both.
     """
     if all_keyboards is None:
-        all_keyboards = qmk.keyboard.list_keyboards()
+        all_keyboards = list_keyboards()
 
     if keyboard == 'all':
         if keymap == 'all':
@@ -90,30 +91,29 @@ def _expand_keymap_target(keyboard: str, keymap: str, all_keyboards: List[str] =
             return [(kb, keymap) for kb in filter(lambda e: e is not None, parallel_map(keyboard_filter, all_keyboards))]
     else:
         if keymap == 'all':
-            keyboard = qmk.keyboard.resolve_keyboard(keyboard)
             cli.log.info(f'Retrieving list of keymaps for keyboard "{keyboard}"...')
             return _all_keymaps(keyboard)
         else:
-            return [(qmk.keyboard.resolve_keyboard(keyboard), keymap)]
+            return [(keyboard, keymap)]
 
 
 def expand_keymap_targets(targets: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
     """Expand a list of (keyboard, keymap) tuples inclusive of 'all', into a list of explicit (keyboard, keymap) tuples.
     """
     overall_targets = []
-    all_keyboards = qmk.keyboard.list_keyboards()
+    all_keyboards = list_keyboards()
     for target in targets:
         overall_targets.extend(_expand_keymap_target(target[0], target[1], all_keyboards))
     return list(sorted(set(overall_targets)))
 
 
-def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str] = [], print_vals: List[str] = []) -> List[Tuple[str, str, List[Tuple[str, str]]]]:
+def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str] = []) -> List[BuildTarget]:
     """Filter a list of (keyboard, keymap) tuples based on the supplied filters.
 
     Optionally includes the values of the queried info.json keys.
     """
-    if len(filters) == 0 and len(print_vals) == 0:
+    if len(filters) == 0:
-        targets = [(kb, km, {}) for kb, km in target_list]
+        targets = [KeyboardKeymapBuildTarget(keyboard=kb, keymap=km) for kb, km in target_list]
     else:
         cli.log.info('Parsing data for all matching keyboard/keymap combinations...')
         valid_keymaps = [(e[0], e[1], dotty(e[2])) for e in parallel_map(_load_keymap_info, target_list)]
@@ -172,18 +172,18 @@ def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str
                 cli.log.warning(f'Unrecognized filter expression: {filter_expr}')
                 continue
 
-            targets = [(e[0], e[1], [(p, e[2].get(p)) for p in print_vals]) for e in valid_keymaps]
+            targets = [KeyboardKeymapBuildTarget(keyboard=e[0], keymap=e[1], json=e[2]) for e in valid_keymaps]
 
     return targets
 
 
-def search_keymap_targets(targets: List[Tuple[str, str]] = [('all', 'default')], filters: List[str] = [], print_vals: List[str] = []) -> List[Tuple[str, str, List[Tuple[str, str]]]]:
+def search_keymap_targets(targets: List[Tuple[str, str]] = [('all', 'default')], filters: List[str] = []) -> List[BuildTarget]:
     """Search for build targets matching the supplied criteria.
     """
-    return list(sorted(_filter_keymap_targets(expand_keymap_targets(targets), filters, print_vals), key=lambda e: (e[0], e[1])))
+    return _filter_keymap_targets(expand_keymap_targets(targets), filters)
 
 
-def search_make_targets(targets: List[str], filters: List[str] = [], print_vals: List[str] = []) -> List[Tuple[str, str, List[Tuple[str, str]]]]:
+def search_make_targets(targets: List[str], filters: List[str] = []) -> List[BuildTarget]:
     """Search for build targets matching the supplied criteria.
     """
-    return list(sorted(_filter_keymap_targets(expand_make_targets(targets), filters, print_vals), key=lambda e: (e[0], e[1])))
+    return _filter_keymap_targets(expand_make_targets(targets), filters)