diff --git a/ci/drone-runner-docker/default.nix b/ci/drone-runner-docker/default.nix index 6ad4fa8..af634c6 100644 --- a/ci/drone-runner-docker/default.nix +++ b/ci/drone-runner-docker/default.nix @@ -1,7 +1,7 @@ { fetchFromGitHub, buildGoModule, - writeScript + writeScript, }: let source = builtins.fromJSON (builtins.readFile ./source.json); src = fetchFromGitHub { @@ -34,8 +34,8 @@ in } ]; }; - passthru.updateScript = writeScript "update-matrix-media-repo" '' - ${../../scripts/update-git.sh} "https://github.com/drone-runners/drone-runner-docker" ci/drone-runner-docker/source.json - SRC_PATH=$(nix-build -E '(import ./. {}).${pname}.src') - ${../../scripts/update-go.sh} ./ci/drone-runner-docker ci/drone-runner-docker/goVendor.hash ''; + passthru.updateScript = writeScript "update-matrix-media-repo" '' + ${../../scripts/update-git.sh} "https://github.com/drone-runners/drone-runner-docker" ci/drone-runner-docker/source.json + SRC_PATH=$(nix-build -E '(import ./. {}).${pname}.src') + ${../../scripts/update-go.sh} ./ci/drone-runner-docker ci/drone-runner-docker/goVendor.hash ''; } diff --git a/default.nix b/default.nix index f372cbd..273c568 100644 --- a/default.nix +++ b/default.nix @@ -33,4 +33,6 @@ matrix-media-repo = pkgs.callPackage ./matrix/matrix-media-repo {}; mautrix-discord = pkgs.callPackage ./matrix/mautrix-discord {}; mautrix-whatsapp = pkgs.callPackage ./matrix/mautrix-whatsapp {}; + mautrix-signal = pkgs.callPackage ./matrix/mautrix-signal {}; + python-mautrix = pkgs.python3Packages.callPackage ./python/mautrix.nix {}; } diff --git a/matrix/mautrix-signal.nix b/matrix/mautrix-signal.nix deleted file mode 100644 index 91c93a4..0000000 --- a/matrix/mautrix-signal.nix +++ /dev/null @@ -1,44 +0,0 @@ -{ - inputs, - pkgs, -}: -with pkgs; let - python-packages = import ../python/packages.nix {inherit inputs pkgs;}; -in { - mautrix-signal = with python3Packages; - buildPythonPackage { - pname = "mautrix-signal"; - version = inputs.mautrix-signal.lastModifiedDate; - src = inputs.mautrix-signal; - propagatedBuildInputs = [ - CommonMark - aiohttp - inputs.nixpkgs-stable.legacyPackages.${pkgs.system}.python310Packages.asyncpg - attrs - python-packages.mautrix - phonenumbers - pillow - prometheus-client - pycryptodome - python-olm - python-magic - qrcode - ruamel-yaml - unpaddedbase64 - yarl - ]; - doCheck = false; - postInstall = '' - mkdir -p $out/bin - # Make a little wrapper for running mautrix-signal with its dependencies - echo "$mautrixSignalScript" > $out/bin/mautrix-signal - echo "#!/bin/sh - exec python -m mautrix_signal \"\$@\" - " > $out/bin/mautrix-signal - chmod +x $out/bin/mautrix-signal - wrapProgram $out/bin/mautrix-signal \ - --set PATH ${python3}/bin \ - --set PYTHONPATH "$PYTHONPATH" - ''; - }; -} diff --git a/matrix/mautrix-signal/default.nix b/matrix/mautrix-signal/default.nix new file mode 100644 index 0000000..9b4b78b --- /dev/null +++ b/matrix/mautrix-signal/default.nix @@ -0,0 +1,65 @@ +{ + lib, + python3, + fetchFromGitHub, +}: let + source = builtins.fromJSON (builtins.readFile ./source.json); +in + python3.pkgs.buildPythonPackage rec { + pname = "mautrix-signal"; + version = source.date; + src = fetchFromGitHub { + owner = "mautrix"; + repo = "signal"; + inherit (source) rev sha256; + }; + propagatedBuildInputs = with python3.pkgs; [ + CommonMark + aiohttp + asyncpg + attrs + (python3.pkgs.callPackage ../../python/mautrix.nix {}) + phonenumbers + pillow + prometheus-client + pycryptodome + python-olm + python-magic + qrcode + ruamel-yaml + unpaddedbase64 + yarl + ]; + doCheck = false; + + postPatch = '' + substituteInPlace requirements.txt \ + --replace "asyncpg>=0.20,<0.26" "asyncpg>=0.20" \ + --replace "mautrix>=0.16.0,<0.17" "mautrix>=0.16.0" + ''; + + postInstall = '' + mkdir -p $out/bin + # Make a little wrapper for running mautrix-signal with its dependencies + echo "$mautrixSignalScript" > $out/bin/mautrix-signal + echo "#!/bin/sh + exec python -m mautrix_signal \"\$@\" + " > $out/bin/mautrix-signal + chmod +x $out/bin/mautrix-signal + wrapProgram $out/bin/mautrix-signal \ + --prefix PATH : "${python3}/bin" \ + --prefix PYTHONPATH : "$PYTHONPATH" + ''; + + meta = with lib; { + homepage = "https://github.com/mautrix/signal"; + description = "A Matrix-Signal puppeting bridge"; + license = licenses.agpl3Plus; + platforms = platforms.linux; + }; + passthru.updateScript = [ + ../../scripts/update-git.sh + "https://github.com/mautrix/signal" + "matrix/mautrix-signal/source.json" + ]; + } diff --git a/matrix/mautrix-signal/source.json b/matrix/mautrix-signal/source.json new file mode 100644 index 0000000..24e2855 --- /dev/null +++ b/matrix/mautrix-signal/source.json @@ -0,0 +1,11 @@ +{ + "url": "https://github.com/mautrix/signal", + "rev": "2798e463ca6774d8199944aa9dbc588d61b310bf", + "date": "2022-09-23T20:48:29+03:00", + "path": "/nix/store/n39d0y356xvm963f51y8n8bcjxra0mj3-signal", + "sha256": "1sq52p4q3iajm8zpvljzy0svms9x91r62sygr5c1r4g2cd71zv2m", + "fetchLFS": false, + "fetchSubmodules": false, + "deepClone": false, + "leaveDotGit": false +} diff --git a/python/mautrix.nix b/python/mautrix.nix new file mode 100644 index 0000000..05386fd --- /dev/null +++ b/python/mautrix.nix @@ -0,0 +1,46 @@ +{ + lib, + buildPythonPackage, + fetchPypi, + aiohttp, + pythonOlder, + sqlalchemy, + ruamel-yaml, + CommonMark, + lxml, + aiosqlite, +}: +buildPythonPackage rec { + pname = "mautrix"; + version = "0.18.2"; + + src = fetchPypi { + inherit pname version; + sha256 = "sha256-ZO50BmQgDlt7IyH5zWziHM+gLal22YJXj90GXygmGN0="; + }; + + propagatedBuildInputs = [ + aiohttp + + # defined in optional-requirements.txt + sqlalchemy + aiosqlite + ruamel-yaml + CommonMark + lxml + ]; + + disabled = pythonOlder "3.8"; + + # no tests available + doCheck = false; + + pythonImportsCheck = ["mautrix"]; + + meta = with lib; { + homepage = "https://github.com/tulir/mautrix-python"; + description = "A Python 3 asyncio Matrix framework."; + license = licenses.mpl20; + }; + passthru.updateScript = [../scripts/update-python-libraries "python/mautrix.nix"]; +} diff --git a/python/tarballs.nix b/python/tarballs.nix deleted file mode 100644 index 5142797..0000000 --- a/python/tarballs.nix +++ /dev/null @@ -1,61 +0,0 @@ -{ - inputs, - pkgs, -}: let - inherit (pkgs) fetchurl; -in rec { - plover-plugins-manager-src = fetchurl { - url = "https://files.pythonhosted.org/packages/6c/a1/2db1489c868fd9f88ce164c48bdbf9fcf6e13b7fd323ddba24cd0a7e2e94/plover_plugins_manager-0.7.1.tar.gz"; - sha256 = "fd18966c6c4fb66fb49a10e2d2178455bea204abb2066e883aadb2ae3d7b9fdb"; - passthru.pname = "plover-plugins-manager"; - passthru.version = "0.7.1"; - }; - plover-stroke-src = fetchurl { - url = "https://files.pythonhosted.org/packages/cc/53/92635d8bf00b883bfbc6ab9dd48b6df2ed01c241379fe99f063a41530cab/plover_stroke-1.1.0.tar.gz"; - sha256 = "de03b23f4aee66b65f69f7d4ecc4233681b43541502d86bf14fde29eaa72d153"; - passthru.pname = "plover-stroke"; - passthru.version = "1.1.0"; - }; - rtf-tokenize-src = fetchurl { - url = "https://files.pythonhosted.org/packages/1b/c3/591998d6a7e19c68933e8c4af3e8f5c0bbc17eb9c50a229c7c6afff349c4/rtf_tokenize-1.0.0.tar.gz"; - sha256 = "5c3df390d00479bd7637c823bfcd6fdfb21ddd1b96ae815463de7e1ed392d608"; - passthru.pname = "rtf-tokenize"; - passthru.version = "1.0.0"; - }; - plover-emoji-src = fetchurl { - url = "https://files.pythonhosted.org/packages/44/b0/1e8e677ee942a5817245731493c3541705aad5424a4a21ae4e3bea6a57b6/plover_emoji-0.0.4.tar.gz"; - sha256 = "db6611cd2a094859844b63f6ba2037df7929c4719457556586e9f6ab4f0b57ea"; - passthru.pname = "plover-emoji"; - passthru.version = "0.0.4"; - }; - plover-tapey-tape-src = fetchurl { - url = "https://files.pythonhosted.org/packages/cf/62/2bddc190fa18009a5e15d1f4efa642aed4a0e0a523c56223616d82ff05c0/plover_tapey_tape-0.0.5.tar.gz"; - sha256 = "226334e874fd9033aba58baf5d9d53523de691b2bba6aad597f71be82923a17c"; - passthru.pname = "plover-tapey-tape"; - passthru.version = "0.0.5"; - }; - plover-yaml-dictionary-src = fetchurl { - url = "https://files.pythonhosted.org/packages/07/2d/8b23162c1eee648afde6030f64b67a1e9a923ae88ed68dbe9a9c3aef7d14/plover_yaml_dictionary-0.0.1.tar.gz"; - sha256 = "12d9aad7ef5e93559ae5b0236a83b0bfb17697acd722f791cff58206e33023d6"; - passthru.pname = "plover-yaml-dictionary"; - passthru.version = "0.0.1"; - }; - simplefuzzyset-src = fetchurl { - url = "https://files.pythonhosted.org/packages/ce/bc/7e5d5eaa5566ade033cda9ff0eb51b0942ab2138288b445c469d2814cd2f/simplefuzzyset-0.0.12.tar.gz"; - sha256 = "9a1b30c38b6afb76c6600bdd66c1c1dc3d8505b082e9e3d466f60f40e8b7e1f2"; - passthru.pname = "simplefuzzyset"; - passthru.version = "0.0.12"; - }; - mautrix-src = fetchurl { - url = "https://files.pythonhosted.org/packages/37/1f/f0d75f3fc7abb101a622a0ee3b70a5b16ccc35f7ddde41df6f7e97f9d900/mautrix-0.18.2.tar.gz"; - sha256 = "64ee740664200e5b7b2321f9cd6ce21ccfa02da976d982578fdd065f282618dd"; - passthru.pname = "mautrix"; - passthru.version = "0.18.2"; - }; - tulir-telethon-src = fetchurl { - url = "https://files.pythonhosted.org/packages/5e/d6/9199b56df6b5e12672f49e724b19eb6a7e2d1936993203f8b73394f031a7/tulir-telethon-1.26.0a4.tar.gz"; - sha256 = "1fca3906b780a2351fda03fc00be00877cce337a096b49637572b093606525d5"; - passthru.pname = "tulir-telethon"; - passthru.version = "1.26.0a4"; - }; -} diff --git a/python/update.sh b/python/update.sh index 9446223..be31920 100755 --- a/python/update.sh +++ b/python/update.sh @@ -4,9 +4,7 @@ set -e PACKAGES="plover-plugins-manager plover-stroke rtf-tokenize plover-emoji plover-tapey-tape plover-yaml-dictionary simplefuzzyset mautrix tulir-telethon" cat > tarballs.nix << EOF -{ inputs, pkgs }: let - inherit (pkgs) fetchurl; -in rec { +{ fetchurl }: rec { EOF for package in $PACKAGES; do diff --git a/scripts/update-python-libraries b/scripts/update-python-libraries new file mode 100755 index 0000000..cc35ac7 --- /dev/null +++ b/scripts/update-python-libraries @@ -0,0 +1,5 @@ +#!/bin/sh +build=`nix-build -E "with import (fetchTarball "channel:nixpkgs-unstable") {}; python3.withPackages(ps: with ps; [ packaging requests toolz ])"` +python=${build}/bin/python +exec ${python} scripts/update-python-libraries.py $@ + diff --git a/scripts/update-python-libraries.py b/scripts/update-python-libraries.py new file mode 100755 index 0000000..3843497 --- /dev/null +++ b/scripts/update-python-libraries.py @@ -0,0 +1,474 @@ +#!/usr/bin/env python3 + +""" +Update a Python package expression by passing in the `.nix` file, or the directory containing it. +You can pass in multiple files or paths. + +You'll likely want to use +`` + $ ./update-python-libraries ../../pkgs/development/python-modules/**/default.nix +`` +to update all non-pinned libraries in that folder. +""" + +import argparse +import os +import pathlib +import re +import requests +from concurrent.futures import ThreadPoolExecutor as Pool +from packaging.version import Version as _Version +from packaging.version import InvalidVersion +from packaging.specifiers import SpecifierSet +import collections +import subprocess + +INDEX = "https://pypi.io/pypi" +"""url of PyPI""" + +EXTENSIONS = ['tar.gz', 'tar.bz2', 'tar', 'zip', '.whl'] +"""Permitted file extensions. These are evaluated from left to right and the first occurance is returned.""" + +PRERELEASES = False + +GIT = "git" + +NIXPGKS_ROOT = subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).decode('utf-8').strip() + +import logging +logging.basicConfig(level=logging.INFO) + + +class Version(_Version, collections.abc.Sequence): + + def __init__(self, version): + super().__init__(version) + # We cannot use `str(Version(0.04.21))` because that becomes `0.4.21` + # https://github.com/avian2/unidecode/issues/13#issuecomment-354538882 + self.raw_version = version + + def __getitem__(self, i): + return self._version.release[i] + + def __len__(self): + return len(self._version.release) + + def __iter__(self): + yield from self._version.release + + +def _get_values(attribute, text): + """Match attribute in text and return all matches. + + :returns: List of matches. + """ + regex = '{}\s+=\s+"(.*)";'.format(attribute) + regex = re.compile(regex) + values = regex.findall(text) + return values + +def _get_unique_value(attribute, text): + """Match attribute in text and return unique match. + + :returns: Single match. + """ + values = _get_values(attribute, text) + n = len(values) + if n > 1: + raise ValueError("found too many values for {}".format(attribute)) + elif n == 1: + return values[0] + else: + raise ValueError("no value found for {}".format(attribute)) + +def _get_line_and_value(attribute, text): + """Match attribute in text. Return the line and the value of the attribute.""" + regex = '({}\s+=\s+"(.*)";)'.format(attribute) + regex = re.compile(regex) + value = regex.findall(text) + n = len(value) + if n > 1: + raise ValueError("found too many values for {}".format(attribute)) + elif n == 1: + return value[0] + else: + raise ValueError("no value found for {}".format(attribute)) + + +def _replace_value(attribute, value, text): + """Search and replace value of attribute in text.""" + old_line, old_value = _get_line_and_value(attribute, text) + new_line = old_line.replace(old_value, value) + new_text = text.replace(old_line, new_line) + return new_text + + +def _fetch_page(url): + r = requests.get(url) + if r.status_code == requests.codes.ok: + return r.json() + else: + raise ValueError("request for {} failed".format(url)) + + +def _fetch_github(url): + headers = {} + token = os.environ.get('GITHUB_API_TOKEN') + if token: + headers["Authorization"] = f"token {token}" + r = requests.get(url, headers=headers) + + if r.status_code == requests.codes.ok: + return r.json() + else: + raise ValueError("request for {} failed".format(url)) + + +SEMVER = { + 'major' : 0, + 'minor' : 1, + 'patch' : 2, +} + + +def _determine_latest_version(current_version, target, versions): + """Determine latest version, given `target`. + """ + current_version = Version(current_version) + + def _parse_versions(versions): + for v in versions: + try: + yield Version(v) + except InvalidVersion: + pass + + versions = _parse_versions(versions) + + index = SEMVER[target] + + ceiling = list(current_version[0:index]) + if len(ceiling) == 0: + ceiling = None + else: + ceiling[-1]+=1 + ceiling = Version(".".join(map(str, ceiling))) + + # We do not want prereleases + versions = SpecifierSet(prereleases=PRERELEASES).filter(versions) + + if ceiling is not None: + versions = SpecifierSet(f"<{ceiling}").filter(versions) + + return (max(sorted(versions))).raw_version + + +def _get_latest_version_pypi(package, extension, current_version, target): + """Get latest version and hash from PyPI.""" + url = "{}/{}/json".format(INDEX, package) + json = _fetch_page(url) + + versions = json['releases'].keys() + version = _determine_latest_version(current_version, target, versions) + + try: + releases = json['releases'][version] + except KeyError as e: + raise KeyError('Could not find version {} for {}'.format(version, package)) from e + for release in releases: + if release['filename'].endswith(extension): + # TODO: In case of wheel we need to do further checks! + sha256 = release['digests']['sha256'] + break + else: + sha256 = None + return version, sha256, None + + +def _get_latest_version_github(package, extension, current_version, target): + def strip_prefix(tag): + return re.sub("^[^0-9]*", "", tag) + + def get_prefix(string): + matches = re.findall(r"^([^0-9]*)", string) + return next(iter(matches), "") + + # when invoked as an updateScript, UPDATE_NIX_ATTR_PATH will be set + # this allows us to work with packages which live outside of python-modules + attr_path = os.environ.get("UPDATE_NIX_ATTR_PATH", f"python3Packages.{package}") + try: + homepage = subprocess.check_output( + ["nix", "eval", "-f", f"{NIXPGKS_ROOT}/default.nix", "--raw", f"{attr_path}.src.meta.homepage"])\ + .decode('utf-8') + except Exception as e: + raise ValueError(f"Unable to determine homepage: {e}") + owner_repo = homepage[len("https://github.com/"):] # remove prefix + owner, repo = owner_repo.split("/") + + url = f"https://api.github.com/repos/{owner}/{repo}/releases" + all_releases = _fetch_github(url) + releases = list(filter(lambda x: not x['prerelease'], all_releases)) + + if len(releases) == 0: + raise ValueError(f"{homepage} does not contain any stable releases") + + versions = map(lambda x: strip_prefix(x['tag_name']), releases) + version = _determine_latest_version(current_version, target, versions) + + release = next(filter(lambda x: strip_prefix(x['tag_name']) == version, releases)) + prefix = get_prefix(release['tag_name']) + try: + sha256 = subprocess.check_output(["nix-prefetch-url", "--type", "sha256", "--unpack", f"{release['tarball_url']}"], stderr=subprocess.DEVNULL)\ + .decode('utf-8').strip() + except: + # this may fail if they have both a branch and a tag of the same name, attempt tag name + tag_url = str(release['tarball_url']).replace("tarball","tarball/refs/tags") + sha256 = subprocess.check_output(["nix-prefetch-url", "--type", "sha256", "--unpack", tag_url], stderr=subprocess.DEVNULL)\ + .decode('utf-8').strip() + + + return version, sha256, prefix + + +FETCHERS = { + 'fetchFromGitHub' : _get_latest_version_github, + 'fetchPypi' : _get_latest_version_pypi, + 'fetchurl' : _get_latest_version_pypi, +} + + +DEFAULT_SETUPTOOLS_EXTENSION = 'tar.gz' + + +FORMATS = { + 'setuptools' : DEFAULT_SETUPTOOLS_EXTENSION, + 'wheel' : 'whl', + 'pyproject' : 'tar.gz', + 'flit' : 'tar.gz' +} + +def _determine_fetcher(text): + # Count occurences of fetchers. + nfetchers = sum(text.count('src = {}'.format(fetcher)) for fetcher in FETCHERS.keys()) + if nfetchers == 0: + raise ValueError("no fetcher.") + elif nfetchers > 1: + raise ValueError("multiple fetchers.") + else: + # Then we check which fetcher to use. + for fetcher in FETCHERS.keys(): + if 'src = {}'.format(fetcher) in text: + return fetcher + + +def _determine_extension(text, fetcher): + """Determine what extension is used in the expression. + + If we use: + - fetchPypi, we check if format is specified. + - fetchurl, we determine the extension from the url. + - fetchFromGitHub we simply use `.tar.gz`. + """ + if fetcher == 'fetchPypi': + try: + src_format = _get_unique_value('format', text) + except ValueError as e: + src_format = None # format was not given + + try: + extension = _get_unique_value('extension', text) + except ValueError as e: + extension = None # extension was not given + + if extension is None: + if src_format is None: + src_format = 'setuptools' + elif src_format == 'other': + raise ValueError("Don't know how to update a format='other' package.") + extension = FORMATS[src_format] + + elif fetcher == 'fetchurl': + url = _get_unique_value('url', text) + extension = os.path.splitext(url)[1] + if 'pypi' not in url: + raise ValueError('url does not point to PyPI.') + + elif fetcher == 'fetchFromGitHub': + if "fetchSubmodules" in text: + raise ValueError("fetchFromGitHub fetcher doesn't support submodules") + extension = "tar.gz" + + return extension + + +def _update_package(path, target): + + # Read the expression + with open(path, 'r') as f: + text = f.read() + + # Determine pname. Many files have more than one pname + pnames = _get_values('pname', text) + + # Determine version. + version = _get_unique_value('version', text) + + # First we check how many fetchers are mentioned. + fetcher = _determine_fetcher(text) + + extension = _determine_extension(text, fetcher) + + # Attempt a fetch using each pname, e.g. backports-zoneinfo vs backports.zoneinfo + successful_fetch = False + for pname in pnames: + try: + new_version, new_sha256, prefix = FETCHERS[fetcher](pname, extension, version, target) + successful_fetch = True + break + except ValueError: + continue + + if not successful_fetch: + raise ValueError(f"Unable to find correct package using these pnames: {pnames}") + + if new_version == version: + logging.info("Path {}: no update available for {}.".format(path, pname)) + return False + elif Version(new_version) <= Version(version): + raise ValueError("downgrade for {}.".format(pname)) + if not new_sha256: + raise ValueError("no file available for {}.".format(pname)) + + text = _replace_value('version', new_version, text) + # hashes from pypi are 16-bit encoded sha256's, normalize it to sri to avoid merge conflicts + # sri hashes have been the default format since nix 2.4+ + try: + sri_hash = subprocess.check_output(["nix", "hash", "to-sri", "--type", "sha256", new_sha256]).decode('utf-8').strip() + except subprocess.CalledProcessError: + # nix<2.4 compat + sri_hash = subprocess.check_output(["nix", "to-sri", "--type", "sha256", new_sha256]).decode('utf-8').strip() + + + # fetchers can specify a sha256, or a sri hash + try: + text = _replace_value('sha256', sri_hash, text) + except ValueError: + text = _replace_value('hash', sri_hash, text) + + if fetcher == 'fetchFromGitHub': + # in the case of fetchFromGitHub, it's common to see `rev = version;` or `rev = "v${version}";` + # in which no string value is meant to be substituted. However, we can just overwrite the previous value. + regex = '(rev\s+=\s+[^;]*;)' + regex = re.compile(regex) + matches = regex.findall(text) + n = len(matches) + + if n == 0: + raise ValueError("Unable to find rev value for {}.".format(pname)) + else: + # forcefully rewrite rev, incase tagging conventions changed for a release + match = matches[0] + text = text.replace(match, f'rev = "refs/tags/{prefix}${{version}}";') + # incase there's no prefix, just rewrite without interpolation + text = text.replace('"${version}";', 'version;') + + with open(path, 'w') as f: + f.write(text) + + logging.info("Path {}: updated {} from {} to {}".format(path, pname, version, new_version)) + + result = { + 'path' : path, + 'target': target, + 'pname': pname, + 'old_version' : version, + 'new_version' : new_version, + #'fetcher' : fetcher, + } + + return result + + +def _update(path, target): + + # We need to read and modify a Nix expression. + if os.path.isdir(path): + path = os.path.join(path, 'default.nix') + + # If a default.nix does not exist, we quit. + if not os.path.isfile(path): + logging.info("Path {}: does not exist.".format(path)) + return False + + # If file is not a Nix expression, we quit. + if not path.endswith(".nix"): + logging.info("Path {}: does not end with `.nix`.".format(path)) + return False + + try: + return _update_package(path, target) + except ValueError as e: + logging.warning("Path {}: {}".format(path, e)) + return False + + +def _commit(path, pname, old_version, new_version, pkgs_prefix="python: ", **kwargs): + """Commit result. + """ + + msg = f'{pkgs_prefix}{pname}: {old_version} -> {new_version}' + + try: + subprocess.check_call([GIT, 'add', path]) + subprocess.check_call([GIT, 'commit', '-m', msg]) + except subprocess.CalledProcessError as e: + subprocess.check_call([GIT, 'checkout', path]) + raise subprocess.CalledProcessError(f'Could not commit {path}') from e + + return True + + +def main(): + + epilog = """ +environment variables: + GITHUB_API_TOKEN\tGitHub API token used when updating github packages + """ + parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter, epilog=epilog) + parser.add_argument('package', type=str, nargs='+') + parser.add_argument('--target', type=str, choices=SEMVER.keys(), default='major') + parser.add_argument('--commit', action='store_true', help='Create a commit for each package update') + parser.add_argument('--use-pkgs-prefix', action='store_true', help='Use python3Packages.${pname}: instead of python: ${pname}: when making commits') + + args = parser.parse_args() + target = args.target + + packages = list(map(os.path.abspath, args.package)) + + logging.info("Updating packages...") + + # Use threads to update packages concurrently + with Pool() as p: + results = list(filter(bool, p.map(lambda pkg: _update(pkg, target), packages))) + + logging.info("Finished updating packages.") + + commit_options = {} + if args.use_pkgs_prefix: + logging.info("Using python3Packages. prefix for commits") + commit_options["pkgs_prefix"] = "python3Packages." + + # Commits are created sequentially. + if args.commit: + logging.info("Committing updates...") + # list forces evaluation + list(map(lambda x: _commit(**x, **commit_options), results)) + logging.info("Finished committing updates") + + count = len(results) + logging.info("{} package(s) updated".format(count)) + + + +if __name__ == '__main__': + main()