From 9af28045cb9cd2f1defd61101c1ef9323da95660 Mon Sep 17 00:00:00 2001 From: "Jens W. Klein" Date: Fri, 31 Oct 2025 23:03:24 +0100 Subject: [PATCH] Add support for multiple push URLs in Git repositories Enables pushing to multiple remotes (e.g., GitHub + GitLab mirrors) by configuring multiple push URLs using multiline syntax, consistent with version-overrides and ignores patterns. Implementation: - config.py: Add parse_multiline_list() to parse multiline pushurl values - config.py: Detect and store multiple pushurls as list with backward compat - vcs/git.py: Update git_set_pushurl() to use --add flag for additional URLs - First pushurl set without --add, subsequent ones with --add Configuration example: ```ini [my-package] url = https://github.com/org/repo.git pushurl = git@github.com:org/repo.git git@gitlab.com:org/repo.git git@bitbucket.org:org/repo.git ``` Backward compatibility: - Single pushurl: Works unchanged (no pushurls list created) - Multiple pushurls: New multiline syntax (pushurls list created) - First pushurl stored in pushurl key for backward compat - Smart threading logic unchanged (checks for pushurl presence) Tests: - test_config_parse_multiple_pushurls: Config parsing validation - test_git_set_pushurl_multiple: Git command sequence verification - test_git_checkout_with_multiple_pushurls: End-to-end integration All 197 tests pass (194 existing + 3 new). All linting checks pass. Documentation updated in README.md and CHANGES.md. --- CHANGES.md | 2 + README.md | 19 ++++++- src/mxdev/config.py | 30 +++++++++++ src/mxdev/vcs/git.py | 44 +++++++++++++++-- tests/test_config.py | 39 +++++++++++++++ tests/test_git_additional.py | 96 ++++++++++++++++++++++++++++++++++++ 6 files changed, 225 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 3bc2a65..cfcc286 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ## 5.0.2 (2025-10-23) +- Feature: Git repositories can now specify multiple push URLs using multiline syntax in the `pushurl` configuration option. This enables pushing to multiple remotes (e.g., GitHub + GitLab mirrors) automatically. Syntax follows the same multiline pattern as `version-overrides` and `ignores`. Example: `pushurl =` followed by indented URLs on separate lines. When `git push` is run in the checked-out repository, it will push to all configured pushurls sequentially, mirroring Git's native multi-pushurl behavior. Backward compatible with single pushurl strings. + [jensens] - Feature: Added `--version` command-line option to display the current mxdev version. The version is automatically derived from git tags via hatch-vcs during build. Example: `mxdev --version` outputs "mxdev 5.0.1" for releases or "mxdev 5.0.1.dev27+g62877d7" for development versions. [jensens] - Fix #70: HTTP-referenced requirements/constraints files are now properly cached and respected in offline mode. Previously, offline mode only skipped VCS operations but still fetched HTTP URLs. Now mxdev caches all HTTP content in `.mxdev_cache/` during online mode and reuses it during offline mode, enabling true offline operation. This fixes the inconsistent behavior where `-o/--offline` didn't prevent all network activity. diff --git a/README.md b/README.md index e4fa05e..21ae6a3 100644 --- a/README.md +++ b/README.md @@ -239,7 +239,7 @@ For package sources, the section name is the package name: `[PACKAGENAME]` | `extras` | optional | Comma-separated package extras (e.g., `test,dev`) | empty | | `subdirectory` | optional | Path to Python package when not in repository root | empty | | `target` | optional | Custom target directory (overrides `default-target`) | `default-target` | -| `pushurl` | optional | Writable URL for pushes (not applied after initial checkout) | — | +| `pushurl` | optional | Writable URL(s) for pushes. Supports single URL or multiline list for pushing to multiple remotes. Not applied after initial checkout. | — | **VCS Support Status:** - `git` (stable, tested) @@ -266,6 +266,23 @@ For package sources, the section name is the package name: `[PACKAGENAME]` - **`checkout`**: Submodules only fetched during checkout, existing submodules stay untouched - **`recursive`**: Fetches submodules recursively, results in `git clone --recurse-submodules` on checkout and `submodule update --init --recursive` on update +##### Multiple Push URLs + +You can configure a package to push to multiple remotes (e.g., mirroring to GitHub and GitLab): + +```ini +[my-package] +url = https://github.com/org/repo.git +pushurl = + git@github.com:org/repo.git + git@gitlab.com:org/repo.git + git@bitbucket.org:org/repo.git +``` + +When you run `git push` in the checked-out repository, Git will push to all configured pushurls sequentially. + +**Note:** Multiple pushurls only work with the `git` VCS type. This mirrors Git's native behavior where a remote can have multiple push URLs. + ### Usage Run `mxdev` (for more options run `mxdev --help`). diff --git a/src/mxdev/config.py b/src/mxdev/config.py index a8db4c1..2dea2e0 100644 --- a/src/mxdev/config.py +++ b/src/mxdev/config.py @@ -16,6 +16,26 @@ def to_bool(value): return value.lower() in ("true", "on", "yes", "1") +def parse_multiline_list(value: str) -> list[str]: + """Parse a multiline configuration value into a list of non-empty strings. + + Handles multiline format where items are separated by newlines: + value = " + item1 + item2 + item3" + + Returns a list of non-empty, stripped strings. + """ + if not value: + return [] + + # Split by newlines and strip whitespace + items = [line.strip() for line in value.strip().splitlines()] + # Filter out empty lines + return [item for item in items if item] + + class Configuration: settings: dict[str, str] overrides: dict[str, str] @@ -125,6 +145,16 @@ def is_ns_member(name) -> bool: if not package.get("url"): raise ValueError(f"Section {name} has no URL set!") + # Special handling for pushurl to support multiple values + if "pushurl" in package: + pushurls = parse_multiline_list(package["pushurl"]) + if len(pushurls) > 1: + # Store as list for multiple pushurls + package["pushurls"] = pushurls + # Keep first one in "pushurl" for backward compatibility + package["pushurl"] = pushurls[0] + # If single pushurl, leave as-is (no change to existing behavior) + # Handle deprecated "direct" mode for per-package install-mode pkg_mode = package.get("install-mode") if pkg_mode == "direct": diff --git a/src/mxdev/vcs/git.py b/src/mxdev/vcs/git.py index 4d5f420..07ecb89 100644 --- a/src/mxdev/vcs/git.py +++ b/src/mxdev/vcs/git.py @@ -349,21 +349,57 @@ def update(self, **kwargs) -> str | None: return self.git_update(**kwargs) def git_set_pushurl(self, stdout_in, stderr_in) -> tuple[str, str]: + """Set one or more push URLs for the remote. + + Supports both single pushurl (backward compat) and multiple pushurls. + """ + # Check for multiple pushurls (new format) + pushurls = self.source.get("pushurls", []) + + # Fallback to single pushurl (backward compat) + if not pushurls and "pushurl" in self.source: + pushurls = [self.source["pushurl"]] + + if not pushurls: + return (stdout_in, stderr_in) + + # Set first pushurl (without --add) cmd = self.run_git( [ "config", f"remote.{self._upstream_name}.pushurl", - self.source["pushurl"], + pushurls[0], ], cwd=self.source["path"], ) stdout, stderr = cmd.communicate() if cmd.returncode != 0: - raise GitError( - "git config remote.{}.pushurl {} \nfailed.\n".format(self._upstream_name, self.source["pushurl"]) + raise GitError(f"git config remote.{self._upstream_name}.pushurl {pushurls[0]} \nfailed.\n") + + stdout_in += stdout + stderr_in += stderr + + # Add additional pushurls with --add flag + for pushurl in pushurls[1:]: + cmd = self.run_git( + [ + "config", + "--add", + f"remote.{self._upstream_name}.pushurl", + pushurl, + ], + cwd=self.source["path"], ) - return (stdout_in + stdout, stderr_in + stderr) + stdout, stderr = cmd.communicate() + + if cmd.returncode != 0: + raise GitError(f"git config --add remote.{self._upstream_name}.pushurl {pushurl} \nfailed.\n") + + stdout_in += stdout + stderr_in += stderr + + return (stdout_in, stderr_in) def git_init_submodules(self, stdout_in, stderr_in) -> tuple[str, str, list]: cmd = self.run_git(["submodule", "init"], cwd=self.source["path"]) diff --git a/tests/test_config.py b/tests/test_config.py index 703d4f4..3fb37b4 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -272,3 +272,42 @@ def test_per_package_target_override(): pathlib.Path(pkg_interpolated["path"]).as_posix() == pathlib.Path(pkg_interpolated["target"]).joinpath("package.with.interpolated.target").as_posix() ) + + +def test_config_parse_multiple_pushurls(tmp_path): + """Test configuration parsing of multiple pushurls.""" + from mxdev.config import Configuration + + config_content = """ +[settings] +requirements-in = requirements.txt + +[package1] +url = https://github.com/test/repo.git +pushurl = + git@github.com:test/repo.git + git@gitlab.com:test/repo.git + git@bitbucket.org:test/repo.git + +[package2] +url = https://github.com/test/repo2.git +pushurl = git@github.com:test/repo2.git +""" + + config_file = tmp_path / "mx.ini" + config_file.write_text(config_content) + + config = Configuration(str(config_file)) + + # package1 should have multiple pushurls + assert "pushurls" in config.packages["package1"] + assert len(config.packages["package1"]["pushurls"]) == 3 + assert config.packages["package1"]["pushurls"][0] == "git@github.com:test/repo.git" + assert config.packages["package1"]["pushurls"][1] == "git@gitlab.com:test/repo.git" + assert config.packages["package1"]["pushurls"][2] == "git@bitbucket.org:test/repo.git" + # First pushurl should be kept for backward compatibility + assert config.packages["package1"]["pushurl"] == "git@github.com:test/repo.git" + + # package2 should have single pushurl (no pushurls list) + assert "pushurls" not in config.packages["package2"] + assert config.packages["package2"]["pushurl"] == "git@github.com:test/repo2.git" diff --git a/tests/test_git_additional.py b/tests/test_git_additional.py index f13b108..1bf514a 100644 --- a/tests/test_git_additional.py +++ b/tests/test_git_additional.py @@ -1007,3 +1007,99 @@ def test_smart_threading_separates_https_with_pushurl(): # HTTPS with pushurl, SSH, and fs should be in parallel queue assert set(other_pkgs) == {"https-with-pushurl", "ssh-url", "fs-url"} + + +def test_git_set_pushurl_multiple(): + """Test git_set_pushurl with multiple URLs.""" + from mxdev.vcs.git import GitWorkingCopy + from unittest.mock import Mock + from unittest.mock import patch + + with patch("mxdev.vcs.common.which", return_value="/usr/bin/git"): + source = { + "name": "test-package", + "url": "https://github.com/test/repo.git", + "path": "/tmp/test", + "pushurls": [ + "git@github.com:test/repo.git", + "git@gitlab.com:test/repo.git", + ], + "pushurl": "git@github.com:test/repo.git", + } + + wc = GitWorkingCopy(source) + + mock_process = Mock() + mock_process.returncode = 0 + mock_process.communicate.return_value = (b"output", b"") + + with patch.object(wc, "run_git", return_value=mock_process) as mock_git: + stdout, stderr = wc.git_set_pushurl(b"", b"") + + # Should be called twice + assert mock_git.call_count == 2 + + # First call: without --add + first_call_args = mock_git.call_args_list[0][0][0] + assert first_call_args == [ + "config", + "remote.origin.pushurl", + "git@github.com:test/repo.git", + ] + + # Second call: with --add + second_call_args = mock_git.call_args_list[1][0][0] + assert second_call_args == [ + "config", + "--add", + "remote.origin.pushurl", + "git@gitlab.com:test/repo.git", + ] + + +def test_git_checkout_with_multiple_pushurls(tempdir): + """Test git_checkout with multiple pushurls.""" + from mxdev.vcs.git import GitWorkingCopy + from unittest.mock import Mock + from unittest.mock import patch + + with patch("mxdev.vcs.common.which", return_value="/usr/bin/git"): + source = { + "name": "test-package", + "url": "https://github.com/test/repo.git", + "path": str(tempdir / "test-multi-pushurl"), + "pushurls": [ + "git@github.com:test/repo.git", + "git@gitlab.com:test/repo.git", + "git@bitbucket.org:test/repo.git", + ], + "pushurl": "git@github.com:test/repo.git", # First one for compat + } + + wc = GitWorkingCopy(source) + + mock_process = Mock() + mock_process.returncode = 0 + mock_process.communicate.return_value = (b"", b"") + + with patch.object(wc, "run_git", return_value=mock_process) as mock_git: + with patch("os.path.exists", return_value=False): + wc.git_checkout(submodules="never") + + # Verify git config was called 3 times for pushurls + config_calls = [call for call in mock_git.call_args_list if "config" in call[0][0]] + + # Should have 3 config calls for the 3 pushurls + pushurl_config_calls = [call for call in config_calls if "pushurl" in " ".join(call[0][0])] + assert len(pushurl_config_calls) == 3 + + # First call should be without --add + assert "--add" not in pushurl_config_calls[0][0][0] + assert "git@github.com:test/repo.git" in pushurl_config_calls[0][0][0] + + # Second and third calls should have --add + assert "--add" in pushurl_config_calls[1][0][0] + assert "git@gitlab.com:test/repo.git" in pushurl_config_calls[1][0][0] + + assert "--add" in pushurl_config_calls[2][0][0] + assert "git@bitbucket.org:test/repo.git" in pushurl_config_calls[2][0][0]