@@ -16,6 +16,7 @@
Skipping contrib/automation/hgautomation/aws.py it has no-che?k-code (glob)
Skipping contrib/automation/hgautomation/cli.py it has no-che?k-code (glob)
Skipping contrib/automation/hgautomation/linux.py it has no-che?k-code (glob)
+ Skipping contrib/automation/hgautomation/pypi.py it has no-che?k-code (glob)
Skipping contrib/automation/hgautomation/ssh.py it has no-che?k-code (glob)
Skipping contrib/automation/hgautomation/windows.py it has no-che?k-code (glob)
Skipping contrib/automation/hgautomation/winrm.py it has no-che?k-code (glob)
@@ -1,3 +1,4 @@
boto3
paramiko
pypsrp
+twine
@@ -26,6 +26,10 @@
--hash=sha256:d7bdc26475679dd073ba0ed2766445bb5b20ca4793ca0db32b399dccc6bc84b7 \
--hash=sha256:ff032765bb8716d9387fd5376d987a937254b0619eff0972779515b5c98820bc \
# via paramiko
+bleach==3.1.0 \
+ --hash=sha256:213336e49e102af26d9cde77dd2d0397afabc5a6bf2fed985dc35b5d1e285a16 \
+ --hash=sha256:3fdf7f77adcf649c9911387df51254b813185e32b2c6619f690b593a617e19fa \
+ # via readme-renderer
boto3==1.9.223 \
--hash=sha256:12ceb047c3cfbd2363b35e1c24b082808a1bb9b90f4f0b7375e83d21015bf47b \
--hash=sha256:6e833a9068309c24d7752e280b2925cf5968a88111bc95fcebc451a09f8b424e
@@ -93,7 +97,7 @@
--hash=sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0 \
--hash=sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827 \
--hash=sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99 \
- # via botocore
+ # via botocore, readme-renderer
idna==2.8 \
--hash=sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407 \
--hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c \
@@ -109,9 +113,17 @@
paramiko==2.6.0 \
--hash=sha256:99f0179bdc176281d21961a003ffdb2ec369daac1a1007241f53374e376576cf \
--hash=sha256:f4b2edfa0d226b70bd4ca31ea7e389325990283da23465d572ed1f70a7583041
+pkginfo==1.5.0.1 \
+ --hash=sha256:7424f2c8511c186cd5424bbf31045b77435b37a8d604990b79d4e70d741148bb \
+ --hash=sha256:a6d9e40ca61ad3ebd0b72fbadd4fba16e4c0e4df0428c041e01e06eb6ee71f32 \
+ # via twine
pycparser==2.19 \
--hash=sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3 \
# via cffi
+pygments==2.4.2 \
+ --hash=sha256:71e430bc85c88a430f000ac1d9b331d2407f681d6f6aec95e8bcfbc3df5b0127 \
+ --hash=sha256:881c4c157e45f30af185c1ffe8d549d48ac9127433f2c380c24b84572ad66297 \
+ # via readme-renderer
pynacl==1.3.0 \
--hash=sha256:05c26f93964373fc0abe332676cb6735f0ecad27711035b9472751faa8521255 \
--hash=sha256:0c6100edd16fefd1557da078c7a31e7b7d7a52ce39fdca2bec29d4f7b6e7600c \
@@ -140,10 +152,18 @@
--hash=sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb \
--hash=sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e \
# via botocore
+readme-renderer==24.0 \
+ --hash=sha256:bb16f55b259f27f75f640acf5e00cf897845a8b3e4731b5c1a436e4b8529202f \
+ --hash=sha256:c8532b79afc0375a85f10433eca157d6b50f7d6990f337fa498c96cd4bfc203d \
+ # via twine
+requests-toolbelt==0.9.1 \
+ --hash=sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f \
+ --hash=sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0 \
+ # via twine
requests==2.22.0 \
--hash=sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4 \
--hash=sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31 \
- # via pypsrp
+ # via pypsrp, requests-toolbelt, twine
s3transfer==0.2.1 \
--hash=sha256:6efc926738a3cd576c2a79725fed9afde92378aa5c6a957e3af010cb019fac9d \
--hash=sha256:b780f2411b824cb541dbcd2c713d0cb61c7d1bcadae204cdddda2b35cef493ba \
@@ -151,8 +171,23 @@
six==1.12.0 \
--hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c \
--hash=sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73 \
- # via bcrypt, cryptography, pynacl, pypsrp, python-dateutil
+ # via bcrypt, bleach, cryptography, pynacl, pypsrp, python-dateutil, readme-renderer
+tqdm==4.35.0 \
+ --hash=sha256:1be3e4e3198f2d0e47b928e9d9a8ec1b63525db29095cec1467f4c5a4ea8ebf9 \
+ --hash=sha256:7e39a30e3d34a7a6539378e39d7490326253b7ee354878a92255656dc4284457 \
+ # via twine
+twine==1.13.0 \
+ --hash=sha256:0fb0bfa3df4f62076cab5def36b1a71a2e4acb4d1fa5c97475b048117b1a6446 \
+ --hash=sha256:d6c29c933ecfc74e9b1d9fa13aa1f87c5d5770e119f5a4ce032092f0ff5b14dc
urllib3==1.25.3 \
--hash=sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1 \
--hash=sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232 \
# via botocore, requests
+webencodings==0.5.1 \
+ --hash=sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78 \
+ --hash=sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923 \
+ # via bleach
+
+# WARNING: The following packages were not pinned, but pip requires them to be
+# pinned when the requirements file includes hashes. Consider using the --allow-unsafe flag.
+# setuptools==41.2.0 # via twine
@@ -7,12 +7,17 @@
# no-check-code because Python 3 native.
+import datetime
import os
+import paramiko
import pathlib
import re
import subprocess
import tempfile
+from .pypi import (
+ upload as pypi_upload,
+)
from .winrm import (
run_powershell,
)
@@ -100,6 +105,26 @@
}}
'''
+X86_WHEEL_FILENAME = 'mercurial-{version}-cp27-cp27m-win32.whl'
+X64_WHEEL_FILENAME = 'mercurial-{version}-cp27-cp27m-win_amd64.whl'
+X86_EXE_FILENAME = 'Mercurial-{version}.exe'
+X64_EXE_FILENAME = 'Mercurial-{version}-x64.exe'
+X86_MSI_FILENAME = 'mercurial-{version}-x86.msi'
+X64_MSI_FILENAME = 'mercurial-{version}-x64.msi'
+
+MERCURIAL_SCM_BASE_URL = 'https://mercurial-scm.org/release/windows'
+
+X86_USER_AGENT_PATTERN = '.*Windows.*'
+X64_USER_AGENT_PATTERN = '.*Windows.*(WOW|x)64.*'
+
+X86_EXE_DESCRIPTION = ('Mercurial {version} Inno Setup installer - x86 Windows '
+ '- does not require admin rights')
+X64_EXE_DESCRIPTION = ('Mercurial {version} Inno Setup installer - x64 Windows '
+ '- does not require admin rights')
+X86_MSI_DESCRIPTION = ('Mercurial {version} MSI installer - x86 Windows '
+ '- requires admin rights')
+X64_MSI_DESCRIPTION = ('Mercurial {version} MSI installer - x64 Windows '
+ '- requires admin rights')
def get_vc_prefix(arch):
if arch == 'x86':
@@ -296,3 +321,152 @@
)
run_powershell(winrm_client, ps)
+
+
+def resolve_wheel_artifacts(dist_path: pathlib.Path, version: str):
+ return (
+ dist_path / X86_WHEEL_FILENAME.format(version=version),
+ dist_path / X64_WHEEL_FILENAME.format(version=version),
+ )
+
+
+def resolve_all_artifacts(dist_path: pathlib.Path, version: str):
+ return (
+ dist_path / X86_WHEEL_FILENAME.format(version=version),
+ dist_path / X64_WHEEL_FILENAME.format(version=version),
+ dist_path / X86_EXE_FILENAME.format(version=version),
+ dist_path / X64_EXE_FILENAME.format(version=version),
+ dist_path / X86_MSI_FILENAME.format(version=version),
+ dist_path / X64_MSI_FILENAME.format(version=version),
+ )
+
+
+def generate_latest_dat(version: str):
+ x86_exe_filename = X86_EXE_FILENAME.format(version=version)
+ x64_exe_filename = X64_EXE_FILENAME.format(version=version)
+ x86_msi_filename = X86_MSI_FILENAME.format(version=version)
+ x64_msi_filename = X64_MSI_FILENAME.format(version=version)
+
+ entries = (
+ (
+ '10',
+ version,
+ X86_USER_AGENT_PATTERN,
+ '%s/%s' % (MERCURIAL_SCM_BASE_URL, x86_exe_filename),
+ X86_EXE_DESCRIPTION.format(version=version),
+ ),
+ (
+ '10',
+ version,
+ X64_USER_AGENT_PATTERN,
+ '%s/%s' % (MERCURIAL_SCM_BASE_URL, x64_exe_filename),
+ X64_EXE_DESCRIPTION.format(version=version),
+ ),
+ (
+ '10',
+ version,
+ X86_USER_AGENT_PATTERN,
+ '%s/%s' % (MERCURIAL_SCM_BASE_URL, x86_msi_filename),
+ X86_MSI_DESCRIPTION.format(version=version),
+ ),
+ (
+ '10',
+ version,
+ X64_USER_AGENT_PATTERN,
+ '%s/%s' % (MERCURIAL_SCM_BASE_URL, x64_msi_filename),
+ X64_MSI_DESCRIPTION.format(version=version)
+ )
+ )
+
+ lines = ['\t'.join(e) for e in entries]
+
+ return '\n'.join(lines) + '\n'
+
+
+def publish_artifacts_pypi(dist_path: pathlib.Path, version: str):
+ """Publish Windows release artifacts to PyPI."""
+
+ wheel_paths = resolve_wheel_artifacts(dist_path, version)
+
+ for p in wheel_paths:
+ if not p.exists():
+ raise Exception('%s not found' % p)
+
+ print('uploading wheels to PyPI (you may be prompted for credentials)')
+ pypi_upload(wheel_paths)
+
+
+def publish_artifacts_mercurial_scm_org(dist_path: pathlib.Path, version: str,
+ ssh_username=None):
+ """Publish Windows release artifacts to mercurial-scm.org."""
+ all_paths = resolve_all_artifacts(dist_path, version)
+
+ for p in all_paths:
+ if not p.exists():
+ raise Exception('%s not found' % p)
+
+ client = paramiko.SSHClient()
+ client.load_system_host_keys()
+ # We assume the system SSH configuration knows how to connect.
+ print('connecting to mercurial-scm.org via ssh...')
+ try:
+ client.connect('mercurial-scm.org', username=ssh_username)
+ except paramiko.AuthenticationException:
+ print('error authenticating; is an SSH key available in an SSH agent?')
+ raise
+
+ print('SSH connection established')
+
+ print('opening SFTP client...')
+ sftp = client.open_sftp()
+ print('SFTP client obtained')
+
+ for p in all_paths:
+ dest_path = '/var/www/release/windows/%s' % p.name
+ print('uploading %s to %s' % (p, dest_path))
+
+ with p.open('rb') as fh:
+ data = fh.read()
+
+ with sftp.open(dest_path, 'wb') as fh:
+ fh.write(data)
+ fh.chmod(0o0664)
+
+ latest_dat_path = '/var/www/release/windows/latest.dat'
+
+ now = datetime.datetime.utcnow()
+ backup_path = dist_path / (
+ 'latest-windows-%s.dat' % now.strftime('%Y%m%dT%H%M%S'))
+ print('backing up %s to %s' % (latest_dat_path, backup_path))
+
+ with sftp.open(latest_dat_path, 'rb') as fh:
+ latest_dat_old = fh.read()
+
+ with backup_path.open('wb') as fh:
+ fh.write(latest_dat_old)
+
+ print('writing %s with content:' % latest_dat_path)
+ latest_dat_content = generate_latest_dat(version)
+ print(latest_dat_content)
+
+ with sftp.open(latest_dat_path, 'wb') as fh:
+ fh.write(latest_dat_content.encode('ascii'))
+
+
+def publish_artifacts(dist_path: pathlib.Path, version: str,
+ pypi=True, mercurial_scm_org=True,
+ ssh_username=None):
+ """Publish Windows release artifacts.
+
+ Files are found in `dist_path`. We will look for files with version string
+ `version`.
+
+ `pypi` controls whether we upload to PyPI.
+ `mercurial_scm_org` controls whether we upload to mercurial-scm.org.
+ """
+ if pypi:
+ publish_artifacts_pypi(dist_path, version)
+
+ if mercurial_scm_org:
+ publish_artifacts_mercurial_scm_org(dist_path, version,
+ ssh_username=ssh_username)
new file mode 100644
@@ -0,0 +1,25 @@
+# pypi.py - Automation around PyPI
+#
+# Copyright 2019 Gregory Szorc <gregory.szorc@gmail.com>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+# no-check-code because Python 3 native.
+
+from twine.commands.upload import (
+ upload as twine_upload,
+)
+from twine.settings import (
+ Settings,
+)
+
+
+def upload(paths):
+ """Upload files to PyPI.
+
+ `paths` is an iterable of `pathlib.Path`.
+ """
+ settings = Settings()
+
+ twine_upload(settings, [str(p) for p in paths])
@@ -185,6 +185,14 @@
test_flags)
+def publish_windows_artifacts(hg: HGAutomation, aws_region, version: str,
+ pypi: bool, mercurial_scm_org: bool,
+ ssh_username: str):
+ windows.publish_artifacts(DIST_PATH, version,
+ pypi=pypi, mercurial_scm_org=mercurial_scm_org,
+ ssh_username=ssh_username)
+
+
def get_parser():
parser = argparse.ArgumentParser()
@@ -403,6 +411,34 @@
)
sp.set_defaults(func=run_tests_windows)
+ sp = subparsers.add_parser(
+ 'publish-windows-artifacts',
+ help='Publish built Windows artifacts (wheels, installers, etc)'
+ )
+ sp.add_argument(
+ '--no-pypi',
+ dest='pypi',
+ action='store_false',
+ default=True,
+ help='Skip uploading to PyPI',
+ )
+ sp.add_argument(
+ '--no-mercurial-scm-org',
+ dest='mercurial_scm_org',
+ action='store_false',
+ default=True,
+ help='Skip uploading to www.mercurial-scm.org',
+ )
+ sp.add_argument(
+ '--ssh-username',
+ help='SSH username for mercurial-scm.org',
+ )
+ sp.add_argument(
+ 'version',
+ help='Mercurial version string to locate local packages',
+ )
+ sp.set_defaults(func=publish_windows_artifacts)
+
return parser
@@ -181,3 +181,25 @@
Documenting them is beyond the scope of this document. Various tests
also require other optional dependencies and missing dependencies will
be printed by the test runner when a test is skipped.
+
+Releasing Windows Artifacts
+===========================
+
+The `automation.py` script can be used to automate the release of Windows
+artifacts::
+
+ $ ./automation.py build-all-windows-packages --revision 5.1.1
+ $ ./automation.py publish-windows-artifacts 5.1.1
+
+The first command will launch an EC2 instance to build all Windows packages
+and copy them into the `dist` directory relative to the repository root. The
+second command will then attempt to upload these files to PyPI (via `twine`)
+and to `mercurial-scm.org` (via SSH).
+
+Uploading to PyPI requires a PyPI account with write access to the `Mercurial`
+package. You can skip PyPI uploading by passing `--no-pypi`.
+
+Uploading to `mercurial-scm.org` requires an SSH account on that server
+with `windows` group membership and for the SSH key for that account to be the
+default SSH key (e.g. `~/.ssh/id_rsa`) or in a running SSH agent. You can
+skip `mercurial-scm.org` uploading by passing `--no-mercurial-scm-org`.