diff options
Diffstat (limited to 'tools/dist/release.py')
| -rwxr-xr-x | tools/dist/release.py | 620 |
1 files changed, 427 insertions, 193 deletions
diff --git a/tools/dist/release.py b/tools/dist/release.py index 5b12c00..141f937 100755 --- a/tools/dist/release.py +++ b/tools/dist/release.py @@ -41,7 +41,10 @@ import sys import glob import fnmatch import shutil -import urllib2 +try: + from urllib.request import urlopen # Python 3 +except: + from urllib2 import urlopen # Python 2 import hashlib import tarfile import logging @@ -52,6 +55,7 @@ import itertools import subprocess import argparse # standard in Python 2.7 import io +import yaml import backport.status @@ -67,78 +71,33 @@ except ImportError: sys.path.remove(ezt_path) +def get_dist_metadata_file_path(): + return os.path.join(os.path.abspath(sys.path[0]), 'release-lines.yaml') + +# Read the dist metadata (about release lines) +with open(get_dist_metadata_file_path(), 'r') as stream: + dist_metadata = yaml.safe_load(stream) + # Our required / recommended release tool versions by release branch -tool_versions = { - 'trunk' : { - 'autoconf' : ['2.69', - '954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969'], - 'libtool' : ['2.4.6', - 'e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3'], - 'swig' : ['3.0.12', - '7cf9f447ae7ed1c51722efc45e7f14418d15d7a1e143ac9f09a668999f4fc94d'], - }, - '1.13' : { - 'autoconf' : ['2.69', - '954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969'], - 'libtool' : ['2.4.6', - 'e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3'], - 'swig' : ['3.0.12', - '7cf9f447ae7ed1c51722efc45e7f14418d15d7a1e143ac9f09a668999f4fc94d'], - }, - '1.12' : { - 'autoconf' : ['2.69', - '954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969'], - 'libtool' : ['2.4.6', - 'e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3'], - 'swig' : ['3.0.12', - '7cf9f447ae7ed1c51722efc45e7f14418d15d7a1e143ac9f09a668999f4fc94d'], - }, - '1.11' : { - 'autoconf' : ['2.69', - '954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969'], - 'libtool' : ['2.4.6', - 'e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3'], - 'swig' : ['3.0.12', - '7cf9f447ae7ed1c51722efc45e7f14418d15d7a1e143ac9f09a668999f4fc94d'], - }, - '1.10' : { - 'autoconf' : ['2.69', - '954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969'], - 'libtool' : ['2.4.6', - 'e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3'], - 'swig' : ['3.0.12', - '7cf9f447ae7ed1c51722efc45e7f14418d15d7a1e143ac9f09a668999f4fc94d'], - }, - '1.9' : { - 'autoconf' : ['2.69', - '954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969'], - 'libtool' : ['2.4.6', - 'e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3'], - 'swig' : ['2.0.12', - '65e13f22a60cecd7279c59882ff8ebe1ffe34078e85c602821a541817a4317f7'], - }, - '1.8' : { - 'autoconf' : ['2.69', - '954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969'], - 'libtool' : ['2.4.3', - '36b4881c1843d7585de9c66c4c3d9a067ed3a3f792bc670beba21f5a4960acdf'], - 'swig' : ['2.0.9', - '586954000d297fafd7e91d1ad31089cc7e249f658889d11a44605d3662569539'], - }, -} +tool_versions = dist_metadata['tool_versions'] # The version that is our current recommended release -# ### TODO: derive this from svn_version.h; see ../../build/getversion.py -recommended_release = '1.12' +recommended_release = dist_metadata['recommended_release'] # For clean-dist, a whitelist of artifacts to keep, by version. -supported_release_lines = frozenset({"1.9", "1.10", "1.12", "1.13"}) +supported_release_lines = frozenset(dist_metadata['supported_release_lines']) +# Long-Term Support (LTS) versions +lts_release_lines = frozenset(dist_metadata['lts_release_lines']) # Some constants -svn_repos = 'https://svn.apache.org/repos/asf/subversion' -dist_repos = 'https://dist.apache.org/repos/dist' +svn_repos = os.getenv('SVN_RELEASE_SVN_REPOS', + 'https://svn.apache.org/repos/asf/subversion') +dist_repos = os.getenv('SVN_RELEASE_DIST_REPOS', + 'https://dist.apache.org/repos/dist') dist_dev_url = dist_repos + '/dev/subversion' dist_release_url = dist_repos + '/release/subversion' dist_archive_url = 'https://archive.apache.org/dist/subversion' +buildbot_repos = os.getenv('SVN_RELEASE_BUILDBOT_REPOS', + 'https://svn.apache.org/repos/infra/infrastructure/buildbot/aegis/buildmaster') KEYS = 'https://people.apache.org/keys/group/subversion.asc' extns = ['zip', 'tar.gz', 'tar.bz2'] @@ -183,18 +142,6 @@ class Version(object): def is_prerelease(self): return self.pre != None - def is_recommended(self): - return self.branch == recommended_release - - def get_download_anchor(self): - if self.is_prerelease(): - return 'pre-releases' - else: - if self.is_recommended(): - return 'recommended-release' - else: - return 'supported-releases' - def get_ver_tags(self, revnum): # These get substituted into svn_version.h ver_tag = '' @@ -212,7 +159,7 @@ class Version(object): ver_tag = '" (Nightly Build r%d)"' % revnum ver_numtag = '"-nightly-r%d"' % revnum else: - ver_tag = '" (r%d)"' % revnum + ver_tag = '" (r%d)"' % revnum ver_numtag = '""' return (ver_tag, ver_numtag) @@ -282,15 +229,12 @@ def get_exportdir(base_dir, version, revnum): return os.path.join(get_tempdir(base_dir), 'subversion-%s-r%d' % (version, revnum)) -def get_deploydir(base_dir): - return os.path.join(base_dir, 'deploy') - def get_target(args): "Return the location of the artifacts" if args.target: return args.target else: - return get_deploydir(args.base_dir) + return os.path.join(args.base_dir, 'deploy') def get_branch_path(args): if not args.branch: @@ -309,12 +253,14 @@ def get_tmplfile(filename): return open(os.path.join(get_tmpldir(), filename)) except IOError: # Hmm, we had a problem with the local version, let's try the repo - return urllib2.urlopen(svn_repos + '/trunk/tools/dist/templates/' + filename) + return urlopen(svn_repos + '/trunk/tools/dist/templates/' + filename) def get_nullfile(): return open(os.path.devnull, 'w') -def run_script(verbose, script, hide_stderr=False): +def run_command(cmd, verbose=True, hide_stderr=False, dry_run=False): + if verbose: + print("+ " + ' '.join(cmd)) stderr = None if verbose: stdout = None @@ -323,23 +269,62 @@ def run_script(verbose, script, hide_stderr=False): if hide_stderr: stderr = get_nullfile() + if not dry_run: + subprocess.check_call(cmd, stdout=stdout, stderr=stderr) + else: + print(' ## dry-run; not executed') + +def run_script(verbose, script, hide_stderr=False): for l in script.split('\n'): - subprocess.check_call(l.split(), stdout=stdout, stderr=stderr) + run_command(l.split(), verbose, hide_stderr) def download_file(url, target, checksum): - response = urllib2.urlopen(url) - target_file = open(target, 'w+') + """Download the file at URL to the local path TARGET. + If CHECKSUM is a string, verify the checksum of the downloaded + file and raise RuntimeError if it does not match. If CHECKSUM + is None, do not verify the downloaded file. + """ + assert checksum is None or isinstance(checksum, str) + + response = urlopen(url) + target_file = open(target, 'w+b') target_file.write(response.read()) target_file.seek(0) m = hashlib.sha256() m.update(target_file.read()) target_file.close() checksum2 = m.hexdigest() - if checksum != checksum2: + if checksum is not None and checksum != checksum2: raise RuntimeError("Checksum mismatch for '%s': "\ "downloaded: '%s'; expected: '%s'" % \ (target, checksum, checksum2)) +def run_svn(cmd, verbose=True, dry_run=False, username=None): + if (username): + cmd[:0] = ['--username', username] + run_command(['svn'] + cmd, verbose=verbose, dry_run=dry_run) + +def run_svnmucc(cmd, verbose=True, dry_run=False, username=None): + if (username): + cmd[:0] = ['--username', username] + run_command(['svnmucc'] + cmd, verbose=verbose, dry_run=dry_run) + +#---------------------------------------------------------------------- +def is_lts(version): + return version.branch in lts_release_lines + +def is_recommended(version): + return version.branch == recommended_release + +def get_download_anchor(version): + if version.is_prerelease(): + return 'pre-releases' + else: + if is_recommended(version): + return 'recommended-release' + else: + return 'supported-releases' + #---------------------------------------------------------------------- # ezt helpers @@ -363,7 +348,7 @@ def cleanup(args): shutil.rmtree(get_prefix(args.base_dir), True) shutil.rmtree(get_tempdir(args.base_dir), True) - shutil.rmtree(get_deploydir(args.base_dir), True) + shutil.rmtree(get_target(args), True) #---------------------------------------------------------------------- @@ -378,7 +363,8 @@ class RollDep(object): def _test_version(self, cmd): proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT, + universal_newlines=True) (stdout, stderr) = proc.communicate() rc = proc.wait() if rc: return '' @@ -520,13 +506,236 @@ def build_env(args): #---------------------------------------------------------------------- +# Create a new minor release branch + +def get_trunk_wc_path(base_dir, path=None): + trunk_wc_path = os.path.join(get_tempdir(base_dir), 'svn-trunk') + if path is None: return trunk_wc_path + return os.path.join(trunk_wc_path, path) + +def get_buildbot_wc_path(base_dir, path=None): + buildbot_wc_path = os.path.join(get_tempdir(base_dir), 'svn-buildmaster') + if path is None: return buildbot_wc_path + return os.path.join(buildbot_wc_path, path) + +def get_trunk_url(revnum=None): + return svn_repos + '/trunk' + '@' + (str(revnum) if revnum else '') + +def get_branch_url(ver): + return svn_repos + '/branches/' + ver.branch + '.x' + +def get_tag_url(ver): + return svn_repos + '/tags/' + ver.base + +def edit_file(path, pattern, replacement): + print("Editing '%s'" % (path,)) + print(" pattern='%s'" % (pattern,)) + print(" replace='%s'" % (replacement,)) + old_text = open(path, 'r').read() + new_text = re.sub(pattern, replacement, old_text) + assert new_text != old_text + open(path, 'w').write(new_text) + +def edit_changes_file(path, newtext): + """Insert NEWTEXT in the 'CHANGES' file found at PATH, + just before the first line that starts with 'Version '. + """ + print("Prepending to '%s'" % (path,)) + print(" text='%s'" % (newtext,)) + lines = open(path, 'r').readlines() + for i, line in enumerate(lines): + if line.startswith('Version '): + with open(path, 'w') as newfile: + newfile.writelines(lines[:i]) + newfile.write(newtext) + newfile.writelines(lines[i:]) + break + +#---------------------------------------------------------------------- +def make_release_branch(args): + ver = args.version + run_svn(['copy', + get_trunk_url(args.revnum), + get_branch_url(ver), + '-m', 'Create the ' + ver.branch + '.x release branch.'], + dry_run=args.dry_run) + +#---------------------------------------------------------------------- +def update_minor_ver_in_trunk(args): + """Change the minor version in trunk to the next (future) minor version. + """ + ver = args.version + trunk_wc = get_trunk_wc_path(args.base_dir) + run_svn(['checkout', + get_trunk_url(args.revnum), + trunk_wc]) + + prev_ver = Version('1.%d.0' % (ver.minor - 1,)) + next_ver = Version('1.%d.0' % (ver.minor + 1,)) + relpaths = [] + + relpath = 'subversion/include/svn_version.h' + relpaths.append(relpath) + edit_file(get_trunk_wc_path(args.base_dir, relpath), + r'(#define SVN_VER_MINOR *)%s' % (ver.minor,), + r'\g<1>%s' % (next_ver.minor,)) + + relpath = 'subversion/tests/cmdline/svntest/main.py' + relpaths.append(relpath) + edit_file(get_trunk_wc_path(args.base_dir, relpath), + r'(SVN_VER_MINOR = )%s' % (ver.minor,), + r'\g<1>%s' % (next_ver.minor,)) + + relpath = 'subversion/bindings/javahl/src/org/apache/subversion/javahl/NativeResources.java' + relpaths.append(relpath) + try: + # since r1817921 (just after branching 1.10) + edit_file(get_trunk_wc_path(args.base_dir, relpath), + r'SVN_VER_MINOR = %s;' % (ver.minor,), + r'SVN_VER_MINOR = %s;' % (next_ver.minor,)) + except: + # before r1817921: two separate places + edit_file(get_trunk_wc_path(args.base_dir, relpath), + r'version.isAtLeast\(1, %s, 0\)' % (ver.minor,), + r'version.isAtLeast\(1, %s, 0\)' % (next_ver.minor,)) + edit_file(get_trunk_wc_path(args.base_dir, relpath), + r'1.%s.0, but' % (ver.minor,), + r'1.%s.0, but' % (next_ver.minor,)) + + relpath = 'CHANGES' + relpaths.append(relpath) + # insert at beginning of CHANGES file + edit_changes_file(get_trunk_wc_path(args.base_dir, relpath), + 'Version ' + next_ver.base + '\n' + + '(?? ??? 20XX, from /branches/' + next_ver.branch + '.x)\n' + + get_tag_url(next_ver) + '\n' + + '\n') + + log_msg = '''\ +Increment the trunk version number to %s, and introduce a new CHANGES +section, following the creation of the %s.x release branch. + +* subversion/include/svn_version.h, + subversion/bindings/javahl/src/org/apache/subversion/javahl/NativeResources.java, + subversion/tests/cmdline/svntest/main.py + (SVN_VER_MINOR): Increment to %s. + +* CHANGES: New section for %s.0. +''' % (next_ver.branch, ver.branch, next_ver.minor, next_ver.branch) + commit_paths = [get_trunk_wc_path(args.base_dir, p) for p in relpaths] + run_svn(['commit'] + commit_paths + ['-m', log_msg], + dry_run=args.dry_run) + +#---------------------------------------------------------------------- +def create_status_file_on_branch(args): + ver = args.version + branch_wc = get_workdir(args.base_dir) + branch_url = get_branch_url(ver) + run_svn(['checkout', branch_url, branch_wc, '--depth=immediates']) + + status_local_path = os.path.join(branch_wc, 'STATUS') + template_filename = 'STATUS.ezt' + data = { 'major-minor' : ver.branch, + 'major-minor-patch' : ver.base, + } + + template = ezt.Template(compress_whitespace=False) + template.parse(get_tmplfile(template_filename).read()) + + with open(status_local_path, 'wx') as g: + template.generate(g, data) + run_svn(['add', status_local_path]) + run_svn(['commit', status_local_path, + '-m', '* branches/' + ver.branch + '.x/STATUS: New file.'], + dry_run=args.dry_run) + +#---------------------------------------------------------------------- +def update_backport_bot(args): + ver = args.version + print("""\ + +*** MANUAL STEP REQUIRED *** + + Ask someone with appropriate access to add the %s.x branch + to the backport merge bot. See + http://subversion.apache.org/docs/community-guide/releasing.html#backport-merge-bot + +*** + +""" % (ver.branch,)) + +#---------------------------------------------------------------------- +def update_buildbot_config(args): + """Add the new branch to the list of branches monitored by the buildbot + master. + """ + ver = args.version + buildbot_wc = get_buildbot_wc_path(args.base_dir) + run_svn(['checkout', buildbot_repos, buildbot_wc]) + + prev_ver = Version('1.%d.0' % (ver.minor - 1,)) + next_ver = Version('1.%d.0' % (ver.minor + 1,)) + + relpath = 'master1/projects/subversion.conf' + edit_file(get_buildbot_wc_path(args.base_dir, relpath), + r'(MINOR_LINES=\[.*%s)(\])' % (prev_ver.minor,), + r'\1, %s\2' % (ver.minor,)) + + log_msg = '''\ +Subversion: start monitoring the %s branch. +''' % (ver.branch) + commit_paths = [get_buildbot_wc_path(args.base_dir, relpath)] + run_svn(['commit'] + commit_paths + ['-m', log_msg], + dry_run=args.dry_run) + +#---------------------------------------------------------------------- +def create_release_branch(args): + make_release_branch(args) + update_minor_ver_in_trunk(args) + create_status_file_on_branch(args) + update_backport_bot(args) + update_buildbot_config(args) + + +#---------------------------------------------------------------------- +def write_release_notes(args): + + # Create a skeleton release notes file from template + + template_filename = \ + 'release-notes-lts.ezt' if is_lts(args.version) else 'release-notes.ezt' + + prev_ver = Version('%d.%d.0' % (args.version.major, args.version.minor - 1)) + data = { 'major-minor' : args.version.branch, + 'previous-major-minor' : prev_ver.branch, + } + + template = ezt.Template(compress_whitespace=False) + template.parse(get_tmplfile(template_filename).read()) + + if args.edit_html_file: + with open(args.edit_html_file, 'w') as g: + template.generate(g, data) + else: + template.generate(sys.stdout, data) + + # Add an "in progress" entry in the release notes index + # + index_file = os.path.normpath(args.edit_html_file + '/../index.html') + marker = '<ul id="release-notes-list">\n' + new_item = '<li><a href="%s.html">Subversion %s</a> – <i>in progress</i></li>\n' % (args.version.branch, args.version.branch) + edit_file(index_file, + re.escape(marker), + (marker + new_item).replace('\\', r'\\')) + +#---------------------------------------------------------------------- # Create release artifacts def compare_changes(repos, branch, revision): mergeinfo_cmd = ['svn', 'mergeinfo', '--show-revs=eligible', repos + '/trunk/CHANGES', repos + '/' + branch + '/' + 'CHANGES'] - stdout = subprocess.check_output(mergeinfo_cmd) + stdout = subprocess.check_output(mergeinfo_cmd, universal_newlines=True) if stdout: # Treat this as a warning since we are now putting entries for future # minor releases in CHANGES on trunk. @@ -544,7 +753,7 @@ def check_copyright_year(repos, branch, revision): file_url = (repos + '/' + branch + '/' + branch_relpath + '@' + str(revision)) cat_cmd = ['svn', 'cat', file_url] - stdout = subprocess.check_output(cat_cmd) + stdout = subprocess.check_output(cat_cmd, universal_newlines=True) m = _copyright_re.search(stdout) if m: year = m.group('year') @@ -596,17 +805,18 @@ def roll_tarballs(args): compare_changes(svn_repos, branch, args.revnum) # Ensure the output directory doesn't already exist - if os.path.exists(get_deploydir(args.base_dir)): + if os.path.exists(get_target(args)): raise RuntimeError('output directory \'%s\' already exists' - % get_deploydir(args.base_dir)) + % get_target(args)) - os.mkdir(get_deploydir(args.base_dir)) + os.mkdir(get_target(args)) logging.info('Preparing working copy source') shutil.rmtree(get_workdir(args.base_dir), True) - run_script(args.verbose, 'svn checkout %s %s' - % (svn_repos + '/' + branch + '@' + str(args.revnum), - get_workdir(args.base_dir))) + run_svn(['checkout', + svn_repos + '/' + branch + '@' + str(args.revnum), + get_workdir(args.base_dir)], + verbose=args.verbose) # Exclude stuff we don't want in the tarball, it will not be present # in the exported tree. @@ -617,8 +827,8 @@ def roll_tarballs(args): exclude += ['packages', 'www'] cwd = os.getcwd() os.chdir(get_workdir(args.base_dir)) - run_script(args.verbose, - 'svn update --set-depth exclude %s' % " ".join(exclude)) + run_svn(['update', '--set-depth=exclude'] + exclude, + verbose=args.verbose) os.chdir(cwd) if args.patches: @@ -628,10 +838,10 @@ def roll_tarballs(args): for name in os.listdir(args.patches): if name.find(majmin) != -1 and name.endswith('patch'): logging.info('Applying patch %s' % name) - run_script(args.verbose, - '''svn patch %s %s''' - % (os.path.join(args.patches, name), - get_workdir(args.base_dir))) + run_svn(['patch', + os.path.join(args.patches, name), + get_workdir(args.base_dir)], + verbose=args.verbose) # Massage the new version number into svn_version.h. ver_tag, ver_numtag = args.version.get_ver_tags(args.revnum) @@ -666,11 +876,12 @@ def roll_tarballs(args): def export(windows): shutil.rmtree(exportdir, True) if windows: - eol_style = "--native-eol CRLF" + eol_style = "--native-eol=CRLF" else: - eol_style = "--native-eol LF" - run_script(args.verbose, "svn export %s %s %s" - % (eol_style, get_workdir(args.base_dir), exportdir)) + eol_style = "--native-eol=LF" + run_svn(['export', + eol_style, get_workdir(args.base_dir), exportdir], + verbose=args.verbose) def transform_sql(): for root, dirs, files in os.walk(exportdir): @@ -744,25 +955,33 @@ def roll_tarballs(args): for e in extns: filename = basename + '.' + e filepath = os.path.join(get_tempdir(args.base_dir), filename) - shutil.move(filepath, get_deploydir(args.base_dir)) - filepath = os.path.join(get_deploydir(args.base_dir), filename) + shutil.move(filepath, get_target(args)) + filepath = os.path.join(get_target(args), filename) if args.version < Version("1.11.0-alpha1"): # 1.10 and earlier generate *.sha1 files for compatibility reasons. # They are deprecated, however, so we don't publicly link them in # the announcements any more. m = hashlib.sha1() - m.update(open(filepath, 'r').read()) + m.update(open(filepath, 'rb').read()) open(filepath + '.sha1', 'w').write(m.hexdigest()) m = hashlib.sha512() - m.update(open(filepath, 'r').read()) + m.update(open(filepath, 'rb').read()) open(filepath + '.sha512', 'w').write(m.hexdigest()) # Nightlies do not get tagged so do not need the header if args.version.pre != 'nightly': shutil.copy(os.path.join(get_workdir(args.base_dir), 'subversion', 'include', 'svn_version.h'), - os.path.join(get_deploydir(args.base_dir), - 'svn_version.h.dist-%s' % str(args.version))) + os.path.join(get_target(args), + 'svn_version.h.dist-%s' + % (str(args.version),))) + + # Download and "tag" the KEYS file (in case a signing key is removed + # from a committer's LDAP profile down the road) + basename = 'subversion-%s.KEYS' % (str(args.version),) + filepath = os.path.join(get_tempdir(args.base_dir), basename) + download_file(KEYS, filepath, None) + shutil.move(filepath, get_target(args)) # And we're done! @@ -804,14 +1023,12 @@ def post_candidates(args): logging.info('Importing tarballs to %s' % dist_dev_url) ver = str(args.version) - svn_cmd = ['svn', 'import', '-m', + svn_cmd = ['import', '-m', 'Add Subversion %s candidate release artifacts' % ver, '--auto-props', '--config-option', 'config:auto-props:*.asc=svn:eol-style=native;svn:mime-type=text/plain', target, dist_dev_url] - if (args.username): - svn_cmd += ['--username', args.username] - subprocess.check_call(svn_cmd) + run_svn(svn_cmd, verbose=args.verbose, username=args.username) #---------------------------------------------------------------------- # Create tag @@ -828,10 +1045,7 @@ def create_tag_only(args): tag = svn_repos + '/tags/' + str(args.version) - svnmucc_cmd = ['svnmucc', '-m', - 'Tagging release ' + str(args.version)] - if (args.username): - svnmucc_cmd += ['--username', args.username] + svnmucc_cmd = ['-m', 'Tagging release ' + str(args.version)] svnmucc_cmd += ['cp', str(args.revnum), branch_url, tag] svnmucc_cmd += ['put', os.path.join(target, 'svn_version.h.dist' + '-' + str(args.version)), @@ -839,7 +1053,7 @@ def create_tag_only(args): # don't redirect stdout/stderr since svnmucc might ask for a password try: - subprocess.check_call(svnmucc_cmd) + run_svnmucc(svnmucc_cmd, verbose=args.verbose, username=args.username) except subprocess.CalledProcessError: if args.version.is_prerelease(): logging.error("Do you need to pass --branch=trunk?") @@ -878,7 +1092,8 @@ def bump_versions_on_branch(args): args.version.patch + 1)) HEAD = subprocess.check_output(['svn', 'info', '--show-item=revision', - '--', branch_url]).strip() + '--', branch_url], + universal_newlines=True).strip() HEAD = int(HEAD) def file_object_for(relpath): fd = tempfile.NamedTemporaryFile() @@ -898,13 +1113,14 @@ def bump_versions_on_branch(args): svn_version_h.seek(0, os.SEEK_SET) STATUS.seek(0, os.SEEK_SET) - subprocess.check_call(['svnmucc', '-r', str(HEAD), - '-m', 'Post-release housekeeping: ' - 'bump the %s branch to %s.' - % (branch_url.split('/')[-1], str(new_version)), - 'put', svn_version_h.name, svn_version_h.url, - 'put', STATUS.name, STATUS.url, - ]) + run_svnmucc(['-r', str(HEAD), + '-m', 'Post-release housekeeping: ' + 'bump the %s branch to %s.' + % (branch_url.split('/')[-1], str(new_version)), + 'put', svn_version_h.name, svn_version_h.url, + 'put', STATUS.name, STATUS.url, + ], + verbose=args.verbose, username=args.username) del svn_version_h del STATUS @@ -924,7 +1140,8 @@ def clean_dist(args): '''Clean the distribution directory of release artifacts of no-longer-supported minor lines.''' - stdout = subprocess.check_output(['svn', 'list', dist_release_url]) + stdout = subprocess.check_output(['svn', 'list', dist_release_url], + universal_newlines=True) def minor(version): """Return the minor release line of the parameter, which must be @@ -946,10 +1163,8 @@ def clean_dist(args): for i in sorted(to_keep): logging.info("Saving release '%s'", i) - svnmucc_cmd = ['svnmucc', '-m', 'Remove old Subversion releases.\n' + + svnmucc_cmd = ['-m', 'Remove old Subversion releases.\n' + 'They are still available at ' + dist_archive_url] - if (args.username): - svnmucc_cmd += ['--username', args.username] for filename in filenames: if Version(filename) not in to_keep: logging.info("Removing %r", filename) @@ -957,7 +1172,7 @@ def clean_dist(args): # don't redirect stdout/stderr since svnmucc might ask for a password if 'rm' in svnmucc_cmd: - subprocess.check_call(svnmucc_cmd) + run_svnmucc(svnmucc_cmd, verbose=args.verbose, username=args.username) else: logging.info("Nothing to remove") @@ -967,16 +1182,15 @@ def clean_dist(args): def move_to_dist(args): 'Move candidate artifacts to the distribution directory.' - stdout = subprocess.check_output(['svn', 'list', dist_dev_url]) + stdout = subprocess.check_output(['svn', 'list', dist_dev_url], + universal_newlines=True) filenames = [] for entry in stdout.split('\n'): if fnmatch.fnmatch(entry, 'subversion-%s.*' % str(args.version)): filenames.append(entry) - svnmucc_cmd = ['svnmucc', '-m', + svnmucc_cmd = ['-m', 'Publish Subversion-%s.' % str(args.version)] - if (args.username): - svnmucc_cmd += ['--username', args.username] svnmucc_cmd += ['rm', dist_dev_url + '/' + 'svn_version.h.dist' + '-' + str(args.version)] for filename in filenames: @@ -985,20 +1199,24 @@ def move_to_dist(args): # don't redirect stdout/stderr since svnmucc might ask for a password logging.info('Moving release artifacts to %s' % dist_release_url) - subprocess.check_call(svnmucc_cmd) + run_svnmucc(svnmucc_cmd, verbose=args.verbose, username=args.username) #---------------------------------------------------------------------- # Write announcements def write_news(args): 'Write text for the Subversion website.' - data = { 'date' : datetime.date.today().strftime('%Y%m%d'), - 'date_pres' : datetime.date.today().strftime('%Y-%m-%d'), + if args.news_release_date: + release_date = datetime.datetime.strptime(args.news_release_date, '%Y-%m-%d') + else: + release_date = datetime.date.today() + data = { 'date' : release_date.strftime('%Y%m%d'), + 'date_pres' : release_date.strftime('%Y-%m-%d'), 'major-minor' : args.version.branch, 'version' : str(args.version), 'version_base' : args.version.base, - 'anchor': args.version.get_download_anchor(), - 'is_recommended': ezt_bool(args.version.is_recommended()), + 'anchor': get_download_anchor(args.version), + 'is_recommended': ezt_bool(is_recommended(args.version)), 'announcement_url': args.announcement_url, } @@ -1058,7 +1276,7 @@ def write_announcement(args): 'siginfo' : "\n".join(siginfo) + "\n", 'major-minor' : args.version.branch, 'major-minor-patch' : args.version.base, - 'anchor' : args.version.get_download_anchor(), + 'anchor' : get_download_anchor(args.version), } if args.version.is_prerelease(): @@ -1153,9 +1371,9 @@ def get_siginfo(args, quiet=False): % (n, filename, key_end)) sys.exit(1) - fd, fn = tempfile.mkstemp() - os.write(fd, key_start + key) - os.close(fd) + fd, fn = tempfile.mkstemp(text=True) + with os.fdopen(fd, 'w') as key_file: + key_file.write(key_start + key) verified = gpg.verify_file(open(fn, 'rb'), filename[:-4]) os.unlink(fn) @@ -1177,6 +1395,7 @@ def get_siginfo(args, quiet=False): gpg_output = subprocess.check_output( ['gpg', '--fixed-list-mode', '--with-colons', '--fingerprint', id], stderr=subprocess.STDOUT, + universal_newlines=True, ) gpg_output = gpg_output.splitlines() @@ -1244,7 +1463,7 @@ def get_keys(args): 'Import the LDAP-based KEYS file to gpg' # We use a tempfile because urlopen() objects don't have a .fileno() with tempfile.SpooledTemporaryFile() as fd: - fd.write(urllib2.urlopen(KEYS).read()) + fd.write(urlopen(KEYS).read()) fd.flush() fd.seek(0) subprocess.check_call(['gpg', '--import'], stdin=fd) @@ -1256,18 +1475,18 @@ def add_to_changes_dict(changes_dict, audience, section, change, revision): if section: section = section.lower() change = change.strip() - + if not audience in changes_dict: changes_dict[audience] = dict() if not section in changes_dict[audience]: changes_dict[audience][section] = dict() - + changes = changes_dict[audience][section] if change in changes: changes[change].add(revision) else: changes[change] = set([revision]) - + def print_section(changes_dict, audience, section, title, mandatory=False): if audience in changes_dict: audience_changes = changes_dict[audience] @@ -1304,7 +1523,7 @@ def write_changelog(args): # Putting [skip], [ignore], [c:skip] or [c:ignore] somewhere in the # log message means this commit must be ignored for Changelog processing # (ignored even with the --include-unlabeled-summaries option). - # + # # If there is no changes label anywhere in the commit message, and the # --include-unlabeled-summaries option is used, we'll consider the summary # line of the commit message (= first line except if it starts with a *) @@ -1323,9 +1542,10 @@ def write_changelog(args): previous = svn_repos + '/' + args.previous include_unlabeled = args.include_unlabeled separator_line = ('-' * 72) + '\n' - + mergeinfo = subprocess.check_output(['svn', 'mergeinfo', '--show-revs', - 'eligible', '--log', branch_url, previous]) + 'eligible', '--log', branch_url, previous], + universal_newlines=True) log_messages_dict = { # This is a dictionary mapping revision numbers to their respective # log messages. The expression in the "key:" part of the dict @@ -1336,7 +1556,7 @@ def write_changelog(args): for log_message in mergeinfo.split(separator_line)[1:-1] } mergeinfo = mergeinfo.splitlines() - + separator_pattern = re.compile('^-{72}$') revline_pattern = re.compile('^r(\d+) \| [^\|]+ \| [^\|]+ \| \d+ lines?$') changes_prefix_pattern = re.compile(r'^\[(U|D)?:?([^\]]+)?\](.+)$') @@ -1353,7 +1573,7 @@ def write_changelog(args): audience = None section = None message = None - + for line in mergeinfo: if separator_pattern.match(line): # New revision section. Reset variables. @@ -1370,7 +1590,7 @@ def write_changelog(args): # logic, in order to extract CHANGES_PREFIX_PATTERN # and CHANGES_SUFFIX_PATTERN lines from the trunk log # message. - + # 2. Parse the STATUS entry this_log_message = log_messages_dict[revision] status_paragraph = this_log_message.split('\n\n')[2] @@ -1411,7 +1631,7 @@ def write_changelog(args): if re.search(r'\[(c:)?(skip|ignore)\]', line, re.IGNORECASE): changes_ignore = True - + prefix_match = changes_prefix_pattern.match(line) if prefix_match: audience = prefix_match.group(1) @@ -1459,17 +1679,25 @@ def main(): parser = argparse.ArgumentParser( description='Create an Apache Subversion release.') parser.add_argument('--clean', action='store_true', default=False, - help='Remove any directories previously created by %(prog)s') + help='''Remove any directories previously created by %(prog)s, + including the 'prefix' dir, the 'temp' dir, and the + default or specified target dir.''') parser.add_argument('--verbose', action='store_true', default=False, help='Increase output verbosity') parser.add_argument('--base-dir', default=os.getcwd(), help='''The directory in which to create needed files and folders. The default is the current working directory.''') + parser.add_argument('--target', + help='''The full path to the directory containing + release artifacts. Default: <BASE_DIR>/deploy''') parser.add_argument('--branch', help='''The branch to base the release on, as a path relative to ^/subversion/. Default: 'branches/MAJOR.MINOR.x'.''') + parser.add_argument('--username', + help='Username for committing to ' + svn_repos + + ' or ' + dist_repos + '.') subparsers = parser.add_subparsers(title='subcommands') # Setup the parser for the build-env subcommand @@ -1487,6 +1715,40 @@ def main(): help='''Attempt to use existing build dependencies before downloading and building a private set.''') + # Setup the parser for the create-release-branch subcommand + subparser = subparsers.add_parser('create-release-branch', + help='''Create a minor release branch: branch from trunk, + update version numbers on trunk, create status + file on branch, update backport bot, + update buildbot config.''') + subparser.set_defaults(func=create_release_branch) + subparser.add_argument('version', type=Version, + help='''A version number to indicate the branch, such as + '1.7.0' (the '.0' is required).''') + subparser.add_argument('revnum', type=lambda arg: int(arg.lstrip('r')), + nargs='?', default=None, + help='''The trunk revision number to base the branch on. + Default is HEAD.''') + subparser.add_argument('--dry-run', action='store_true', default=False, + help='Avoid committing any changes to repositories.') + + # Setup the parser for the create-release-branch subcommand + subparser = subparsers.add_parser('write-release-notes', + help='''Write a template release-notes file.''') + subparser.set_defaults(func=write_release_notes) + subparser.add_argument('version', type=Version, + help='''A version number to indicate the branch, such as + '1.7.0' (the '.0' is required).''') + subparser.add_argument('revnum', type=lambda arg: int(arg.lstrip('r')), + nargs='?', default=None, + help='''The trunk revision number to base the branch on. + Default is HEAD.''') + subparser.add_argument('--edit-html-file', + help='''Write the template release-notes to this file, + and update 'index.html' in the same directory.''') + subparser.add_argument('--dry-run', action='store_true', default=False, + help='Avoid committing any changes to repositories.') + # Setup the parser for the roll subcommand subparser = subparsers.add_parser('roll', help='''Create the release artifacts.''') @@ -1504,9 +1766,6 @@ def main(): subparser.set_defaults(func=sign_candidates) subparser.add_argument('version', type=Version, help='''The release label, such as '1.7.0-alpha1'.''') - subparser.add_argument('--target', - help='''The full path to the directory containing - release artifacts.''') subparser.add_argument('--userid', help='''The (optional) USER-ID specifying the key to be used for signing, such as '110B1C95' (Key-ID). If @@ -1519,11 +1778,6 @@ def main(): subparser.set_defaults(func=post_candidates) subparser.add_argument('version', type=Version, help='''The release label, such as '1.7.0-alpha1'.''') - subparser.add_argument('--username', - help='''Username for ''' + dist_repos + '''.''') - subparser.add_argument('--target', - help='''The full path to the directory containing - release artifacts.''') # Setup the parser for the create-tag subcommand subparser = subparsers.add_parser('create-tag', @@ -1534,11 +1788,6 @@ def main(): help='''The release label, such as '1.7.0-alpha1'.''') subparser.add_argument('revnum', type=lambda arg: int(arg.lstrip('r')), help='''The revision number to base the release on.''') - subparser.add_argument('--username', - help='''Username for ''' + svn_repos + '''.''') - subparser.add_argument('--target', - help='''The full path to the directory containing - release artifacts.''') # Setup the parser for the bump-versions-on-branch subcommand subparser = subparsers.add_parser('bump-versions-on-branch', @@ -1548,11 +1797,6 @@ def main(): help='''The release label, such as '1.7.0-alpha1'.''') subparser.add_argument('revnum', type=lambda arg: int(arg.lstrip('r')), help='''The revision number to base the release on.''') - subparser.add_argument('--username', - help='''Username for ''' + svn_repos + '''.''') - subparser.add_argument('--target', - help='''The full path to the directory containing - release artifacts.''') # The clean-dist subcommand subparser = subparsers.add_parser('clean-dist', @@ -1560,8 +1804,6 @@ def main(): subparser.set_defaults(func=clean_dist) subparser.add_argument('--dist-dir', help='''The directory to clean.''') - subparser.add_argument('--username', - help='''Username for ''' + dist_repos + '''.''') # The move-to-dist subcommand subparser = subparsers.add_parser('move-to-dist', @@ -1571,8 +1813,6 @@ def main(): subparser.set_defaults(func=move_to_dist) subparser.add_argument('version', type=Version, help='''The release label, such as '1.7.0-alpha1'.''') - subparser.add_argument('--username', - help='''Username for ''' + dist_repos + '''.''') # The write-news subcommand subparser = subparsers.add_parser('write-news', @@ -1581,6 +1821,9 @@ def main(): subparser.set_defaults(func=write_news) subparser.add_argument('--announcement-url', help='''The URL to the archived announcement email.''') + subparser.add_argument('--news-release-date', + help='''The release date for the news, as YYYY-MM-DD. + Default: today.''') subparser.add_argument('--edit-html-file', help='''Insert the text into this file news.html, index.html).''') @@ -1595,9 +1838,6 @@ def main(): subparser.add_argument('--security', action='store_true', default=False, help='''The release being announced includes security fixes.''') - subparser.add_argument('--target', - help='''The full path to the directory containing - release artifacts.''') subparser.add_argument('version', type=Version, help='''The release label, such as '1.7.0-alpha1'.''') @@ -1606,9 +1846,6 @@ def main(): help='''Output to stdout template text for the download table for subversion.apache.org''') subparser.set_defaults(func=write_downloads) - subparser.add_argument('--target', - help='''The full path to the directory containing - release artifacts.''') subparser.add_argument('version', type=Version, help='''The release label, such as '1.7.0-alpha1'.''') @@ -1619,9 +1856,6 @@ def main(): subparser.set_defaults(func=check_sigs) subparser.add_argument('version', type=Version, help='''The release label, such as '1.7.0-alpha1'.''') - subparser.add_argument('--target', - help='''The full path to the directory containing - release artifacts.''') # get-keys subparser = subparsers.add_parser('get-keys', @@ -1641,7 +1875,7 @@ def main(): like [U:client], [D:api], [U], ...''') subparser.set_defaults(func=write_changelog) subparser.add_argument('previous', - help='''The "previous" branch or tag, relative to + help='''The "previous" branch or tag, relative to ^/subversion/, to compare "branch" against.''') subparser.add_argument('--include-unlabeled-summaries', dest='include_unlabeled', @@ -1652,7 +1886,7 @@ def main(): summary line contains 'STATUS', 'CHANGES', 'Post-release housekeeping', 'Follow-up' or starts with '*').''') - + # Parse the arguments args = parser.parse_args() |