diff options
author | Dimitri John Ledkov <xnox@ubuntu.com> | 2019-12-19 17:18:03 +0000 |
---|---|---|
committer | Andrej Shadura <andrew.shadura@collabora.co.uk> | 2019-12-23 13:46:31 +0100 |
commit | 00a09294d4b4aae9e21b5ce39181da9b7262a115 (patch) | |
tree | 3584888f79cf5fed284dc51217289455074ae993 | |
parent | 728c0eec63f729ef4d112024e5d3368cd7b0bf72 (diff) |
Switch to python3
Closes: #830873, #937218
-rw-r--r-- | .travis.yml | 6 | ||||
-rw-r--r-- | NEWS | 11 | ||||
-rw-r--r-- | debian/changelog | 8 | ||||
-rw-r--r-- | debian/compat | 1 | ||||
-rw-r--r-- | debian/control | 16 | ||||
-rwxr-xr-x | debian/rules | 2 | ||||
-rw-r--r-- | osc/babysitter.py | 11 | ||||
-rw-r--r-- | osc/build.py | 83 | ||||
-rw-r--r-- | osc/cmdln.py | 20 | ||||
-rw-r--r-- | osc/commandline.py | 145 | ||||
-rw-r--r-- | osc/core.py | 175 | ||||
-rw-r--r-- | osc/fetch.py | 13 | ||||
-rw-r--r-- | osc/oscssl.py | 52 | ||||
-rw-r--r-- | osc/util/debquery.py | 4 | ||||
-rw-r--r-- | osc/util/helper.py | 69 | ||||
-rw-r--r-- | osc/util/packagequery.py | 9 | ||||
-rw-r--r-- | osc/util/repodata.py | 6 | ||||
-rw-r--r-- | osc/util/rpmquery.py | 18 | ||||
-rwxr-xr-x | setup.py | 18 | ||||
-rw-r--r-- | tests/suite.py | 2 | ||||
-rw-r--r-- | tests/test_helpers.py | 35 |
21 files changed, 236 insertions, 468 deletions
diff --git a/.travis.yml b/.travis.yml index ea7e690..d1efa2c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,9 @@ language: python python: - '2.7' -- '3.3' -- '3.4' -- '3.6' +#- '3.3' +#- '3.4' +#- 3.5-dev addons: apt: packages: @@ -1,14 +1,3 @@ -0.165.1 - - fix oscssl "urldefrag is not defined error" - - osc release command now python3 compatible - - add more decode logic in get_commitlog - - osc add <dir> in compressed mode now works with python3 - - osc getbinaries now prints the output instead of - using the quiet mode as a default - -0.165.0 - - full python3 compatibility - 0.164.2 - deleterequest for entire projects needs the --all option as additional protection - rewrite packagequery to support python3 diff --git a/debian/changelog b/debian/changelog index ffad830..5378dde 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,9 +1,9 @@ -osc (0.165.1-1) UNRELEASED; urgency=medium +osc (0.164.2-2) unstable; urgency=medium - * New upstream release. - * Add myself to uploaders (Closes: #923351) + * QA upload. + * Switch to python3. Closes: #830873, #937218 - -- Héctor Orón Martínez <zumbi@debian.org> Sat, 15 Jun 2019 20:31:38 +0200 + -- Dimitri John Ledkov <xnox@ubuntu.com> Thu, 19 Dec 2019 17:18:03 +0000 osc (0.164.2-1) unstable; urgency=medium diff --git a/debian/compat b/debian/compat deleted file mode 100644 index ec63514..0000000 --- a/debian/compat +++ /dev/null @@ -1 +0,0 @@ -9 diff --git a/debian/control b/debian/control index 7ed25ff..d92ae5d 100644 --- a/debian/control +++ b/debian/control @@ -1,14 +1,14 @@ Source: osc Maintainer: RPM packaging team <team+pkg-rpm@tracker.debian.org> -Uploaders: Héctor Orón Martínez <zumbi@debian.org> +Uploaders: Michal Čihař <nijel@debian.org> Section: devel Priority: optional Build-Depends: bash-completion, - debhelper (>= 9), + debhelper-compat (=12), dh-exec, dh-python, - python, - python-urlgrabber + python3, + python3-m2crypto, Standards-Version: 4.3.0 Vcs-Browser: https://salsa.debian.org/pkg-rpm-team/osc Vcs-Git: https://salsa.debian.org/pkg-rpm-team/osc.git @@ -18,15 +18,13 @@ Package: osc Architecture: all Depends: ${shlibs:Depends}, ${misc:Depends}, - ${python:Depends}, - python-urlgrabber, - python-m2crypto, + ${python3:Depends}, ca-certificates -Recommends: python-rpm, +Recommends: python3-rpm, rpm2cpio, cpio, sensible-utils, - python-keyring, + python3-keyring, obs-build, bash-completion Description: Open Build Service commander diff --git a/debian/rules b/debian/rules index d6337f1..ac1a3d6 100755 --- a/debian/rules +++ b/debian/rules @@ -1,7 +1,7 @@ #!/usr/bin/make -f %: - dh $@ --with python2,bash-completion + dh $@ --with python3,bash-completion --buildsystem=pybuild override_dh_installchangelogs: dh_installchangelogs NEWS diff --git a/osc/babysitter.py b/osc/babysitter.py index 6ca9253..4b2583a 100644 --- a/osc/babysitter.py +++ b/osc/babysitter.py @@ -16,7 +16,6 @@ from osc import oscerr from .oscsslexcp import NoSecureSSLError from osc.util.cpio import CpioError from osc.util.packagequery import PackageError -from osc.util.helper import decode_it try: from M2Crypto.SSL.Checker import SSLVerificationError @@ -113,11 +112,11 @@ def run(prg, argv=None): print(body, file=sys.stderr) if e.code in [400, 403, 404, 500]: - if b'<summary>' in body: - msg = body.split(b'<summary>')[1] - msg = msg.split(b'</summary>')[0] - msg = msg.replace(b'<', b'<').replace(b'>' , b'>').replace(b'&', b'&') - print(decode_it(msg), file=sys.stderr) + if '<summary>' in body: + msg = body.split('<summary>')[1] + msg = msg.split('</summary>')[0] + msg = msg.replace('<', '<').replace('>' , '>').replace('&', '&') + print(msg, file=sys.stderr) if e.code >= 500 and e.code <= 599: print('\nRequest: %s' % e.filename) print('Headers:') diff --git a/osc/build.py b/osc/build.py index 457249e..b26ae9c 100644 --- a/osc/build.py +++ b/osc/build.py @@ -23,7 +23,6 @@ from osc.fetch import * from osc.core import get_buildinfo, store_read_apiurl, store_read_project, store_read_package, meta_exists, quote_plus, get_buildconfig, is_package_dir, dgst from osc.core import get_binarylist, get_binary_file, run_external, return_external, raw_input from osc.util import rpmquery, debquery, archquery -from osc.util.helper import decode_it import osc.conf from . import oscerr import subprocess @@ -441,11 +440,11 @@ def get_prefer_pkgs(dirs, wanted_arch, type, cpio): packageQuery = packagequery.PackageQuery.query(path) packageQueries.add(packageQuery) - prefer_pkgs = dict((decode_it(name), packageQuery.path()) + prefer_pkgs = dict((name, packageQuery.path()) for name, packageQuery in packageQueries.items()) depfile = create_deps(packageQueries.values()) - cpio.add(b'deps', b'\n'.join(depfile)) + cpio.add('deps', '\n'.join(depfile)) return prefer_pkgs @@ -456,22 +455,22 @@ def create_deps(pkgqs): """ depfile = [] for p in pkgqs: - id = b'%s.%s-0/0/0: ' % (p.name(), p.arch()) - depfile.append(b'P:%s%s' % (id, b' '.join(p.provides()))) - depfile.append(b'R:%s%s' % (id, b' '.join(p.requires()))) + id = '%s.%s-0/0/0: ' % (p.name(), p.arch()) + depfile.append('P:%s%s' % (id, ' '.join(p.provides()))) + depfile.append('R:%s%s' % (id, ' '.join(p.requires()))) d = p.conflicts() if d: - depfile.append(b'C:%s%s' % (id, b' '.join(d))) + depfile.append('C:%s%s' % (id, ' '.join(d))) d = p.obsoletes() if d: - depfile.append(b'O:%s%s' % (id, b' '.join(d))) + depfile.append('O:%s%s' % (id, ' '.join(d))) d = p.recommends() if d: - depfile.append(b'r:%s%s' % (id, b' '.join(d))) + depfile.append('r:%s%s' % (id, ' '.join(d))) d = p.supplements() if d: - depfile.append(b's:%s%s' % (id, b' '.join(d))) - depfile.append(b'I:%s%s-%s 0-%s' % (id, p.name(), p.evr().encode(), p.arch())) + depfile.append('s:%s%s' % (id, ' '.join(d))) + depfile.append('I:%s%s-%s 0-%s' % (id, p.name(), p.evr(), p.arch())) return depfile @@ -499,22 +498,6 @@ def check_trusted_projects(apiurl, projects): config['api_host_options'][apiurl]['trusted_prj'] = trusted conf.config_set_option(apiurl, 'trusted_prj', ' '.join(trusted)) -def get_kiwipath_from_buildinfo(apiurl, bi_filename, prj, repo): - bi = Buildinfo(bi_filename, apiurl, 'kiwi') - # If the project does not have a path defined we need to get the config - # via the repositories in the kiwi file. Unfortunately the buildinfo - # does not include a hint if this is the case, so we rely on a heuristic - # here: if the path list contains our own repo, it probably does not - # come from the kiwi file and thus a path is defined in the config. - # It is unlikely that our own repo is included in the kiwi file, as it - # contains no packages. - myprp = prj + '/' + repo - if myprp in bi.pathes: - return None - kiwipath = bi.pathes - kiwipath.insert(0, myprp) - return kiwipath - def main(apiurl, opts, argv): repo = argv[0] @@ -678,24 +661,24 @@ def main(apiurl, opts, argv): extra_pkgs += xp prefer_pkgs = {} - build_descr_data = open(build_descr, 'rb').read() + build_descr_data = open(build_descr).read() # XXX: dirty hack but there's no api to provide custom defines if opts.without: s = '' for i in opts.without: s += "%%define _without_%s 1\n" % i - build_descr_data = s.encode() + build_descr_data + build_descr_data = s + build_descr_data if opts._with: s = '' for i in opts._with: s += "%%define _with_%s 1\n" % i - build_descr_data = s.encode() + build_descr_data + build_descr_data = s + build_descr_data if opts.define: s = '' for i in opts.define: s += "%%define %s\n" % i - build_descr_data = s.encode + build_descr_data + build_descr_data = s + build_descr_data cpiodata = None servicefile = os.path.join(os.path.dirname(build_descr), "_service") @@ -725,12 +708,12 @@ def main(apiurl, opts, argv): prefer_pkgs = get_prefer_pkgs(opts.prefer_pkgs, arch, build_type, cpiodata) if cpiodata: - cpiodata.add(os.path.basename(build_descr.encode()), build_descr_data) + cpiodata.add(os.path.basename(build_descr), build_descr_data) # buildenv must come last for compatibility reasons... if buildenvfile: - cpiodata.add(b"buildenv", open(buildenvfile, 'rb').read()) + cpiodata.add("buildenv", open(buildenvfile).read()) if servicefile: - cpiodata.add(b"_service", open(servicefile, 'rb').read()) + cpiodata.add("_service", open(servicefile).read()) build_descr_data = cpiodata.get() # special handling for overlay and rsync-src/dest @@ -784,27 +767,23 @@ def main(apiurl, opts, argv): raise oscerr.WrongOptions('--offline is not possible, no local buildconfig file') else: print('Getting buildinfo from server and store to %s' % bi_filename) - - bi_text = decode_it(get_buildinfo(apiurl, - prj, - pac, - repo, - arch, - specfile=build_descr_data, - addlist=extra_pkgs)) + bi_text = ''.join(get_buildinfo(apiurl, + prj, + pac, + repo, + arch, + specfile=build_descr_data, + addlist=extra_pkgs)) if not bi_file: bi_file = open(bi_filename, 'w') # maybe we should check for errors before saving the file bi_file.write(bi_text) bi_file.flush() - kiwipath = None - if build_type == 'kiwi': - kiwipath = get_kiwipath_from_buildinfo(apiurl, bi_filename, prj, repo) print('Getting buildconfig from server and store to %s' % bc_filename) - bc = get_buildconfig(apiurl, prj, repo, kiwipath) + bc = get_buildconfig(apiurl, prj, repo) if not bc_file: bc_file = open(bc_filename, 'w') - bc_file.write(decode_it(bc)) + bc_file.write(bc) bc_file.flush() except HTTPError as e: if e.code == 404: @@ -835,7 +814,7 @@ def main(apiurl, opts, argv): # Set default binary type if cannot be detected binary_type = 'rpm' if os.path.exists('/usr/lib/build/queryconfig'): - binary_type = decode_it(return_external('/usr/lib/build/queryconfig', '--dist', bc_filename, 'binarytype')).strip() + binary_type = return_external('/usr/lib/build/queryconfig', '--dist', bc_filename, 'binarytype').decode('utf-8').strip() # If binary type is set to a useless value, reset to 'rpm' if binary_type == 'UNDEFINED': binary_type = 'rpm' @@ -1163,7 +1142,7 @@ def main(apiurl, opts, argv): if bi.installonly_list: rpmlist.append('installonly: ' + ' '.join(bi.installonly_list) + '\n') - rpmlist_file = NamedTemporaryFile(mode='w+t', prefix='rpmlist.') + rpmlist_file = NamedTemporaryFile(prefix='rpmlist.') rpmlist_filename = rpmlist_file.name rpmlist_file.writelines(rpmlist) rpmlist_file.flush() @@ -1263,13 +1242,13 @@ def main(apiurl, opts, argv): (s_built, b_built) = get_built_files(pacdir, bi.buildtype) print() - if s_built: print(decode_it(s_built)) + if s_built: print(s_built) print() - print(decode_it(b_built)) + print(b_built) if opts.keep_pkgs: for i in b_built.splitlines() + s_built.splitlines(): - shutil.copy2(i, os.path.join(opts.keep_pkgs, os.path.basename(decode_it(i)))) + shutil.copy2(i, os.path.join(opts.keep_pkgs, os.path.basename(i))) if bi_file: bi_file.close() diff --git a/osc/cmdln.py b/osc/cmdln.py index 918a1da..3d303b0 100644 --- a/osc/cmdln.py +++ b/osc/cmdln.py @@ -621,16 +621,16 @@ class RawCmdln(cmd.Cmd): ${name} man """ mandate = datetime.utcfromtimestamp(int(os.environ.get('SOURCE_DATE_EPOCH', time.time()))) - self.stdout.write( + self.stdout.write(bytes( self.man_header % { 'date': mandate.strftime('%b %Y'), 'version': self.get_version(), 'name': self.name, 'ucname': self.name.upper() - } - ) + }, + "utf-8")) - self.stdout.write(self.man_commands_header) + self.stdout.write(bytes(self.man_commands_header, "utf-8")) commands = self._help_get_command_list() for command, doc in commands: cmdname = command.split(' ')[0] @@ -641,14 +641,14 @@ class RawCmdln(cmd.Cmd): line = line[8:] lines.append(man_escape(line)) - self.stdout.write( - '.TP\n\\fB%s\\fR\n%s\n' % (command, '\n'.join(lines))) + self.stdout.write(bytes( + '.TP\n\\fB%s\\fR\n%s\n' % (command, '\n'.join(lines)), "utf-8")) - self.stdout.write(self.man_options_header) - self.stdout.write( - man_escape(self._help_preprocess('${option_list}', None))) + self.stdout.write(bytes(self.man_options_header, "utf-8")) + self.stdout.write(bytes( + man_escape(self._help_preprocess('${option_list}', None)), "utf-8")) - self.stdout.write(self.man_footer) + self.stdout.write(bytes(self.man_footer, "utf-8")) self.stdout.flush() diff --git a/osc/commandline.py b/osc/commandline.py index 05faaac..e96a117 100644 --- a/osc/commandline.py +++ b/osc/commandline.py @@ -29,13 +29,6 @@ from optparse import SUPPRESS_HELP from .core import * from .util import safewriter -try: - from functools import cmp_to_key -except ImportError: - from .util.helper import cmp_to_key - -from operator import itemgetter - MAN_HEADER = r""".TH %(ucname)s "1" "%(date)s" "%(name)s %(version)s" "User Commands" .SH NAME %(name)s \- openSUSE build service command-line tool. @@ -442,8 +435,7 @@ class Osc(cmdln.Cmdln): break m = show_files_meta(apiurl, project, package) li = Linkinfo() - root = ET.fromstring(m) - li.read(root.find('linkinfo')) + li.read(ET.fromstring(''.join(m)).find('linkinfo')) if li.haserror(): raise oscerr.LinkExpandError(project, package, li.error) project, package, rev = li.project, li.package, li.rev @@ -761,7 +753,7 @@ class Osc(cmdln.Cmdln): buf = f.read(16384) if not buf: break - sys.stdout.write(decode_it(buf)) + sys.stdout.write(buf) elif opts.delete: print("Delete token") @@ -781,7 +773,7 @@ class Osc(cmdln.Cmdln): raise oscerr.WrongArgs("Did you mean --" + args[0] + "?") # just list token for data in streamfile(url, http_GET): - sys.stdout.write(decode_it(data)) + sys.stdout.write(data) @cmdln.option('-a', '--attribute', metavar='ATTRIBUTE', @@ -946,22 +938,22 @@ class Osc(cmdln.Cmdln): # show if not opts.edit and not opts.file and not opts.delete and not opts.create and not opts.set: if cmd == 'prj': - sys.stdout.write(decode_it(b''.join(show_project_meta(apiurl, project, rev=opts.revision, blame=opts.blame)))) + sys.stdout.write(''.join(show_project_meta(apiurl, project, rev=opts.revision, blame=opts.blame))) elif cmd == 'pkg': - sys.stdout.write(decode_it(b''.join(show_package_meta(apiurl, project, package, blame=opts.blame)))) + sys.stdout.write(''.join(show_package_meta(apiurl, project, package, blame=opts.blame))) elif cmd == 'attribute': - sys.stdout.write(decode_it(b''.join(show_attribute_meta(apiurl, project, package, subpackage, - opts.attribute, opts.attribute_defaults, opts.attribute_project)))) + sys.stdout.write(''.join(show_attribute_meta(apiurl, project, package, subpackage, + opts.attribute, opts.attribute_defaults, opts.attribute_project))) elif cmd == 'prjconf': - sys.stdout.write(decode_it(b''.join(show_project_conf(apiurl, project, rev=opts.revision, blame=opts.blame)))) + sys.stdout.write(''.join(show_project_conf(apiurl, project, rev=opts.revision, blame=opts.blame))) elif cmd == 'user': r = get_user_meta(apiurl, user) if r: - sys.stdout.write(decode_it(r)) + sys.stdout.write(''.join(r)) elif cmd == 'group': r = get_group_meta(apiurl, group) if r: - sys.stdout.write(decode_it(r)) + sys.stdout.write(''.join(r)) elif cmd == 'pattern': if pattern: r = show_pattern_meta(apiurl, project, pattern) @@ -1034,7 +1026,7 @@ class Osc(cmdln.Cmdln): d = '<attributes><attribute namespace=\'%s\' name=\'%s\' >%s</attribute></attributes>' % (aname[0], aname[1], values) url = makeurl(apiurl, attributepath) for data in streamfile(url, http_POST, data=d): - sys.stdout.write(decode_it(data)) + sys.stdout.write(data) # upload file if opts.file: @@ -1398,9 +1390,9 @@ Please submit there instead, or use --nodevelproject to force direct submission. if opts.diff or not opts.message: try: rdiff = 'old: %s/%s\nnew: %s/%s rev %s\n' % (dst_project, dst_package, src_project, src_package, rev) - rdiff += decode_it(server_diff(apiurl, + rdiff += server_diff(apiurl, dst_project, dst_package, None, - src_project, src_package, rev, True)) + src_project, src_package, rev, True) except: rdiff = '' @@ -2491,7 +2483,7 @@ Please submit there instead, or use --nodevelproject to force direct submission. action.tgt_project, action.tgt_package) diff += submit_action_diff(apiurl, action) diff += '\n\n' - run_pager(decode_it(diff), tmp_suffix='') + run_pager(diff, tmp_suffix='') # checkout elif cmd == 'checkout' or cmd == 'co': @@ -2991,7 +2983,7 @@ Please submit there instead, or use --nodevelproject to force direct submission. revision=rev, comment=comment, keep_link=opts.keep_link) - print(decode_it(r)) + print(r) @cmdln.option('-r', '--repo', metavar='REPO', @@ -3058,7 +3050,7 @@ Please submit there instead, or use --nodevelproject to force direct submission. buf = f.read(16384) if not buf: break - sys.stdout.write(decode_it(buf)) + sys.stdout.write(buf) @cmdln.option('-m', '--message', metavar='TEXT', @@ -3503,7 +3495,7 @@ Please submit there instead, or use --nodevelproject to force direct submission. devloc = None if not exists and (srcprj != args[0] or srcpkg != args[1]): try: - root = ET.fromstring(b''.join(show_attribute_meta(apiurl, args[0], None, None, + root = ET.fromstring(''.join(show_attribute_meta(apiurl, args[0], None, None, conf.config['maintained_update_project_attribute'], False, False))) # this might raise an AttributeError uproject = root.find('attribute').find('value').text @@ -3657,7 +3649,7 @@ Please submit there instead, or use --nodevelproject to force direct submission. kind = 'pkg' path_args = (project, package) meta = meta_exists(kind, path_args, create_new=False, apiurl=apiurl) - root = ET.fromstring(b''.join(meta)) + root = ET.fromstring(''.join(meta)) if root.find('lock') is not None: print('Already locked', file=sys.stderr) sys.exit(1) @@ -3862,9 +3854,9 @@ Please submit there instead, or use --nodevelproject to force direct submission. for i in pac.get_diff(rev1): diff += ''.join(i) else: - diff += decode_it(server_diff_noex(pac.apiurl, pac.prjname, pac.name, rev1, + diff += server_diff_noex(pac.apiurl, pac.prjname, pac.name, rev1, pac.prjname, pac.name, rev2, - not opts.plain, opts.missingok, opts.meta, not opts.unexpand)) + not opts.plain, opts.missingok, opts.meta, not opts.unexpand) run_pager(diff) @@ -4143,12 +4135,12 @@ Please submit there instead, or use --nodevelproject to force direct submission. stdin=subprocess.PIPE, stdout=subprocess.PIPE, close_fds=True) - p.stdin.write(rdiff) + p.stdin.write(rdiff.encode()) p.stdin.close() - print("".join(decode_it(x) for x in p.stdout.readlines())) + print("".join(x.decode() for x in p.stdout.readlines())) elif opts.unified: print() - print(decode_it(rdiff)) + print(rdiff) #run_pager(rdiff) def _prdiff_output_matching_requests(self, opts, requests, @@ -4293,7 +4285,7 @@ Please submit there instead, or use --nodevelproject to force direct submission. else: raise oscerr.WrongArgs('Wrong number of arguments') - root = ET.fromstring(b''.join(show_configuration(apiurl))) + root = ET.fromstring(''.join(show_configuration(apiurl))) elm = root.find('download_url') if elm is None or not elm.text: raise oscerr.APIError('download_url configuration element expected') @@ -4538,7 +4530,7 @@ Please submit there instead, or use --nodevelproject to force direct submission. # don't exclude packages with state ' ' because the packages # might have modified etc. files prj_excl = [st for st in excl_states if st != ' '] - for st, pac in sorted(prj.get_status(*prj_excl), key=cmp_to_key(compare)): + for st, pac in sorted(prj.get_status(*prj_excl), lambda x, y: cmp(x[1], y[1])): p = prj.get_pacobj(pac) if p is None: # state is != ' ' @@ -4549,11 +4541,11 @@ Please submit there instead, or use --nodevelproject to force direct submission. elif st == ' ' and opts.verbose or st != ' ': lines.append(statfrmt(st, os.path.normpath(os.path.join(prj.dir, pac)))) states = p.get_status(opts.show_excluded, *excl_states) - for st, filename in sorted(states, key=cmp_to_key(compare)): + for st, filename in sorted(states, lambda x, y: cmp(x[1], y[1])): lines.append(statfrmt(st, os.path.normpath(os.path.join(p.dir, filename)))) else: p = findpacs([arg])[0] - for st, filename in sorted(p.get_status(opts.show_excluded, *excl_states), key=cmp_to_key(compare)): + for st, filename in sorted(p.get_status(opts.show_excluded, *excl_states), lambda x, y: cmp(x[1], y[1])): lines.append(statfrmt(st, os.path.normpath(os.path.join(p.dir, filename)))) if lines: print('\n'.join(lines)) @@ -5235,7 +5227,7 @@ Please submit there instead, or use --nodevelproject to force direct submission. del kwargs['showexcl'] for xml in get_package_results(**kwargs): if opts.xml: - print(decode_it(xml), end='') + print(xml, end='') else: # csv formatting results = [r for r, _ in result_xml_to_dicts(xml)] @@ -5298,7 +5290,7 @@ Please submit there instead, or use --nodevelproject to force direct submission. kwargs['arch'] = opts.arch kwargs['wait'] = opts.watch for results in get_package_results(apiurl, project, **kwargs): - print(decode_it(results)) + print(results) return if opts.watch: @@ -5353,7 +5345,7 @@ Please submit there instead, or use --nodevelproject to force direct submission. else: raise oscerr.WrongArgs('please provide project package repository arch.') - print(decode_it(get_rpmlint_log(apiurl, project, package, repository, arch))) + print(get_rpmlint_log(apiurl, project, package, repository, arch)) @cmdln.alias('bl') @cmdln.alias('blt') @@ -5799,7 +5791,7 @@ Please submit there instead, or use --nodevelproject to force direct submission. build_descr_data = None if not build_descr is None: - build_descr_data = open(build_descr, 'rb').read() + build_descr_data = open(build_descr, 'r').read() if opts.prefer_pkgs and build_descr_data is None: raise oscerr.WrongArgs('error: a build description is needed if \'--prefer-pkgs\' is used') elif opts.prefer_pkgs: @@ -5810,13 +5802,13 @@ Please submit there instead, or use --nodevelproject to force direct submission. prefer_pkgs = get_prefer_pkgs(opts.prefer_pkgs, arch, os.path.splitext(build_descr)[1], cpiodata) - cpiodata.add(os.path.basename(build_descr.encode()), build_descr_data) + cpiodata.add(os.path.basename(build_descr), build_descr_data) build_descr_data = cpiodata.get() if opts.multibuild_package: package = package + ":" + opts.multibuild_package - print(decode_it(get_buildinfo(apiurl, + print(''.join(get_buildinfo(apiurl, project, package, repository, arch, specfile=build_descr_data, debug=opts.debug, @@ -5863,7 +5855,7 @@ Please submit there instead, or use --nodevelproject to force direct submission. else: raise oscerr.WrongArgs('Wrong number of arguments.') - print(decode_it(get_buildconfig(apiurl, project, repository))) + print(''.join(get_buildconfig(apiurl, project, repository))) def do_workerinfo(self, subcmd, opts, worker): @@ -6105,7 +6097,6 @@ Please submit there instead, or use --nodevelproject to force direct submission. recipe = recipe.strip() if recipe == 'arch': recipe = 'PKGBUILD' - recipe = decode_it(recipe) pac = os.path.basename(os.getcwd()) if is_package_dir(os.getcwd()): pac = store_read_package(os.getcwd()) @@ -6300,16 +6291,14 @@ Please submit there instead, or use --nodevelproject to force direct submission. args = self.parse_repoarchdescr(args, opts.noinit or opts.offline, opts.alternative_project, False, opts.vm_type, opts.multibuild_package) # check for source services - if not opts.offline and not opts.noservice: - p = Package('.') - r = p.run_source_services(verbose=True) - if r: - print('Source service run failed!', file=sys.stderr) - sys.exit(1) - else: - msg = ('WARNING: source services from package or project will not' - 'be executed. This may not be the same build as on server!') - print(msg) + r = None + try: + if not opts.offline and not opts.noservice: + p = Package('.') + r = p.run_source_services(verbose=True) + except: + print("WARNING: package is not existing on server yet") + opts.local_package = True if not opts.local_package: try: @@ -6322,6 +6311,15 @@ Please submit there instead, or use --nodevelproject to force direct submission. except oscerr.NoWorkingCopy: pass + if opts.offline or opts.local_package or r == None: + print("WARNING: source service from package or project will not be executed. This may not be the same build as on server!") + elif (conf.config['local_service_run'] and not opts.noservice) and not opts.noinit: + if r != 0: + print('Source service run failed!', file=sys.stderr) + sys.exit(1) + # that is currently unreadable on cli, we should not have a backtrace on standard errors: + #raise oscerr.ServiceRuntimeError('Service run failed: \'%s\'', r) + if conf.config['no_verify']: opts.no_verify = True @@ -7375,7 +7373,7 @@ Please submit there instead, or use --nodevelproject to force direct submission. what = {'project': ''} elif type in args_sr: requests = get_request_collection(apiurl, 'creator', req_who=user) - for r in sorted(requests, key=lambda x: x.reqid): + for r in sorted(requests): print(r.list_view(), '\n') return elif not type in args_pkg: @@ -7504,7 +7502,7 @@ Please submit there instead, or use --nodevelproject to force direct submission. if list_requests: # old style, only for OBS 2.1 and before. Should not be used, since it is slow and incomplete requests = get_user_projpkgs_request_list(apiurl, user, projpkgs=request_todo) - for r in sorted(requests, key=lambda x: x.reqid): + for r in sorted(requests): print(r.list_view(), '\n') if not len(requests): print(" -> try also 'osc my sr' to see more.") @@ -7743,12 +7741,10 @@ Please submit there instead, or use --nodevelproject to force direct submission. continue # construct a sorted, flat list # Sort by first column, follwed by second column if we have two columns, else sort by first. - if len(results[0]) > 1: - sorted_results = sorted(results, key=itemgetter(0,1)) - else: - sorted_results = sorted(results, key=itemgetter(0)) + results.sort(lambda x, y: ( cmp(x[0], y[0]) or + (len(x)>1 and len(y)>1 and cmp(x[1], y[1])) )) new = [] - for i in sorted_results: + for i in results: new.extend(i) results = new headline = [] @@ -7966,7 +7962,7 @@ Please submit there instead, or use --nodevelproject to force direct submission. data=opts.data, file=opts.file, headers=opts.headers) - out = decode_it(r.read()) + out = r.read() if opts.edit: text = edit_text(out) @@ -7974,7 +7970,7 @@ Please submit there instead, or use --nodevelproject to force direct submission. url, data=text, headers=opts.headers) - out = decode_it(r.read()) + out = r.read() sys.stdout.write(out) @@ -8188,7 +8184,7 @@ Please submit there instead, or use --nodevelproject to force direct submission. else: if pac: m = show_package_meta(apiurl, prj, pac) - metaroot = ET.fromstring(b''.join(m)) + metaroot = ET.fromstring(''.join(m)) if not opts.nodevelproject: while metaroot.findall('devel'): d = metaroot.find('devel') @@ -8197,18 +8193,18 @@ Please submit there instead, or use --nodevelproject to force direct submission. if opts.verbose: print("Following to the development space: %s/%s" % (prj, pac)) m = show_package_meta(apiurl, prj, pac) - metaroot = ET.fromstring(b''.join(m)) + metaroot = ET.fromstring(''.join(m)) if not metaroot.findall('person') and not metaroot.findall('group'): if opts.verbose: print("No dedicated persons in package defined, showing the project persons.") pac = None m = show_project_meta(apiurl, prj) - metaroot = ET.fromstring(b''.join(m)) + metaroot = ET.fromstring(''.join(m)) else: # fallback to project lookup for old servers if prj and not searchresult: m = show_project_meta(apiurl, prj) - metaroot = ET.fromstring(b''.join(m)) + metaroot = ET.fromstring(''.join(m)) # extract the maintainers projects = [] @@ -8369,13 +8365,10 @@ Please submit there instead, or use --nodevelproject to force direct submission. u = makeurl(apiurl, ['source', project, package, filename], query=query) if subcmd == 'less': f = http_GET(u) - run_pager(b''.join(f.readlines())) + run_pager(''.join(f.readlines())) else: for data in streamfile(u): - if isinstance(data, str): - sys.stdout.write(data) - else: - sys.stdout.write(decode_it(data)) + sys.stdout.write(data) # helper function to download a file from a specific revision @@ -8471,7 +8464,7 @@ Please submit there instead, or use --nodevelproject to force direct submission. u = makeurl(apiurl, ['source', prj, package], query=query) f = http_GET(u) meta = f.readlines() - root_new = ET.fromstring(b''.join(meta)) + root_new = ET.fromstring(''.join(meta)) dir_new = { 'apiurl': apiurl, 'project': prj, 'package': package } dir_new['srcmd5'] = root_new.get('srcmd5') dir_new['entries'] = [[n.get('name'), n.get('md5')] for n in root_new.findall('entry')] @@ -8515,7 +8508,7 @@ Please submit there instead, or use --nodevelproject to force direct submission. os.mkdir(destdir) Package.init_package(apiurl, target_prj, target_package, destdir) - store_write_string(destdir, '_files', b''.join(meta) + b'\n') + store_write_string(destdir, '_files', ''.join(meta) + '\n') store_write_string(destdir, '_linkrepair', '') pac = Package(destdir) @@ -8624,7 +8617,7 @@ Please submit there instead, or use --nodevelproject to force direct submission. u = makeurl(p.apiurl, ['source', p.prjname, p.name], query=query) f = http_GET(u) meta = f.readlines() - root_new = ET.fromstring(b''.join(meta)) + root_new = ET.fromstring(''.join(meta)) linkinfo_new = root_new.find('linkinfo') if linkinfo_new == None: raise oscerr.APIError('link is not a really a link?') @@ -8797,7 +8790,7 @@ Please submit there instead, or use --nodevelproject to force direct submission. buf = f.read(16384) if not buf: break - sys.stdout.write(decode_it(buf)) + sys.stdout.write(buf) @cmdln.option('-m', '--message', help='add MESSAGE to changes (do not open an editor)') @@ -8837,7 +8830,7 @@ Please submit there instead, or use --nodevelproject to force direct submission. import glob, re try: fn_changelog = glob.glob('*.changes')[0] - fp = open(fn_changelog) + fp = file(fn_changelog) titleline = fp.readline() fp.close() if re.match('^\*\W+(.+\W+\d{1,2}\W+20\d{2})\W+(.+)\W+<(.+)>\W+(.+)$', titleline): diff --git a/osc/core.py b/osc/core.py index 990291e..0a24d08 100644 --- a/osc/core.py +++ b/osc/core.py @@ -5,7 +5,7 @@ from __future__ import print_function -__version__ = '0.165.1' +__version__ = '0.164.2' # __store_version__ is to be incremented when the format of the working copy # "store" changes in an incompatible way. Please add any needed migration @@ -24,7 +24,6 @@ import errno import shlex import hashlib - try: from urllib.parse import urlsplit, urlunsplit, urlparse, quote_plus, urlencode, unquote from urllib.error import HTTPError @@ -51,13 +50,6 @@ from . import oscerr from . import conf try: - from functools import cmp_to_key -except ImportError: - from .util.helper import cmp_to_key - -from osc.util.helper import decode_list, decode_it - -try: # python 2.6 and python 2.7 unicode ET_ENCODING = "utf-8" @@ -70,11 +62,6 @@ except: unicode = lambda x, *args: x ET_ENCODING = "unicode" -def compare(a, b): return cmp(a[1:], b[1:]) - -def cmp(a, b): - return (a > b) - (a < b) - DISTURL_RE = re.compile(r"^(?P<bs>.*)://(?P<apiurl>.*?)/(?P<project>.*?)/(?P<repository>.*?)/(?P<revision>.*)-(?P<source>.*)$") BUILDLOGURL_RE = re.compile(r"^(?P<apiurl>https?://.*?)/build/(?P<project>.*?)/(?P<repository>.*?)/(?P<arch>.*?)/(?P<package>.*?)/_log$") BUFSIZE = 1024*1024 @@ -344,7 +331,7 @@ class Serviceinfo: def addVerifyFile(self, serviceinfo_node, filename): import hashlib - f = open(filename, 'rb') + f = open(filename, 'r') digest = hashlib.sha256(f.read()).hexdigest() f.close() @@ -1862,8 +1849,8 @@ class Package: meta = show_package_meta(self.apiurl, self.prjname, self.name) if meta != "": # is empty for _project for example - meta = b''.join(meta) - store_write_string(self.absdir, '_meta', meta + b'\n') + meta = ''.join(meta) + store_write_string(self.absdir, '_meta', meta + '\n') def findfilebyname(self, n): for i in self.filelist: @@ -2128,7 +2115,7 @@ rev: %s argument force supress the confirm question """ - m = b''.join(show_package_meta(self.apiurl, self.prjname, self.name)) + m = ''.join(show_package_meta(self.apiurl, self.prjname, self.name)) root = ET.fromstring(m) root.find('title').text = self.summary @@ -2143,7 +2130,7 @@ rev: %s if not force: print('*' * 36, 'old', '*' * 36) - print(decode_it(m)) + print(m) print('*' * 36, 'new', '*' * 36) print(ET.tostring(root, encoding=ET_ENCODING)) print('*' * 72) @@ -3578,7 +3565,7 @@ def show_attribute_meta(apiurl, prj, pac, subpac, attribute, with_defaults, with def show_devel_project(apiurl, prj, pac): m = show_package_meta(apiurl, prj, pac) - node = ET.fromstring(b''.join(m)).find('devel') + node = ET.fromstring(''.join(m)).find('devel') if node is None: return None, None else: @@ -3587,7 +3574,7 @@ def show_devel_project(apiurl, prj, pac): def set_devel_project(apiurl, prj, pac, devprj=None, devpac=None): meta = show_package_meta(apiurl, prj, pac) - root = ET.fromstring(b''.join(meta)) + root = ET.fromstring(''.join(meta)) node = root.find('devel') if node is None: if devprj is None: @@ -3660,12 +3647,8 @@ class metafile: self.url = url self.change_is_required = change_is_required (fd, self.filename) = tempfile.mkstemp(prefix = 'osc_metafile.', suffix = file_ext) - if not input or isinstance(input[0], str) or isinstance(input, str): - f = os.fdopen(fd, 'w') - f.write(''.join(input)) - else: - f = os.fdopen(fd, 'wb') - f.write(b''.join(input)) + f = os.fdopen(fd, 'w') + f.write(''.join(input)) f.close() self.hash_orig = dgst(self.filename) @@ -3698,8 +3681,8 @@ class metafile: # examine the error - we can't raise an exception because we might want # to try again data = e.read() - if b'<summary>' in data: - print(data.split(b'<summary>')[1].split(b'</summary>')[0], file=sys.stderr) + if '<summary>' in data: + print(data.split('<summary>')[1].split('</summary>')[0], file=sys.stderr) ri = raw_input('Try again? ([y/N]): ') if ri not in ['y', 'Y']: break @@ -3816,20 +3799,7 @@ def edit_meta(metatype, if metatype == 'pkg': # check if the package is a link to a different project project, package = path_args - # data can be a bytes object, a list with strings, a list with bytes, just a string. - # So we need the following even if it is ugly. - if sys.version_info >= (3, 0): - if isinstance(data, bytes): - data = decode_it(data) - orgprj = ET.fromstring(''.join(data)).get('project') - elif isinstance(data, list): - decode_data = decode_list(data) - orgprj = ET.fromstring(''.join(decode_data)).get('project') - else: - orgprj = ET.fromstring(''.join(data)).get('project') - else: - orgprj = ET.fromstring(''.join(data)).get('project') - + orgprj = ET.fromstring(''.join(data)).get('project') if orgprj is not None and unquote(project) != orgprj: print('The package is linked from a different project.') print('If you want to edit the meta of the package create first a branch.') @@ -3869,7 +3839,7 @@ def show_files_meta(apiurl, prj, pac, revision=None, expand=False, linkrev=None, def show_upstream_srcmd5(apiurl, prj, pac, expand=False, revision=None, meta=False, include_service_files=False, deleted=False): m = show_files_meta(apiurl, prj, pac, expand=expand, revision=revision, meta=meta, deleted=deleted) - et = ET.fromstring(m) + et = ET.fromstring(''.join(m)) if include_service_files: try: sinfo = et.find('serviceinfo') @@ -3882,7 +3852,7 @@ def show_upstream_srcmd5(apiurl, prj, pac, expand=False, revision=None, meta=Fal def show_upstream_xsrcmd5(apiurl, prj, pac, revision=None, linkrev=None, linkrepair=False, meta=False, include_service_files=False): m = show_files_meta(apiurl, prj, pac, revision=revision, linkrev=linkrev, linkrepair=linkrepair, meta=meta, expand=include_service_files) - et = ET.fromstring(m) + et = ET.fromstring(''.join(m)) if include_service_files: return et.get('srcmd5') @@ -3921,7 +3891,7 @@ def get_project_sourceinfo(apiurl, project, nofilename, *packages): raise if len(packages) == 1: raise oscerr.APIError('package name too long: %s' % packages[0]) - n = int(len(packages) / 2) + n = len(packages) / 2 pkgs = packages[:n] res = get_project_sourceinfo(apiurl, project, nofilename, *pkgs) pkgs = packages[n:] @@ -3936,12 +3906,12 @@ def get_project_sourceinfo(apiurl, project, nofilename, *packages): def show_upstream_rev_vrev(apiurl, prj, pac, revision=None, expand=False, meta=False): m = show_files_meta(apiurl, prj, pac, revision=revision, expand=expand, meta=meta) - et = ET.fromstring(m) + et = ET.fromstring(''.join(m)) return et.get('rev'), et.get('vrev') def show_upstream_rev(apiurl, prj, pac, revision=None, expand=False, linkrev=None, meta=False, include_service_files=False): m = show_files_meta(apiurl, prj, pac, revision=revision, expand=expand, linkrev=linkrev, meta=meta) - et = ET.fromstring(m) + et = ET.fromstring(''.join(m)) if include_service_files: try: sinfo = et.find('serviceinfo') @@ -4042,16 +4012,10 @@ def run_pager(message, tmp_suffix=''): return if not sys.stdout.isatty(): - if isinstance(message, str): - print(message) - else: - print(decode_it(message)) + print(message) else: tmpfile = tempfile.NamedTemporaryFile(suffix=tmp_suffix) - if isinstance(message, str): - tmpfile.write(bytes(message, 'utf-8')) - else: - tmpfile.write(message) + tmpfile.write(message) tmpfile.flush() pager = os.getenv('PAGER', default=get_default_pager()) cmd = shlex.split(pager) + [tmpfile.name] @@ -4078,8 +4042,6 @@ def _edit_message_open_editor(filename, data, orig_mtime): import tempfile editor = _editor_command() mtime = os.stat(filename).st_mtime - if isinstance(data, str): - data = bytes(data, 'utf-8') if mtime == orig_mtime: # prepare file for editors if editor[0] in ('vi', 'vim'): @@ -4089,7 +4051,7 @@ def _edit_message_open_editor(filename, data, orig_mtime): editor.extend(['-c', ':r %s' % f.name, filename]) run_external(editor[0], *editor[1:]) else: - with open(filename, 'wb') as f: + with open(filename, 'w') as f: f.write(data) orig_mtime = os.stat(filename).st_mtime run_editor(filename) @@ -4581,7 +4543,7 @@ def get_group_meta(apiurl, group): u = makeurl(apiurl, ['group', quote_plus(group)]) try: f = http_GET(u) - return b''.join(f.readlines()) + return ''.join(f.readlines()) except HTTPError: print('group \'%s\' not found' % group) return None @@ -4590,7 +4552,7 @@ def get_user_meta(apiurl, user): u = makeurl(apiurl, ['person', quote_plus(user)]) try: f = http_GET(u) - return b''.join(f.readlines()) + return ''.join(f.readlines()) except HTTPError: print('user \'%s\' not found' % user) return None @@ -4632,10 +4594,7 @@ def download(url, filename, progress_obj = None, mtime = None): try: o = os.fdopen(fd, 'wb') for buf in streamfile(url, http_GET, BUFSIZE, progress_obj=progress_obj): - if isinstance(buf, str): - o.write(bytes(buf, "utf-8")) - else: - o.write(buf) + o.write(bytes(buf, "utf-8")) o.close() os.rename(tmpfile, filename) except: @@ -4848,7 +4807,7 @@ def server_diff_noex(apiurl, msg = None body = None try: - body = decode_it(e.read()) + body = e.read() if not 'bad link' in body: return '# diff failed: ' + body except: @@ -5068,7 +5027,7 @@ def replace_pkg_meta(pkgmeta, new_name, new_prj, keep_maintainers = False, only maintainer (unless keep_maintainers is set). Additionally remove the develproject entry (<devel />) unless keep_develproject is true. """ - root = ET.fromstring(b''.join(pkgmeta)) + root = ET.fromstring(''.join(pkgmeta)) root.set('name', new_name) root.set('project', new_prj) # never take releasename, it needs to be explicit @@ -5077,8 +5036,6 @@ def replace_pkg_meta(pkgmeta, new_name, new_prj, keep_maintainers = False, if not keep_maintainers: for person in root.findall('person'): root.remove(person) - for group in root.findall('group'): - root.remove(group) if not keep_develproject: for dp in root.findall('devel'): root.remove(dp) @@ -5555,7 +5512,7 @@ def get_distibutions(apiurl, discon=False): else: result_line_templ = '%(name)-25s %(project)-25s %(repository)-25s %(reponame)s' f = http_GET(makeurl(apiurl, ['distributions'])) - root = ET.fromstring(b''.join(f)) + root = ET.fromstring(''.join(f)) for node in root.findall('distribution'): rmap = {} @@ -5581,7 +5538,7 @@ def get_platforms_of_project(apiurl, prj): def get_repositories_of_project(apiurl, prj): f = show_project_meta(apiurl, prj) - root = ET.fromstring(b''.join(f)) + root = ET.fromstring(''.join(f)) r = [ node.get('name') for node in root.findall('repository')] return r @@ -5623,7 +5580,7 @@ class Repo: def get_repos_of_project(apiurl, prj): f = show_project_meta(apiurl, prj) - root = ET.fromstring(b''.join(f)) + root = ET.fromstring(''.join(f)) for node in root.findall('repository'): for node2 in node.findall('arch'): @@ -5794,7 +5751,7 @@ def get_package_results(apiurl, project, package=None, wait=False, *args, **kwar while True: waiting = False try: - xml = b''.join(show_results_meta(apiurl, project, package, *args, **kwargs)) + xml = ''.join(show_results_meta(apiurl, project, package, *args, **kwargs)) except HTTPError as e: # check for simple timeout error and fetch again if e.code == 502 or e.code == 504: @@ -5835,7 +5792,7 @@ def get_prj_results(apiurl, prj, hide_legend=False, csv=False, status_filter=Non r = [] f = show_prj_results_meta(apiurl, prj) - root = ET.fromstring(b''.join(f)) + root = ET.fromstring(''.join(f)) pacs = [] # sequence of (repo,arch) tuples @@ -6015,6 +5972,7 @@ def get_prj_results(apiurl, prj, hide_legend=False, csv=False, status_filter=Non return r + def streamfile(url, http_meth = http_GET, bufsize=8192, data=None, progress_obj=None, text=None): """ performs http_meth on url and read bufsize bytes from the response @@ -6074,12 +6032,8 @@ def streamfile(url, http_meth = http_GET, bufsize=8192, data=None, progress_obj= def buildlog_strip_time(data): """Strips the leading build time from the log""" - if isinstance(data, str): - time_regex = re.compile('^\[[^\]]*\] ', re.M) - return time_regex.sub('', data) - else: - time_regex = re.compile(b'^\[[^\]]*\] ', re.M) - return time_regex.sub(b'', data) + time_regex = re.compile('^\[[^\]]*\] ', re.M) + return time_regex.sub('', data) def print_buildlog(apiurl, prj, package, repository, arch, offset=0, strip_time=False, last=False): @@ -6088,14 +6042,11 @@ def print_buildlog(apiurl, prj, package, repository, arch, offset=0, strip_time= def print_data(data, strip_time=False): if strip_time: data = buildlog_strip_time(data) - sys.stdout.write(decode_it(data.translate(all_bytes, remove_bytes))) + sys.stdout.write(data.translate(all_bytes, remove_bytes)) # to protect us against control characters import string - if sys.version_info >= (3, 0): - all_bytes = bytes.maketrans(b'', b'') - else: - all_bytes = string.maketrans(b'', b'') + all_bytes = string.maketrans('', '') remove_bytes = all_bytes[:8] + all_bytes[14:32] # accept tabs and newlines query = {'nostream' : '1', 'start' : '%s' % offset} @@ -6107,7 +6058,7 @@ def print_buildlog(apiurl, prj, package, repository, arch, offset=0, strip_time= start_offset = offset u = makeurl(apiurl, ['build', prj, repository, arch, package, '_log'], query=query) try: - for data in streamfile(u): + for data in streamfile(u, bufsize="line"): offset += len(data) print_data(data, strip_time) except IncompleteRead as e: @@ -6154,12 +6105,8 @@ def get_buildinfo(apiurl, prj, package, repository, arch, specfile=None, addlist return f.read() -def get_buildconfig(apiurl, prj, repository, path=None): - query = [] - if path: - for prp in path: - query.append('path=%s' % quote_plus(prp)) - u = makeurl(apiurl, ['build', prj, repository, '_buildconfig'], query=query) +def get_buildconfig(apiurl, prj, repository): + u = makeurl(apiurl, ['build', prj, repository, '_buildconfig']) f = http_GET(u) return f.read() @@ -6168,7 +6115,7 @@ def get_worker_info(apiurl, worker): u = makeurl(apiurl, ['worker', worker]) f = http_GET(u) - return decode_it(f.read()) + return f.read() def check_constraints(apiurl, prj, repository, arch, package, constraintsfile=None): @@ -6179,7 +6126,7 @@ def check_constraints(apiurl, prj, repository, arch, package, constraintsfile=No query['arch'] = arch u = makeurl(apiurl, ['worker'], query) f = http_POST(u, data=constraintsfile) - root = ET.fromstring(b''.join(f)) + root = ET.fromstring(''.join(f)) return [node.get('name') for node in root.findall('entry')] @@ -6320,7 +6267,7 @@ def get_commitlog(apiurl, prj, package, revision, format = 'text', meta = False, try: comment = node.find('comment').text.encode(locale.getpreferredencoding(), 'replace') except: - comment = b'<no message>' + comment = '<no message>' try: requestid = node.find('requestid').text.encode(locale.getpreferredencoding(), 'replace') except: @@ -6330,7 +6277,7 @@ def get_commitlog(apiurl, prj, package, revision, format = 'text', meta = False, if format == 'csv': s = '%s|%s|%s|%s|%s|%s|%s' % (rev, user, t, srcmd5, version, - decode_it(comment).replace('\\', '\\\\').replace('\n', '\\n').replace('|', '\\|'), requestid) + comment.replace('\\', '\\\\').replace('\n', '\\n').replace('|', '\\|'), requestid) r.append(s) elif format == 'xml': r.append('<logentry') @@ -6339,14 +6286,14 @@ def get_commitlog(apiurl, prj, package, revision, format = 'text', meta = False, r.append('<date>%s</date>' % t) r.append('<requestid>%s</requestid>' % requestid) r.append('<msg>%s</msg>' % - decode_it(comment).replace('&', '&').replace('<', '>').replace('>', '<')) + comment.replace('&', '&').replace('<', '>').replace('>', '<')) r.append('</logentry>') else: if requestid: - requestid = decode_it((b"rq" + requestid)) + requestid = "rq" + requestid s = '-' * 76 + \ '\nr%s | %s | %s | %s | %s | %s\n' % (rev, user, t, srcmd5, version, requestid) + \ - '\n' + decode_it(comment) + '\n' + comment r.append(s) if format not in ['csv', 'xml']: @@ -6480,8 +6427,6 @@ def store_write_string(dir, file, string, subdir=''): fname = os.path.join(dir, store, subdir, file) try: f = open(fname + '.new', 'w') - if not isinstance(string, str): - string = decode_it(string) f.write(string) f.close() os.rename(fname + '.new', fname) @@ -6860,11 +6805,7 @@ def is_rpm(f): except: return False - if isinstance(h, str): - isrpmstr = '\xed\xab\xee\xdb' - else: - isrpmstr = b'\xed\xab\xee\xdb' - if h == isrpmstr: + if h == '\xed\xab\xee\xdb': return True else: return False @@ -6880,8 +6821,7 @@ def is_srcrpm(f): except: return False - issrcrpm = bytes(bytearray([h[7]])).decode('utf-8') - if issrcrpm == '\x01': + if h[7] == '\x01': return True else: return False @@ -6903,7 +6843,7 @@ def addPerson(apiurl, prj, pac, user, role="maintainer"): create_new=False) if data and get_user_meta(apiurl, user) != None: - root = ET.fromstring(b''.join(data)) + root = ET.fromstring(''.join(data)) found = False for person in root.getiterator('person'): if person.get('userid') == user and person.get('role') == role: @@ -7060,7 +7000,7 @@ def addGitSource(url): # for pretty output xmlindent(s) - f = open(service_file, 'w') + f = open(service_file, 'wb') f.write(ET.tostring(s, encoding=ET_ENCODING)) f.close() if addfile: @@ -7081,7 +7021,7 @@ def addDownloadUrlService(url): # for pretty output xmlindent(s) - f = open(service_file, 'w') + f = open(service_file, 'wb') f.write(ET.tostring(s, encoding=ET_ENCODING)) f.close() if addfile: @@ -7103,7 +7043,7 @@ def addDownloadUrlService(url): # for pretty output xmlindent(s) - f = open(service_file, 'w') + f = open(service_file, 'wb') f.write(ET.tostring(s, encoding=ET_ENCODING)) f.close() @@ -7144,11 +7084,10 @@ def addFiles(filenames, prj_obj = None): todo = [os.path.join(p, elm) for p, dirnames, fnames in os.walk(filename, followlinks=False) for elm in dirnames + fnames] - enc_todo = [b'%s' % elem.encode() for elem in todo] with open(archive, 'w') as f: cpio_proc = subprocess.Popen(['cpio', '-o', '-H', 'newc', '-0'], stdin=subprocess.PIPE, stdout=f) - cpio_proc.communicate(b'\0'.join(enc_todo)) + cpio_proc.communicate('\0'.join(todo)) pacs.extend(findpacs([archive])) for pac in pacs: @@ -7265,7 +7204,7 @@ def get_commit_msg(wc_dir, pacs): footer = [] lines = [] for p in pacs: - states = sorted(p.get_status(False, ' ', '?'), key=cmp_to_key(compare)) + states = sorted(p.get_status(False, ' ', '?'), lambda x, y: cmp(x[1], y[1])) changed = [statfrmt(st, os.path.normpath(os.path.join(p.dir, filename))) for st, filename in states] if changed: footer += changed @@ -7344,7 +7283,7 @@ def request_interactive_review(apiurl, request, initial_cmd='', group=None, except (ValueError, IndexError): print('Invalid rpmlintlog index. Please choose between 0 and %i' % (len(lintlogs)-1)) try: - print(decode_it(get_rpmlint_log(apiurl, **lintlogs[lint_n]))) + print(get_rpmlint_log(apiurl, **lintlogs[lint_n])) except HTTPError as e: if e.code == 404: print('No rpmlintlog for %s %s' % (lintlogs[lint_n]['repo'], @@ -7420,12 +7359,12 @@ def request_interactive_review(apiurl, request, initial_cmd='', group=None, tmpfile.close() tmpfile = None if tmpfile is None: - tmpfile = tempfile.NamedTemporaryFile(suffix='.diff', mode='r+') + tmpfile = tempfile.NamedTemporaryFile(suffix='.diff') tmpfile.write(req_summary) tmpfile.write(issues) try: diff = request_diff(apiurl, request.reqid) - tmpfile.write(decode_it(diff)) + tmpfile.write(diff) except HTTPError as e: if e.code != 400: raise diff --git a/osc/fetch.py b/osc/fetch.py index 597f7df..833c8fa 100644 --- a/osc/fetch.py +++ b/osc/fetch.py @@ -23,7 +23,6 @@ from . import oscerr import tempfile import re -from osc.util.helper import decode_it from .meter import create_text_meter class Fetcher: @@ -78,16 +77,16 @@ class Fetcher: raise oscerr.APIError('CPIO archive is incomplete ' '(see .errors file)') if package == '_repository': - n = re.sub(b'\.pkg\.tar\..z$', b'.arch', hdr.filename) - if n.startswith(b'container:'): - n = re.sub(b'\.tar\..z$', b'.tar', hdr.filename) - pac = pkgs[decode_it(n.rsplit(b'.', 1)[0])] + n = re.sub(r'\.pkg\.tar\..z$', '.arch', hdr.filename) + if n.startswith('container:'): + n = re.sub(r'\.tar\..z$', '.tar', hdr.filename) + pac = pkgs[n.rsplit('.', 1)[0]] pac.canonname = hdr.filename else: - pac = pkgs[decode_it(n.rsplit(b'.', 1)[0])] + pac = pkgs[n.rsplit('.', 1)[0]] else: # this is a kiwi product - pac = pkgs[decode_it(hdr.filename)] + pac = pkgs[hdr.filename] # Extract a single file from the cpio archive try: diff --git a/osc/oscssl.py b/osc/oscssl.py index 312c236..186c98d 100644 --- a/osc/oscssl.py +++ b/osc/oscssl.py @@ -5,20 +5,21 @@ from __future__ import print_function +import M2Crypto.httpslib from M2Crypto.SSL.Checker import SSLVerificationError -from M2Crypto import m2, SSL, httpslib +from M2Crypto import m2, SSL import M2Crypto.m2urllib2 import socket import sys import inspect try: - from urllib.parse import urlparse, splithost, splitport, splittype, urldefrag + from urllib.parse import urlparse, splithost, splitport, splittype from urllib.request import addinfourl from http.client import HTTPSConnection except ImportError: #python 2.x - from urlparse import urlparse, urldefrag + from urlparse import urlparse from urllib import addinfourl, splithost, splitport, splittype from httplib import HTTPSConnection @@ -184,28 +185,22 @@ class myHTTPSHandler(M2Crypto.m2urllib2.HTTPSHandler): # "do_open()" and "https_open()" so that we just need to override # the small "https_open()" method...) def https_open(self, req): - # https://docs.python.org/3.3/library/urllib.request.html#urllib.request.Request.get_host - try: # up to python-3.2 - host = req.get_host() - except AttributeError: # from python-3.3 - host = req.host + host = req.get_host() if not host: - raise M2Crypto.m2urllib2.URLError('no host given') + raise M2Crypto.m2urllib2.URLError('no host given: ' + req.get_full_url()) # Our change: Check to see if we're using a proxy. # Then create an appropriate ssl-aware connection. full_url = req.get_full_url() target_host = urlparse(full_url)[1] - if target_host != host: - request_uri = urldefrag(full_url)[0] - h = httpslib.ProxyHTTPSConnection(host=host, ssl_context=self.ctx) + if (target_host != host): + h = myProxyHTTPSConnection(host = host, appname = self.appname, ssl_context = self.ctx) + # M2Crypto.ProxyHTTPSConnection.putrequest expects a fullurl + selector = full_url else: - try: # up to python-3.2 - request_uri = req.get_selector() - except AttributeError: # from python-3.3 - request_uri = req.selector - h = httpslib.HTTPSConnection(host=host, ssl_context=self.ctx) + h = myHTTPSConnection(host = host, appname = self.appname, ssl_context = self.ctx) + selector = req.get_selector() # End our change h.set_debuglevel(self._debuglevel) @@ -219,9 +214,10 @@ class myHTTPSHandler(M2Crypto.m2urllib2.HTTPSHandler): # request. headers["Connection"] = "close" try: - h.request(req.get_method(), request_uri, req.data, headers) + h.request(req.get_method(), selector, req.data, headers) r = h.getresponse() - except socket.error as err: # XXX what error? + except socket.error as err: # XXX what error? + err.filename = full_url raise M2Crypto.m2urllib2.URLError(err) # Pick apart the HTTPResponse object to get the addinfourl @@ -231,26 +227,18 @@ class myHTTPSHandler(M2Crypto.m2urllib2.HTTPSHandler): # for Windows. That adapter calls recv(), so delegate recv() # to read(). This weird wrapping allows the returned object to # have readline() and readlines() methods. + + # XXX It might be better to extract the read buffering code + # out of socket._fileobject() and into a base class. + r.recv = r.read - if (sys.version_info < (3, 0)): - fp = socket._fileobject(r, close=True) - else: - r._decref_socketios = lambda: None - r.ssl = h.sock.ssl - r._timeout = -1.0 - # hack to bypass python3 bug with 0 buffer size and - # http/client.py readinto method for response class - if r.length is not None and r.length == 0: - r.readinto = lambda b: 0 - r.recv_into = r.readinto - fp = socket.SocketIO(r, 'rb') + fp = socket._fileobject(r) resp = addinfourl(fp, r.msg, req.get_full_url()) resp.code = r.status resp.msg = r.reason return resp - class myHTTPSConnection(M2Crypto.httpslib.HTTPSConnection): def __init__(self, *args, **kwargs): self.appname = kwargs.pop('appname', 'generic') diff --git a/osc/util/debquery.py b/osc/util/debquery.py index 20ee096..3d56880 100644 --- a/osc/util/debquery.py +++ b/osc/util/debquery.py @@ -229,9 +229,9 @@ class DebQuery(packagequery.PackageQuery, packagequery.PackageQueryResult): @staticmethod def filename(name, epoch, version, release, arch): if release: - return '%s_%s-%s_%s.deb' % (name, version, release, arch) + return b'%s_%s-%s_%s.deb' % (name, version, release, arch) else: - return '%s_%s_%s.deb' % (name, version, arch) + return b'%s_%s_%s.deb' % (name, version, arch) if __name__ == '__main__': import sys diff --git a/osc/util/helper.py b/osc/util/helper.py deleted file mode 100644 index 05808d4..0000000 --- a/osc/util/helper.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright (C) 2018 SUSE Linux. All rights reserved. -# This program is free software; it may be used, copied, modified -# and distributed under the terms of the GNU General Public Licence, -# either version 2, or (at your option) any later version. - - -def cmp_to_key(mycmp): - """ Converts a cmp= function into a key= function. - """ - - class K(object): - def __init__(self, obj, *args): - self.obj = obj - - def __lt__(self, other): - return mycmp(self.obj, other.obj) < 0 - - def __gt__(self, other): - return mycmp(self.obj, other.obj) > 0 - - def __eq__(self, other): - return mycmp(self.obj, other.obj) == 0 - - def __le__(self, other): - return mycmp(self.obj, other.obj) <= 0 - - def __ge__(self, other): - return mycmp(self.obj, other.obj) >= 0 - - def __ne__(self, other): - return mycmp(self.obj, other.obj) != 0 - - def __hash__(self): - raise TypeError('hash not implemented') - - return K - - -def decode_list(ilist): - """ Decodes the elements of a list if needed - """ - - dlist = [] - for elem in ilist: - if not isinstance(elem, str): - dlist.append(decode_it(elem)) - else: - dlist.append(elem) - return dlist - - -def decode_it(obj): - """ Decodes the given object if obj is not a string - based on the chardet module if possible - """ - - if isinstance(obj, str): - return obj - else: - try: - import chardet - return obj.decode(chardet.detect(obj)['encoding']) - except: - try: - import locale - return obj.decode(locale.getlocale()[1]) - except: - return obj.decode('latin-1') - diff --git a/osc/util/packagequery.py b/osc/util/packagequery.py index ac2c74c..e430744 100644 --- a/osc/util/packagequery.py +++ b/osc/util/packagequery.py @@ -1,6 +1,5 @@ from __future__ import print_function -from osc.util.helper import decode_it class PackageError(Exception): """base class for all package related errors""" @@ -31,11 +30,11 @@ class PackageQueries(dict): self.__setitem__(query.name(), query) def __setitem__(self, name, query): - if decode_it(name) != decode_it(query.name()): + if name != query.name(): raise ValueError("key '%s' does not match " "package query name '%s'" % (name, query.name())) - architecture = decode_it(query.arch()) + architecture = query.arch() if (architecture in [self.wanted_architecture, 'noarch', 'all', 'any'] or self.wanted_architecture in self.architectureMap.get(architecture, @@ -150,10 +149,10 @@ class PackageQueryResult: raise NotImplementedError def evr(self): - evr = decode_it(self.version()) + evr = self.version() if self.release(): - evr += "-" + decode_it(self.release()) + evr += "-" + self.release() epoch = self.epoch() if epoch is not None and epoch != 0: diff --git a/osc/util/repodata.py b/osc/util/repodata.py index 9203170..cb09113 100644 --- a/osc/util/repodata.py +++ b/osc/util/repodata.py @@ -177,13 +177,13 @@ class RepoDataQueryResult(osc.util.packagequery.PackageQueryResult): return None def vercmp(self, other): - res = osc.util.rpmquery.RpmQuery.rpmvercmp(str(self.epoch()).encode(), str(other.epoch()).encode()) + res = osc.util.rpmquery.RpmQuery.rpmvercmp(str(self.epoch()), str(other.epoch())) if res != 0: return res - res = osc.util.rpmquery.RpmQuery.rpmvercmp(self.version().encode(), other.version().encode()) + res = osc.util.rpmquery.RpmQuery.rpmvercmp(self.version(), other.version()) if res != 0: return res - res = osc.util.rpmquery.RpmQuery.rpmvercmp(self.release().encode(), other.release().encode()) + res = osc.util.rpmquery.RpmQuery.rpmvercmp(self.release(), other.release()) return res def version(self): diff --git a/osc/util/rpmquery.py b/osc/util/rpmquery.py index 535cd37..534636a 100644 --- a/osc/util/rpmquery.py +++ b/osc/util/rpmquery.py @@ -5,10 +5,6 @@ import os import re import struct from . import packagequery -from osc.util.helper import decode_it - -def cmp(a, b): - return (a > b) - (a < b) class RpmError(packagequery.PackageError): pass @@ -188,14 +184,14 @@ class RpmQuery(packagequery.PackageQuery, packagequery.PackageQueryResult): continue # RPMSENSE_SENSEMASK = 15 (see rpmlib.h) but ignore RPMSENSE_SERIAL (= 1 << 0) therefore use 14 if flags & 14: - name += b' ' + name += ' ' if flags & self.GREATER: - name += b'>' + name += '>' elif flags & self.LESS: - name += b'<' + name += '<' if flags & self.EQUAL: - name += b'=' - name += b' %s' % ver + name += '=' + name += ' %s' % ver res.append(name) return res @@ -292,7 +288,7 @@ class RpmQuery(packagequery.PackageQuery, packagequery.PackageQueryResult): arch = 'src' else: arch = self.arch() - return RpmQuery.filename(decode_it(self.name()), None, decode_it(self.version()), decode_it(self.release()), decode_it(arch)) + return RpmQuery.filename(self.name(), None, self.version(), self.release(), arch) @staticmethod def query(filename): @@ -322,8 +318,6 @@ class RpmQuery(packagequery.PackageQuery, packagequery.PackageQueryResult): if ver1 == ver2: return 0 res = 0 - ver1 = decode_it(ver1) - ver2 = decode_it(ver2) while res == 0: # remove all leading non alphanumeric or tilde chars ver1 = re.sub('^[^a-zA-Z0-9~]*', '', ver1) @@ -2,13 +2,11 @@ from distutils.core import setup import distutils.core -from distutils.command import build, install_data +import distutils.command.build +import distutils.command.install_data import os.path import osc.core import sys - -import setuptools - from osc import commandline from osc import babysitter # optional support for py2exe @@ -19,7 +17,7 @@ except: HAVE_PY2EXE = False -class build_osc(build.build, object): +class build_osc(distutils.command.build.build, object): """ Custom build command which generates man page. """ @@ -30,7 +28,7 @@ class build_osc(build.build, object): import gzip man_path = os.path.join(self.build_base, 'osc.1.gz') distutils.log.info('generating %s' % man_path) - outfile = gzip.open(man_path, 'wt') + outfile = gzip.open(man_path, 'w') osccli = commandline.Osc(stdout=outfile) # FIXME: we cannot call the main method because osc expects an ~/.oscrc # file (this would break builds in environments like the obs) @@ -62,15 +60,15 @@ class build_docs(distutils.core.Command): src_dir = (self.distribution.package_dir or {'': ''})[''] src_dir = os.path.join(os.getcwd(), src_dir) import sphinx - sphinx.main(['runme', - '-D', 'version=%s' % metadata.get_version(), + sphinx.main(['runme', + '-D', 'version=%s' % metadata.get_version(), os.path.join('docs',), os.path.join(self.built_docs, 'docs')]) # take a potential build-base option into account (for instance, if osc is # build and installed like this: # python setup.py build --build-base=<dir> ... install ...) -class install_data(install_data.install_data, object): +class install_data(distutils.command.install_data.install_data, object): def initialize_options(self): super(install_data, self).initialize_options() self.built_data = None @@ -99,7 +97,7 @@ data_files = [] if sys.platform[:3] != 'win': data_files.append((os.path.join('share', 'man', 'man1'), ['osc.1.gz'])) -setuptools.setup(name='osc', +setup(name='osc', version = osc.core.__version__, description = 'openSUSE commander', long_description = 'Command-line client for the openSUSE Build Service, which allows to access repositories in the openSUSE Build Service in similar way as Subversion repositories.', diff --git a/tests/suite.py b/tests/suite.py index d8fa512..6d94d44 100644 --- a/tests/suite.py +++ b/tests/suite.py @@ -24,7 +24,6 @@ import test_setlinkrev import test_prdiff import test_conf import test_results -import test_helpers suite = unittest.TestSuite() suite.addTests(test_addfiles.suite()) @@ -43,7 +42,6 @@ suite.addTests(test_setlinkrev.suite()) suite.addTests(test_prdiff.suite()) suite.addTests(test_conf.suite()) suite.addTests(test_results.suite()) -suite.addTests(test_helpers.suite()) if have_xmlrunner: result = xmlrunner.XMLTestRunner(output=os.path.join(os.getcwd(), 'junit-xml-results')).run(suite) diff --git a/tests/test_helpers.py b/tests/test_helpers.py deleted file mode 100644 index 571b828..0000000 --- a/tests/test_helpers.py +++ /dev/null @@ -1,35 +0,0 @@ -import unittest -from osc.util.helper import decode_it, decode_list - -def suite(): - return unittest.makeSuite(TestResults) - -class TestResults(unittest.TestCase): - def testDecodeList(self): - strlist = ['Test1', 'Test2', 'Test3'] - mixlist = ['Test1', b'Test2', 'Test3'] - byteslist = [b'Test1', b'Test2', b'Test3'] - - out = decode_list(strlist) - self.assertListEqual(out, strlist) - - out = decode_list(mixlist) - self.assertListEqual(out, strlist) - - out = decode_list(byteslist) - self.assertListEqual(out, strlist) - - - def testDecodeIt(self): - bytes_obj = b'Test the decoding' - string_obj = 'Test the decoding' - - out = decode_it(bytes_obj) - self.assertEqual(out, string_obj) - - out = decode_it(string_obj) - self.assertEqual(out, string_obj) - -if __name__ == '__main__': - unittest.main() - |