itpylint and pep8 fixes - amprolla - devuan's apt repo merger Err parazyd.org 70 hgit clone git://parazyd.org/amprolla.git URL:git://parazyd.org/amprolla.git parazyd.org 70 1Log /git/amprolla/log.gph parazyd.org 70 1Files /git/amprolla/files.gph parazyd.org 70 1Refs /git/amprolla/refs.gph parazyd.org 70 1README /git/amprolla/file/README.md.gph parazyd.org 70 1LICENSE /git/amprolla/file/LICENSE.gph parazyd.org 70 i--- Err parazyd.org 70 1commit 5d279f837f4bfd6d393d01cd8b8f03289d1fbf28 /git/amprolla/commit/5d279f837f4bfd6d393d01cd8b8f03289d1fbf28.gph parazyd.org 70 1parent acdff2408cb6a5fc0bdbbdb89b68ee46bc75f55d /git/amprolla/commit/acdff2408cb6a5fc0bdbbdb89b68ee46bc75f55d.gph parazyd.org 70 hAuthor: parazyd URL:mailto:parazyd@dyne.org parazyd.org 70 iDate: Sun, 17 Dec 2017 02:14:17 +0100 Err parazyd.org 70 i Err parazyd.org 70 ipylint and pep8 fixes Err parazyd.org 70 i Err parazyd.org 70 iDiffstat: Err parazyd.org 70 i M amprolla_merge.py | 6 +++--- Err parazyd.org 70 i M amprolla_merge_contents.py | 2 +- Err parazyd.org 70 i M amprolla_update.py | 18 +++++++++--------- Err parazyd.org 70 i M lib/package.py | 16 ++++++++-------- Err parazyd.org 70 i M lib/parse.py | 40 ++++++++++++++++---------------- Err parazyd.org 70 i M lib/release.py | 55 ++++++++++++++++--------------- Err parazyd.org 70 i Err parazyd.org 70 i6 files changed, 70 insertions(+), 67 deletions(-) Err parazyd.org 70 i--- Err parazyd.org 70 1diff --git a/amprolla_merge.py b/amprolla_merge.py /git/amprolla/file/amprolla_merge.py.gph parazyd.org 70 it@@ -12,8 +12,8 @@ from time import time Err parazyd.org 70 i Err parazyd.org 70 i import lib.globalvars as globalvars Err parazyd.org 70 i from lib.config import (aliases, arches, banpkgs, categories, cpunm, mergedir, Err parazyd.org 70 i- mergesubdir, pkgfiles, repos, repo_order, signrelease, Err parazyd.org 70 i- spooldir, srcfiles, suites, skips) Err parazyd.org 70 i+ mergesubdir, pkgfiles, repos, repo_order, spooldir, Err parazyd.org 70 i+ srcfiles, suites, skips) Err parazyd.org 70 i from lib.lock import check_lock, free_lock Err parazyd.org 70 i from lib.package import (load_packages_file, merge_packages_many, Err parazyd.org 70 i write_packages) Err parazyd.org 70 it@@ -143,7 +143,7 @@ def gen_release(suite): Err parazyd.org 70 i join(spooldir, repos['devuan']['dists'])) Err parazyd.org 70 i Err parazyd.org 70 i print('Writing Release') Err parazyd.org 70 i- write_release(oldrfl, newrfl, filelist, rootdir, sign=signrelease) Err parazyd.org 70 i+ write_release(oldrfl, newrfl, filelist, rootdir) Err parazyd.org 70 i Err parazyd.org 70 i Err parazyd.org 70 i def main_merge(packages_file): Err parazyd.org 70 1diff --git a/amprolla_merge_contents.py b/amprolla_merge_contents.py /git/amprolla/file/amprolla_merge_contents.py.gph parazyd.org 70 it@@ -12,11 +12,11 @@ from os.path import dirname, join, isfile Err parazyd.org 70 i from time import time Err parazyd.org 70 i Err parazyd.org 70 i import lib.globalvars as globalvars Err parazyd.org 70 i-from amprolla_merge import prepare_merge_dict Err parazyd.org 70 i from lib.config import (arches, categories, cpunm, mergedir, mergesubdir, Err parazyd.org 70 i repos, spooldir) Err parazyd.org 70 i from lib.lock import check_lock, free_lock Err parazyd.org 70 i from lib.log import die, info Err parazyd.org 70 i+from amprolla_merge import prepare_merge_dict Err parazyd.org 70 i Err parazyd.org 70 i Err parazyd.org 70 i def merge_contents(filelist): Err parazyd.org 70 1diff --git a/amprolla_update.py b/amprolla_update.py /git/amprolla/file/amprolla_update.py.gph parazyd.org 70 it@@ -11,12 +11,12 @@ from time import time Err parazyd.org 70 i import requests Err parazyd.org 70 i Err parazyd.org 70 i import lib.globalvars as globalvars Err parazyd.org 70 i-from amprolla_merge import gen_release, merge, prepare_merge_dict Err parazyd.org 70 i from lib.config import aliases, cpunm, repos, repo_order, spooldir, skips Err parazyd.org 70 i from lib.lock import check_lock, free_lock Err parazyd.org 70 i from lib.log import info, warn, die Err parazyd.org 70 i from lib.parse import compare_dict, get_date, get_time, parse_release Err parazyd.org 70 i from lib.net import download Err parazyd.org 70 i+from amprolla_merge import gen_release, merge, prepare_merge_dict Err parazyd.org 70 i Err parazyd.org 70 i Err parazyd.org 70 i def remote_is_newer(remote, local): Err parazyd.org 70 it@@ -48,27 +48,27 @@ def perform_update(suite, paths): Err parazyd.org 70 i needsmerge = {} Err parazyd.org 70 i needsmerge['downloads'] = [] # all files that have to be downloaded Err parazyd.org 70 i regenrelease = False Err parazyd.org 70 i- c = 0 Err parazyd.org 70 i+ cnt = 0 Err parazyd.org 70 i for i in repo_order: Err parazyd.org 70 i # i = repository name Err parazyd.org 70 i needsmerge[i] = {} Err parazyd.org 70 i needsmerge[i]['mergelist'] = [] Err parazyd.org 70 i Err parazyd.org 70 i- if paths[c]: Err parazyd.org 70 i+ if paths[cnt]: Err parazyd.org 70 i info('Working on %s repo' % i) Err parazyd.org 70 i- remote_path = paths[c].replace(spooldir, repos[i]['host']) Err parazyd.org 70 i+ remote_path = paths[cnt].replace(spooldir, repos[i]['host']) Err parazyd.org 70 i try: Err parazyd.org 70 i remote_rel = requests.get(join(remote_path, 'Release')) Err parazyd.org 70 i except requests.exceptions.ConnectionError as err: Err parazyd.org 70 i warn('Caught exception: "%s". Retrying...' % err) Err parazyd.org 70 i return perform_update(suite, paths) Err parazyd.org 70 i Err parazyd.org 70 i- local_rel_text = open(join(paths[c], 'Release')).read() Err parazyd.org 70 i+ local_rel_text = open(join(paths[cnt], 'Release')).read() Err parazyd.org 70 i Err parazyd.org 70 i diffs = {} Err parazyd.org 70 i if remote_is_newer(remote_rel.text, local_rel_text): Err parazyd.org 70 i download((join(remote_path, 'Release'), Err parazyd.org 70 i- join(paths[c], 'Release'))) Err parazyd.org 70 i+ join(paths[cnt], 'Release'))) Err parazyd.org 70 i regenrelease = True Err parazyd.org 70 i Err parazyd.org 70 i diffs = compare_dict(parse_release(remote_rel.text), Err parazyd.org 70 it@@ -78,12 +78,12 @@ def perform_update(suite, paths): Err parazyd.org 70 i for k in diffs: Err parazyd.org 70 i if k.endswith('Packages.gz') or k.endswith('Sources.gz'): Err parazyd.org 70 i needsmerge[i]['mergelist'].append(k) Err parazyd.org 70 i- rmt = join(paths[c].replace(spooldir, repos[i]['host']), k) Err parazyd.org 70 i- loc = join(paths[c], k) Err parazyd.org 70 i+ rmt = join(paths[cnt].replace(spooldir, repos[i]['host']), k) Err parazyd.org 70 i+ loc = join(paths[cnt], k) Err parazyd.org 70 i dlf = (rmt, loc) Err parazyd.org 70 i needsmerge['downloads'].append(dlf) Err parazyd.org 70 i Err parazyd.org 70 i- c += 1 Err parazyd.org 70 i+ cnt += 1 Err parazyd.org 70 i # break Err parazyd.org 70 i Err parazyd.org 70 i # download what needs to be downloaded Err parazyd.org 70 1diff --git a/lib/package.py b/lib/package.py /git/amprolla/file/lib/package.py.gph parazyd.org 70 it@@ -27,10 +27,10 @@ def write_packages(packages, filename, sort=True, sources=False): Err parazyd.org 70 i bsnm = 'Packages.gz' Err parazyd.org 70 i if sources: Err parazyd.org 70 i bsnm = 'Sources.gz' Err parazyd.org 70 i- rl = filename.replace(bsnm, 'Release') Err parazyd.org 70 i- sprl = rl.replace(mergedir, join(spooldir, 'devuan')) Err parazyd.org 70 i- if not isfile(rl) and isfile(sprl): Err parazyd.org 70 i- copyfile(sprl, rl) Err parazyd.org 70 i+ rel = filename.replace(bsnm, 'Release') Err parazyd.org 70 i+ sprl = rel.replace(mergedir, join(spooldir, 'devuan')) Err parazyd.org 70 i+ if not isfile(rel) and isfile(sprl): Err parazyd.org 70 i+ copyfile(sprl, rel) Err parazyd.org 70 i Err parazyd.org 70 i gzf = gzip_open(filename, 'w') Err parazyd.org 70 i xzf = lzma_open(filename.replace('.gz', '.xz'), 'w') Err parazyd.org 70 it@@ -48,10 +48,10 @@ def write_packages(packages, filename, sort=True, sources=False): Err parazyd.org 70 i for pkg_name, pkg_contents in pkg_items: Err parazyd.org 70 i for key in keylist: Err parazyd.org 70 i if key in pkg_contents: Err parazyd.org 70 i- s = '%s: %s\n' % (key, pkg_contents[key]) Err parazyd.org 70 i- gzf.write(s.encode('utf-8')) Err parazyd.org 70 i- xzf.write(s.encode('utf-8')) Err parazyd.org 70 i- # f.write(s.encode('utf-8')) Err parazyd.org 70 i+ sin = '%s: %s\n' % (key, pkg_contents[key]) Err parazyd.org 70 i+ gzf.write(sin.encode('utf-8')) Err parazyd.org 70 i+ xzf.write(sin.encode('utf-8')) Err parazyd.org 70 i+ # f.write(sin.encode('utf-8')) Err parazyd.org 70 i gzf.write(b'\n') Err parazyd.org 70 i xzf.write(b'\n') Err parazyd.org 70 i # f.write(b'\n') Err parazyd.org 70 1diff --git a/lib/parse.py b/lib/parse.py /git/amprolla/file/lib/parse.py.gph parazyd.org 70 it@@ -69,9 +69,9 @@ def parse_release_head(reltext): Err parazyd.org 70 i elif line.startswith(splitter): Err parazyd.org 70 i md5sum = True Err parazyd.org 70 i else: Err parazyd.org 70 i- k = line.split(': ')[0] Err parazyd.org 70 i- v = line.split(': ')[1] Err parazyd.org 70 i- metadata[k] = v Err parazyd.org 70 i+ key = line.split(': ')[0] Err parazyd.org 70 i+ val = line.split(': ')[1] Err parazyd.org 70 i+ metadata[key] = val Err parazyd.org 70 i Err parazyd.org 70 i return metadata Err parazyd.org 70 i Err parazyd.org 70 it@@ -92,9 +92,9 @@ def parse_package(entry): Err parazyd.org 70 i else: Err parazyd.org 70 i pkgs[key] = value Err parazyd.org 70 i Err parazyd.org 70 i- v = line.split(':', 1) Err parazyd.org 70 i- key = v[0] Err parazyd.org 70 i- value = v[1][1:] Err parazyd.org 70 i+ val = line.split(':', 1) Err parazyd.org 70 i+ key = val[0] Err parazyd.org 70 i+ value = val[1][1:] Err parazyd.org 70 i Err parazyd.org 70 i if key: Err parazyd.org 70 i pkgs[key] = value Err parazyd.org 70 it@@ -133,41 +133,41 @@ def parse_dependencies(dependencies): Err parazyd.org 70 i Err parazyd.org 70 i {'lib6': '(>= 2.4)', 'libdbus-1-3': '(>= 1.0.2)', 'foo': None} Err parazyd.org 70 i """ Err parazyd.org 70 i- r = {} Err parazyd.org 70 i+ ret = {} Err parazyd.org 70 i Err parazyd.org 70 i for pkg_plus_version in dependencies.split(', '): Err parazyd.org 70 i- v = pkg_plus_version.split(' ', 1) Err parazyd.org 70 i- name = v[0] Err parazyd.org 70 i+ ver = pkg_plus_version.split(' ', 1) Err parazyd.org 70 i+ name = ver[0] Err parazyd.org 70 i Err parazyd.org 70 i # If we get passed an empty string, the name is '', and we just Err parazyd.org 70 i # outright stop Err parazyd.org 70 i if not name: Err parazyd.org 70 i return {} Err parazyd.org 70 i Err parazyd.org 70 i- if len(v) == 2: Err parazyd.org 70 i- version = v[1] Err parazyd.org 70 i- r[name] = version Err parazyd.org 70 i+ if len(ver) == 2: Err parazyd.org 70 i+ version = ver[1] Err parazyd.org 70 i+ ret[name] = version Err parazyd.org 70 i else: Err parazyd.org 70 i- r[name] = None Err parazyd.org 70 i+ ret[name] = None Err parazyd.org 70 i Err parazyd.org 70 i- return r Err parazyd.org 70 i+ return ret Err parazyd.org 70 i Err parazyd.org 70 i Err parazyd.org 70 i-def compare_dict(d1, d2): Err parazyd.org 70 i+def compare_dict(dic1, dic2): Err parazyd.org 70 i """ Err parazyd.org 70 i Compares two dicts Err parazyd.org 70 i Takes two dicts and returns a dict of tuples with the differences. Err parazyd.org 70 i Err parazyd.org 70 i Example input: Err parazyd.org 70 i Err parazyd.org 70 i- d1={'foo': 'bar'}, 22={'foo': 'baz'} Err parazyd.org 70 i+ dic1={'foo': 'bar'}, dic2={'foo': 'baz'} Err parazyd.org 70 i Err parazyd.org 70 i Example output: Err parazyd.org 70 i Err parazyd.org 70 i {'foo': ('bar', 'baz')} Err parazyd.org 70 i """ Err parazyd.org 70 i- d1_keys = set(d1.keys()) Err parazyd.org 70 i- d2_keys = set(d2.keys()) Err parazyd.org 70 i+ d1_keys = set(dic1.keys()) Err parazyd.org 70 i+ d2_keys = set(dic2.keys()) Err parazyd.org 70 i intersect_keys = d1_keys.intersection(d2_keys) Err parazyd.org 70 i- modified = {o: (d1[o], d2[o]) for o in intersect_keys if d1[o] != d2[o]} Err parazyd.org 70 i- return modified Err parazyd.org 70 i+ mod = {o: (dic1[o], dic2[o]) for o in intersect_keys if dic1[o] != dic2[o]} Err parazyd.org 70 i+ return mod Err parazyd.org 70 1diff --git a/lib/release.py b/lib/release.py /git/amprolla/file/lib/release.py.gph parazyd.org 70 it@@ -4,7 +4,7 @@ Err parazyd.org 70 i Release file functions and helpers Err parazyd.org 70 i """ Err parazyd.org 70 i Err parazyd.org 70 i-from datetime import datetime, timedelta Err parazyd.org 70 i+from datetime import datetime # , timedelta Err parazyd.org 70 i from gzip import decompress as gzip_decomp Err parazyd.org 70 i from lzma import compress as lzma_comp Err parazyd.org 70 i from os.path import getsize, isfile Err parazyd.org 70 it@@ -12,7 +12,7 @@ from subprocess import Popen Err parazyd.org 70 i Err parazyd.org 70 i import lib.globalvars as globalvars Err parazyd.org 70 i from lib.config import (checksums, distrolabel, gpgdir, release_aliases, Err parazyd.org 70 i- release_keys, signingkey) Err parazyd.org 70 i+ release_keys, signingkey, signrelease) Err parazyd.org 70 i from lib.log import info Err parazyd.org 70 i from lib.parse import parse_release_head, parse_release Err parazyd.org 70 i Err parazyd.org 70 it@@ -31,23 +31,23 @@ def rewrite_release_head(headers): Err parazyd.org 70 i return headers Err parazyd.org 70 i Err parazyd.org 70 i Err parazyd.org 70 i-def write_release(oldrel, newrel, filelist, r, sign=True, rewrite=True): Err parazyd.org 70 i+def write_release(oldrel, newrel, filelist, rmstr, rewrite=True): Err parazyd.org 70 i """ Err parazyd.org 70 i Generates a valid Release file Err parazyd.org 70 i if sign=False: do not use gnupg to sign the file Err parazyd.org 70 i if rewrite=True: rewrite the Release headers as defined in the config Err parazyd.org 70 i Err parazyd.org 70 i- Arguments taken: oldrel, newrel, filelist, r Err parazyd.org 70 i+ Arguments taken: oldrel, newrel, filelist, rmstr Err parazyd.org 70 i * location of the old Release file (used to take metadata) Err parazyd.org 70 i * location where to write the new Release file Err parazyd.org 70 i * list of files to make checksums Err parazyd.org 70 i * string to remove from the path of the hashed file Err parazyd.org 70 i """ Err parazyd.org 70 i- t1 = datetime.utcnow() Err parazyd.org 70 i- # t2 = datetime.utcnow() + timedelta(days=7) Err parazyd.org 70 i+ time1 = datetime.utcnow() Err parazyd.org 70 i+ # time2 = datetime.utcnow() + timedelta(days=7) Err parazyd.org 70 i Err parazyd.org 70 i- prettyt1 = t1.strftime('%a, %d %b %Y %H:%M:%S UTC') Err parazyd.org 70 i- # prettyt2 = t2.strftime('%a, %d %b %Y %H:%M:%S UTC') Err parazyd.org 70 i+ prettyt1 = time1.strftime('%a, %d %b %Y %H:%M:%S UTC') Err parazyd.org 70 i+ # prettyt2 = time2.strftime('%a, %d %b %Y %H:%M:%S UTC') Err parazyd.org 70 i Err parazyd.org 70 i # this holds our local data in case we don't want to rehash files Err parazyd.org 70 i local_rel = open(newrel).read() Err parazyd.org 70 it@@ -69,7 +69,7 @@ def write_release(oldrel, newrel, filelist, r, sign=True, rewrite=True): Err parazyd.org 70 i new.write('%s: %s\n' % (k, rel_cont[k])) Err parazyd.org 70 i Err parazyd.org 70 i if globalvars.rehash: Err parazyd.org 70 i- rehash_release(filelist, new, r) Err parazyd.org 70 i+ rehash_release(filelist, new, rmstr) Err parazyd.org 70 i else: Err parazyd.org 70 i info('Reusing old checksums') Err parazyd.org 70 i for csum in checksums: Err parazyd.org 70 it@@ -79,32 +79,35 @@ def write_release(oldrel, newrel, filelist, r, sign=True, rewrite=True): Err parazyd.org 70 i Err parazyd.org 70 i new.close() Err parazyd.org 70 i Err parazyd.org 70 i- if sign: Err parazyd.org 70 i+ if signrelease: Err parazyd.org 70 i sign_release(newrel) Err parazyd.org 70 i Err parazyd.org 70 i Err parazyd.org 70 i-def rehash_release(_filelist, fd, r): Err parazyd.org 70 i+def rehash_release(_filelist, fdesc, rmstr): Err parazyd.org 70 i """ Err parazyd.org 70 i Calculates checksums of a given filelist and writes them to the given Err parazyd.org 70 i- file descriptor. Takes r as the third argument, which is a string to Err parazyd.org 70 i+ file descriptor. Takes rmstr as the third argument, which is a string to Err parazyd.org 70 i remove from the path of the hashed file when writing it to a file. Err parazyd.org 70 i """ Err parazyd.org 70 i info('Hashing checksums') Err parazyd.org 70 i for csum in checksums: Err parazyd.org 70 i- fd.write('%s:\n' % csum['name']) Err parazyd.org 70 i- for f in _filelist: Err parazyd.org 70 i- if isfile(f): Err parazyd.org 70 i- cont = open(f, 'rb').read() Err parazyd.org 70 i- fd.write(' %s %8s %s\n' % (csum['f'](cont).hexdigest(), Err parazyd.org 70 i- getsize(f), f.replace(r+'/', ''))) Err parazyd.org 70 i- elif f.endswith('.xz') and isfile(f.replace('.xz', '.gz')): Err parazyd.org 70 i- xzstr = lzma_comp(open(f.replace('.xz', '.gz'), 'rb').read()) Err parazyd.org 70 i- fd.write(' %s %8s %s\n' % (csum['f'](xzstr).hexdigest(), Err parazyd.org 70 i- len(xzstr), f.replace(r+'/', ''))) Err parazyd.org 70 i- elif not f.endswith('.gz') and isfile(f+'.gz'): Err parazyd.org 70 i- uncomp = gzip_decomp(open(f+'.gz', 'rb').read()) Err parazyd.org 70 i- fd.write(' %s %8s %s\n' % (csum['f'](uncomp).hexdigest(), Err parazyd.org 70 i- len(uncomp), f.replace(r+'/', ''))) Err parazyd.org 70 i+ fdesc.write('%s:\n' % csum['name']) Err parazyd.org 70 i+ for i in _filelist: Err parazyd.org 70 i+ if isfile(i): Err parazyd.org 70 i+ cont = open(i, 'rb').read() Err parazyd.org 70 i+ fdesc.write(' %s %8s %s\n' % (csum['f'](cont).hexdigest(), Err parazyd.org 70 i+ getsize(i), Err parazyd.org 70 i+ i.replace(rmstr+'/', ''))) Err parazyd.org 70 i+ elif i.endswith('.xz') and isfile(i.replace('.xz', '.gz')): Err parazyd.org 70 i+ xzstr = lzma_comp(open(i.replace('.xz', '.gz'), 'rb').read()) Err parazyd.org 70 i+ fdesc.write(' %s %8s %s\n' % (csum['f'](xzstr).hexdigest(), Err parazyd.org 70 i+ len(xzstr), Err parazyd.org 70 i+ i.replace(rmstr+'/', ''))) Err parazyd.org 70 i+ elif not i.endswith('.gz') and isfile(i+'.gz'): Err parazyd.org 70 i+ uncomp = gzip_decomp(open(i+'.gz', 'rb').read()) Err parazyd.org 70 i+ fdesc.write(' %s %8s %s\n' % (csum['f'](uncomp).hexdigest(), Err parazyd.org 70 i+ len(uncomp), Err parazyd.org 70 i+ i.replace(rmstr+'/', ''))) Err parazyd.org 70 i return Err parazyd.org 70 i Err parazyd.org 70 i Err parazyd.org 70 .