timplement generation of Release files - amprolla - devuan's apt repo merger (HTM) git clone git://parazyd.org/amprolla.git (DIR) Log (DIR) Files (DIR) Refs (DIR) README (DIR) LICENSE --- (DIR) commit 94adcf6b6e35c9bbe6dbae6f3db6c15472ba3d65 (DIR) parent fcae27d990e48253848006c52f79247a30ccd9f7 (HTM) Author: parazyd <parazyd@dyne.org> Date: Mon, 5 Jun 2017 20:05:27 +0200 implement generation of Release files Diffstat: M amprolla_merge.py | 7 +++---- M lib/config.py | 28 +++++++++++++++++++++++++++- M lib/parse.py | 23 +++++++++++++++++++++++ A lib/release.py | 49 +++++++++++++++++++++++++++++++ M orchestrate.py | 75 +++++++++++++++++++++++++------ 5 files changed, 164 insertions(+), 18 deletions(-) --- (DIR) diff --git a/amprolla_merge.py b/amprolla_merge.py t@@ -113,7 +113,7 @@ def main(packages_file): # print(packages_file) to_merge = prepare_merge_dict() - tt1 = time() + #tt1 = time() for suite in to_merge: pkg_list = [] for rep in to_merge[suite]: t@@ -122,11 +122,10 @@ def main(packages_file): else: pkg_list.append(None) - # print(pkg_list) merge(pkg_list) - tt2 = time() - print('total time:', tt2-tt1) + #tt2 = time() + #print('total time:', tt2-tt1) if __name__ == '__main__': (DIR) diff --git a/lib/config.py b/lib/config.py t@@ -16,7 +16,6 @@ checksums = [ {'name': 'SHA1', 'f': sha1}, {'name': 'SHA256', 'f': sha256}, ] -# checksums = [ 'md5sum', 'sha1', 'sha256', 'sha512' ] repo_order = ['devuan', 'debian-sec', 'debian'] t@@ -111,6 +110,33 @@ mainrepofiles = [ 'Release.gpg' ] +pkgfiles = [ + # 'Packages', + 'Packages.gz', + # 'Packages.xz', + 'Release' +] + +srcfiles = [ + # 'Sources', + 'Sources.gz', + # 'Sources.xz, + 'Release' +] + +release_keys = [ + 'Origin', + 'Label', + 'Suite', + 'Version', + 'Codename', + 'Date', + 'Valid-Until', + 'Architectures', + 'Components', + 'Description' +] + packages_keys = [ 'Package', 'Version', (DIR) diff --git a/lib/parse.py b/lib/parse.py t@@ -48,6 +48,29 @@ def parse_release(reltext): return hashes +def parse_release_head(reltext): + """ + Parses the header of the release file to grab potentially needed + metadata + """ + metadata = {} + + contents = reltext.split('\n') + + md5sum = False + for line in contents: + if md5sum is True: + break + elif line.startswith('MD5Sum:'): + md5sum = True + else: + k = line.split(': ')[0] + v = line.split(': ')[1] + metadata[k] = v + + return metadata + + def parse_release_re(reltext): """ (DIR) diff --git a/lib/release.py b/lib/release.py t@@ -0,0 +1,49 @@ +# see LICENSE file for copyright and license details + +""" +Release file functions and helpers +""" + +from datetime import datetime, timedelta +from os.path import getsize + +from lib.config import release_keys, checksums +from lib.parse import parse_release_head + + +def write_release(oldrel, newrel, filelist, r): + """ + Generates a valid Release file + + Arguments taken: oldrel, newrel, filelist, r + * location of the old Release file (used to take metadata) + * location where to write the new Release file + * list of files to make checksums + * string to remove from the path of the hashed file + """ + t1 = datetime.utcnow() + t2 = datetime.utcnow() + timedelta(days=7) + + prettyt1 = t1.strftime('%a, %d %b %Y %H:%M:%S UTC') + prettyt2 = t2.strftime('%a, %d %b %Y %H:%M:%S UTC') + + old = open(oldrel).read() + new = open(newrel, 'w') + + rel_cont = parse_release_head(old) + + rel_cont['Date'] = prettyt1 + rel_cont['Valid-Until'] = prettyt2 + + for k in release_keys: + if k in rel_cont: + new.write('%s: %s\n' % (k, rel_cont[k])) + + for csum in checksums: + new.write('%s:\n' % csum['name']) + for f in filelist: + cont = open(f, 'rb').read() + new.write(' %s %8s %s\n' % (csum['f'](cont).hexdigest(), + getsize(f), f.replace(r+'/', ''))) + + new.close() (DIR) diff --git a/orchestrate.py b/orchestrate.py t@@ -1,23 +1,72 @@ #!/usr/bin/env python3 # see LICENSE file for copyright and license details +""" +Module used to orchestrace the entire amprolla merge +""" + from os.path import join -from lib.config import arches, categories from multiprocessing import Pool +from lib.config import (arches, categories, suites, mergedir, mergesubdir, + pkgfiles, srcfiles, spooldir, repos) +from lib.release import write_release + +# from pprint import pprint + + +def do_merge(): + """ + Crawls the entire directory structure and orchestrates the merge + in a queue using multiprocessing + """ + pkg = [] + for i in arches: + for j in categories: + if i == 'source': + mrgfile = 'Sources.gz' + else: + mrgfile = 'Packages.gz' + pkg.append(join(j, 'debian-installer', i, mrgfile)) + + pkg.append(join(j, i, mrgfile)) + + am = __import__('amprolla_merge') + + p = Pool(4) + p.map(am.main, pkg) + + +def gen_release(s): + """ + Generates a Release file for a given main suite (jessie/ascii/unstable) + """ + + for suite in suites[s]: + filelist = [] + rootdir = join(mergedir, mergesubdir, suite) + for cat in categories: + for arch in arches: + if arch == 'source': + flist = srcfiles + else: + flist = pkgfiles + + for fl in flist: + filelist.append(join(rootdir, cat, arch, fl)) + if arch != 'source': + filelist.append(join(rootdir, cat, + 'debian-installer', arch, fl)) + + newrfl = join(rootdir, 'Release') + oldrfl = newrfl.replace(join(mergedir, mergesubdir), + join(spooldir, repos['devuan']['dists'])) -pkg = [] -for i in arches: - for j in categories: - if i == 'source': - mrgfile = 'Sources.gz' - else: - mrgfile = 'Packages.gz' - pkg.append(join(j, 'debian-installer', i, mrgfile)) + write_release(oldrfl, newrfl, filelist, rootdir) + break - pkg.append(join(j, i, mrgfile)) -am = __import__('amprolla_merge') +do_merge() -p = Pool(4) -p.map(am.main, pkg) +for st in suites: + gen_release(st)