timplement generation of Release files - amprolla - devuan's apt repo merger
(HTM) git clone git://parazyd.org/amprolla.git
(DIR) Log
(DIR) Files
(DIR) Refs
(DIR) README
(DIR) LICENSE
---
(DIR) commit 94adcf6b6e35c9bbe6dbae6f3db6c15472ba3d65
(DIR) parent fcae27d990e48253848006c52f79247a30ccd9f7
(HTM) Author: parazyd <parazyd@dyne.org>
Date: Mon, 5 Jun 2017 20:05:27 +0200
implement generation of Release files
Diffstat:
M amprolla_merge.py | 7 +++----
M lib/config.py | 28 +++++++++++++++++++++++++++-
M lib/parse.py | 23 +++++++++++++++++++++++
A lib/release.py | 49 +++++++++++++++++++++++++++++++
M orchestrate.py | 75 +++++++++++++++++++++++++------
5 files changed, 164 insertions(+), 18 deletions(-)
---
(DIR) diff --git a/amprolla_merge.py b/amprolla_merge.py
t@@ -113,7 +113,7 @@ def main(packages_file):
# print(packages_file)
to_merge = prepare_merge_dict()
- tt1 = time()
+ #tt1 = time()
for suite in to_merge:
pkg_list = []
for rep in to_merge[suite]:
t@@ -122,11 +122,10 @@ def main(packages_file):
else:
pkg_list.append(None)
- # print(pkg_list)
merge(pkg_list)
- tt2 = time()
- print('total time:', tt2-tt1)
+ #tt2 = time()
+ #print('total time:', tt2-tt1)
if __name__ == '__main__':
(DIR) diff --git a/lib/config.py b/lib/config.py
t@@ -16,7 +16,6 @@ checksums = [
{'name': 'SHA1', 'f': sha1},
{'name': 'SHA256', 'f': sha256},
]
-# checksums = [ 'md5sum', 'sha1', 'sha256', 'sha512' ]
repo_order = ['devuan', 'debian-sec', 'debian']
t@@ -111,6 +110,33 @@ mainrepofiles = [
'Release.gpg'
]
+pkgfiles = [
+ # 'Packages',
+ 'Packages.gz',
+ # 'Packages.xz',
+ 'Release'
+]
+
+srcfiles = [
+ # 'Sources',
+ 'Sources.gz',
+ # 'Sources.xz,
+ 'Release'
+]
+
+release_keys = [
+ 'Origin',
+ 'Label',
+ 'Suite',
+ 'Version',
+ 'Codename',
+ 'Date',
+ 'Valid-Until',
+ 'Architectures',
+ 'Components',
+ 'Description'
+]
+
packages_keys = [
'Package',
'Version',
(DIR) diff --git a/lib/parse.py b/lib/parse.py
t@@ -48,6 +48,29 @@ def parse_release(reltext):
return hashes
+def parse_release_head(reltext):
+ """
+ Parses the header of the release file to grab potentially needed
+ metadata
+ """
+ metadata = {}
+
+ contents = reltext.split('\n')
+
+ md5sum = False
+ for line in contents:
+ if md5sum is True:
+ break
+ elif line.startswith('MD5Sum:'):
+ md5sum = True
+ else:
+ k = line.split(': ')[0]
+ v = line.split(': ')[1]
+ metadata[k] = v
+
+ return metadata
+
+
def parse_release_re(reltext):
"""
(DIR) diff --git a/lib/release.py b/lib/release.py
t@@ -0,0 +1,49 @@
+# see LICENSE file for copyright and license details
+
+"""
+Release file functions and helpers
+"""
+
+from datetime import datetime, timedelta
+from os.path import getsize
+
+from lib.config import release_keys, checksums
+from lib.parse import parse_release_head
+
+
+def write_release(oldrel, newrel, filelist, r):
+ """
+ Generates a valid Release file
+
+ Arguments taken: oldrel, newrel, filelist, r
+ * location of the old Release file (used to take metadata)
+ * location where to write the new Release file
+ * list of files to make checksums
+ * string to remove from the path of the hashed file
+ """
+ t1 = datetime.utcnow()
+ t2 = datetime.utcnow() + timedelta(days=7)
+
+ prettyt1 = t1.strftime('%a, %d %b %Y %H:%M:%S UTC')
+ prettyt2 = t2.strftime('%a, %d %b %Y %H:%M:%S UTC')
+
+ old = open(oldrel).read()
+ new = open(newrel, 'w')
+
+ rel_cont = parse_release_head(old)
+
+ rel_cont['Date'] = prettyt1
+ rel_cont['Valid-Until'] = prettyt2
+
+ for k in release_keys:
+ if k in rel_cont:
+ new.write('%s: %s\n' % (k, rel_cont[k]))
+
+ for csum in checksums:
+ new.write('%s:\n' % csum['name'])
+ for f in filelist:
+ cont = open(f, 'rb').read()
+ new.write(' %s %8s %s\n' % (csum['f'](cont).hexdigest(),
+ getsize(f), f.replace(r+'/', '')))
+
+ new.close()
(DIR) diff --git a/orchestrate.py b/orchestrate.py
t@@ -1,23 +1,72 @@
#!/usr/bin/env python3
# see LICENSE file for copyright and license details
+"""
+Module used to orchestrace the entire amprolla merge
+"""
+
from os.path import join
-from lib.config import arches, categories
from multiprocessing import Pool
+from lib.config import (arches, categories, suites, mergedir, mergesubdir,
+ pkgfiles, srcfiles, spooldir, repos)
+from lib.release import write_release
+
+# from pprint import pprint
+
+
+def do_merge():
+ """
+ Crawls the entire directory structure and orchestrates the merge
+ in a queue using multiprocessing
+ """
+ pkg = []
+ for i in arches:
+ for j in categories:
+ if i == 'source':
+ mrgfile = 'Sources.gz'
+ else:
+ mrgfile = 'Packages.gz'
+ pkg.append(join(j, 'debian-installer', i, mrgfile))
+
+ pkg.append(join(j, i, mrgfile))
+
+ am = __import__('amprolla_merge')
+
+ p = Pool(4)
+ p.map(am.main, pkg)
+
+
+def gen_release(s):
+ """
+ Generates a Release file for a given main suite (jessie/ascii/unstable)
+ """
+
+ for suite in suites[s]:
+ filelist = []
+ rootdir = join(mergedir, mergesubdir, suite)
+ for cat in categories:
+ for arch in arches:
+ if arch == 'source':
+ flist = srcfiles
+ else:
+ flist = pkgfiles
+
+ for fl in flist:
+ filelist.append(join(rootdir, cat, arch, fl))
+ if arch != 'source':
+ filelist.append(join(rootdir, cat,
+ 'debian-installer', arch, fl))
+
+ newrfl = join(rootdir, 'Release')
+ oldrfl = newrfl.replace(join(mergedir, mergesubdir),
+ join(spooldir, repos['devuan']['dists']))
-pkg = []
-for i in arches:
- for j in categories:
- if i == 'source':
- mrgfile = 'Sources.gz'
- else:
- mrgfile = 'Packages.gz'
- pkg.append(join(j, 'debian-installer', i, mrgfile))
+ write_release(oldrfl, newrfl, filelist, rootdir)
+ break
- pkg.append(join(j, i, mrgfile))
-am = __import__('amprolla_merge')
+do_merge()
-p = Pool(4)
-p.map(am.main, pkg)
+for st in suites:
+ gen_release(st)