tmerge orchestrate.py with amprolla_merge.py - amprolla - devuan's apt repo merger
(HTM) git clone git://parazyd.org/amprolla.git
(DIR) Log
(DIR) Files
(DIR) Refs
(DIR) README
(DIR) LICENSE
---
(DIR) commit 447efaed180df30074f5722963ebf28e91911efd
(DIR) parent c81a62a15175afaa1e706eec7b97924c0df8fe2c
(HTM) Author: parazyd <parazyd@dyne.org>
Date: Tue, 6 Jun 2017 00:30:06 +0200
merge orchestrate.py with amprolla_merge.py
Diffstat:
M README.md | 11 ++++++-----
M amprolla_merge.py | 75 +++++++++++++++++++++++++------
D orchestrate.py | 72 -------------------------------
3 files changed, 67 insertions(+), 91 deletions(-)
---
(DIR) diff --git a/README.md b/README.md
t@@ -31,11 +31,12 @@ app-crypt/gnupg dev-python/requests dev-python/python-gnupg
Basic usage
-----------
-Edit `lib/config.py` to your needs, and then run `amprolla_init.py`. This
-will download the repositories we will merge afterwards. When this is done,
-you can now run `orchestrate.py` which will perform the merge, and finally
-sign the Release files needed.
+Edit `lib/config.py` to your needs, and then run `amprolla_init.py`.
+This will download the repositories we will merge afterwards. When this
+is done, you can now run `amprolla_merge.py` which will perform the
+merge, and finally sign the Release files needed.
-A `nginx` configuration for the amprolla server can be found in `contrib`.
+A `nginx` configuration for the amprolla server can be found in
+`contrib`.
More information on amprolla should be found in the `doc` directory.
(DIR) diff --git a/amprolla_merge.py b/amprolla_merge.py
t@@ -5,14 +5,16 @@
Amprolla main module
"""
-from sys import argv
from os.path import basename, join
-# from time import time
+from multiprocessing import Pool
+# from pprint import pprint
from lib.package import (write_packages, load_packages_file,
merge_packages_many)
-from lib.config import (aliases, banpkgs, repo_order, repos,
- spooldir, suites, mergedir, mergesubdir)
+from lib.config import (aliases, banpkgs, repo_order, repos, spooldir, suites,
+ mergedir, mergesubdir, pkgfiles, srcfiles, categories,
+ arches)
+from lib.release import write_release
def prepare_merge_dict():
t@@ -64,8 +66,6 @@ def merge(packages_list):
"""
Merges the Packages/Sources files given in the package list
"""
- # t1 = time()
-
all_repos = []
print('Loading packages: %s' % packages_list)
t@@ -101,18 +101,44 @@ def merge(packages_list):
else:
write_packages(new_pkgs, new_out)
- # t2 = time()
- # print('time:', t2-t1)
+def gen_release(s):
+ """
+ Generates a Release file for a given main suite (jessie/ascii/unstable)
+ """
-def main(packages_file):
+ for suite in suites[s]:
+ filelist = []
+ print('Crawling %s' % suite)
+ rootdir = join(mergedir, mergesubdir, suite)
+ for cat in categories:
+ for arch in arches:
+ if arch == 'source':
+ flist = srcfiles
+ else:
+ flist = pkgfiles
+
+ for fl in flist:
+ filelist.append(join(rootdir, cat, arch, fl))
+ if arch != 'source':
+ filelist.append(join(rootdir, cat,
+ 'debian-installer', arch, fl))
+
+ newrfl = join(rootdir, 'Release')
+ oldrfl = newrfl.replace(join(mergedir, mergesubdir),
+ join(spooldir, repos['devuan']['dists']))
+
+ print('Writing Release')
+ write_release(oldrfl, newrfl, filelist, rootdir)
+ # break
+
+
+def main_merge(packages_file):
"""
Main function that calls the actual merge
"""
- # print(packages_file)
to_merge = prepare_merge_dict()
- # tt1 = time()
for suite in to_merge:
pkg_list = []
for rep in to_merge[suite]:
t@@ -123,9 +149,30 @@ def main(packages_file):
merge(pkg_list)
- # tt2 = time()
- # print('total time:', tt2-tt1)
+
+def main():
+ """
+ Crawls the entire directory structure and orchestrates the merge
+ in a queue using multiprocessing
+ """
+ pkg = []
+ for i in arches:
+ for j in categories:
+ if i == 'source':
+ mrgfile = 'Sources.gz'
+ else:
+ mrgfile = 'Packages.gz'
+ pkg.append(join(j, 'debian-installer', i, mrgfile))
+
+ pkg.append(join(j, i, mrgfile))
+
+ # pprint(pkg)
+ p = Pool(4) # Set it to the number of CPUs you want to use
+ p.map(main_merge, pkg)
+
+ for st in suites:
+ gen_release(st)
if __name__ == '__main__':
- main(argv[1])
+ main()
(DIR) diff --git a/orchestrate.py b/orchestrate.py
t@@ -1,72 +0,0 @@
-#!/usr/bin/env python3
-# see LICENSE file for copyright and license details
-
-"""
-Module used to orchestrate the entire amprolla merge
-"""
-
-from os.path import join
-from multiprocessing import Pool
-
-from lib.config import (arches, categories, suites, mergedir, mergesubdir,
- pkgfiles, srcfiles, spooldir, repos)
-from lib.release import write_release
-
-
-def do_merge():
- """
- Crawls the entire directory structure and orchestrates the merge
- in a queue using multiprocessing
- """
- pkg = []
- for i in arches:
- for j in categories:
- if i == 'source':
- mrgfile = 'Sources.gz'
- else:
- mrgfile = 'Packages.gz'
- pkg.append(join(j, 'debian-installer', i, mrgfile))
-
- pkg.append(join(j, i, mrgfile))
-
- am = __import__('amprolla_merge')
-
- p = Pool(4) # Set it to the number of CPUs you want to use
- p.map(am.main, pkg)
-
-
-def gen_release(s):
- """
- Generates a Release file for a given main suite (jessie/ascii/unstable)
- """
-
- for suite in suites[s]:
- filelist = []
- print('Crawling %s' % suite)
- rootdir = join(mergedir, mergesubdir, suite)
- for cat in categories:
- for arch in arches:
- if arch == 'source':
- flist = srcfiles
- else:
- flist = pkgfiles
-
- for fl in flist:
- filelist.append(join(rootdir, cat, arch, fl))
- if arch != 'source':
- filelist.append(join(rootdir, cat,
- 'debian-installer', arch, fl))
-
- newrfl = join(rootdir, 'Release')
- oldrfl = newrfl.replace(join(mergedir, mergesubdir),
- join(spooldir, repos['devuan']['dists']))
-
- print('Writing Release')
- write_release(oldrfl, newrfl, filelist, rootdir)
- # break
-
-
-do_merge()
-
-for st in suites:
- gen_release(st)