tmerge sources files as well - amprolla - devuan's apt repo merger
(HTM) git clone git://parazyd.org/amprolla.git
(DIR) Log
(DIR) Files
(DIR) Refs
(DIR) README
(DIR) LICENSE
---
(DIR) commit fcae27d990e48253848006c52f79247a30ccd9f7
(DIR) parent 4eb2e225dc9cfdb272b1cbda5e5f7d4f5afec01c
(HTM) Author: parazyd <parazyd@dyne.org>
Date: Mon, 5 Jun 2017 14:38:49 +0200
merge sources files as well
Diffstat:
M amprolla_merge.py | 22 ++++++++++++++++------
M lib/config.py | 45 +++++++++++++++++++++++++++++++
M lib/package.py | 32 +++++++++++++++++++------------
M orchestrate.py | 9 ++++++---
4 files changed, 87 insertions(+), 21 deletions(-)
---
(DIR) diff --git a/amprolla_merge.py b/amprolla_merge.py
t@@ -6,7 +6,7 @@ Amprolla main module
"""
from sys import argv
-from os.path import join
+from os.path import basename, join
from time import time
from lib.package import (write_packages, load_packages_file,
t@@ -62,7 +62,7 @@ def devuan_rewrite(pkg, repo_name):
def merge(packages_list):
"""
- Merges the Packages files given in the package list
+ Merges the Packages/Sources files given in the package list
"""
t1 = time()
t@@ -81,16 +81,26 @@ def merge(packages_list):
if debian:
all_repos.append({'name': 'debian', 'packages': debian})
- print('Merging packages')
- new_pkgs = merge_packages_many(all_repos, banned_packages=banpkgs,
- rewriter=devuan_rewrite)
+
+ if basename(packages_list[0]) == 'Packages.gz':
+ print('Merging packages')
+ src = False
+ new_pkgs = merge_packages_many(all_repos, banned_packages=banpkgs,
+ rewriter=devuan_rewrite)
+ elif basename(packages_list[0]) == 'Sources.gz':
+ print('Merging sources')
+ src = True
+ new_pkgs = merge_packages_many(all_repos)
print('Writing packages')
# replace the devuan subdir with our mergedir that we plan to fill
new_out = packages_list[0].replace(join(spooldir,
repos['devuan']['dists']),
join(mergedir, mergesubdir))
- write_packages(new_pkgs, new_out)
+ if src:
+ write_packages(new_pkgs, new_out, sources=True)
+ else:
+ write_packages(new_pkgs, new_out)
t2 = time()
print('time:', t2-t1)
(DIR) diff --git a/lib/config.py b/lib/config.py
t@@ -83,6 +83,7 @@ aliases = {
categories = ['main', 'contrib', 'non-free']
arches = [
+ 'source',
# 'binary-all',
# 'binary-alpha',
# 'binary-amd64',
t@@ -170,3 +171,47 @@ packages_keys = [
'SHA1',
'SHA256'
]
+
+sources_keys = [
+ 'Package',
+ 'Binary',
+ 'Version',
+ 'Maintainer',
+ 'Uploaders',
+ 'Build-Depends',
+ 'Architecture',
+ 'Standards-Version',
+ 'Format',
+ 'Files',
+ 'Vcs-Browser',
+ 'Vcs-Svn',
+ 'Checksums-Sha1',
+ 'Checksums-Sha256',
+ 'Homepage',
+ 'Package-List',
+ 'Directory',
+ 'Priority',
+ 'Section',
+ 'Vcs-Git',
+ 'Dm-Upload-Allowed',
+ 'Build-Conflicts',
+ 'Testsuite',
+ 'Build-Depends-Indep',
+ 'Vcs-Bzr',
+ 'Vcs-Mtn',
+ 'Vcs-Hg',
+ 'Ruby-Versions',
+ 'Dgit',
+ 'Vcs-Darcs',
+ 'Extra-Source-Only',
+ 'Python-Version',
+ 'Testsuite-Triggers',
+ 'Autobuild',
+ 'Build-Conflicts-Indep',
+ 'Vcs-Cvs',
+ 'Comment',
+ 'Origin',
+ 'Vcs-Arch',
+ 'Original-Maintainer',
+ 'Python3-Version'
+]
(DIR) diff --git a/lib/package.py b/lib/package.py
t@@ -11,10 +11,10 @@ from lzma import open as lzma_open
from shutil import copyfile
from lib.parse import (parse_packages, parse_dependencies)
-from lib.config import packages_keys, mergedir, spooldir
+from lib.config import packages_keys, sources_keys, mergedir, spooldir
-def write_packages(packages, filename, sort=True):
+def write_packages(packages, filename, sort=True, sources=False):
"""
Writes `packages` to a file (per debian Packages format)
If sort=True, the packages are sorted by name.
t@@ -22,32 +22,40 @@ def write_packages(packages, filename, sort=True):
os.makedirs(os.path.dirname(filename), exist_ok=True)
# Copy the arch-specific Release file from devuan if it's not there
- rl = filename.replace('Packages.gz', 'Release')
+ bsnm = 'Packages.gz'
+ if sources:
+ bsnm = 'Sources.gz'
+ rl = filename.replace(bsnm, 'Release')
if not os.path.isfile(rl):
copyfile(rl.replace(mergedir, join(spooldir, 'devuan')), rl)
gzf = gzip_open(filename, 'w')
- xzf = lzma_open(filename.replace('.gz', '.xz'), 'w')
- f = open(filename.replace('.gz', ''), 'w')
+ #xzf = lzma_open(filename.replace('.gz', '.xz'), 'w')
+ #f = open(filename.replace('.gz', ''), 'w')
pkg_items = packages.items()
if sort:
pkg_items = sorted(pkg_items, key=lambda x: x[0])
+ if sources:
+ keylist = sources_keys
+ else:
+ keylist = packages_keys
+
for pkg_name, pkg_contents in pkg_items:
- for key in packages_keys:
+ for key in keylist:
if key in pkg_contents:
s = '%s: %s\n' % (key, pkg_contents[key])
gzf.write(s.encode('utf-8'))
- xzf.write(s.encode('utf-8'))
- f.write(s)
+ #xzf.write(s.encode('utf-8'))
+ #f.write(s)
gzf.write(b'\n')
- xzf.write(b'\n')
- f.write('\n')
+ #xzf.write(b'\n')
+ #f.write('\n')
gzf.close()
- xzf.close()
- f.close()
+ #xzf.close()
+ #f.close()
def load_packages_file(filename):
(DIR) diff --git a/orchestrate.py b/orchestrate.py
t@@ -9,11 +9,14 @@ from multiprocessing import Pool
pkg = []
for i in arches:
for j in categories:
- pkg.append(join(j, i, 'Packages.gz'))
- pkg.append(join(j, 'debian-installer', i, 'Packages.gz'))
+ if i == 'source':
+ mrgfile = 'Sources.gz'
+ else:
+ mrgfile = 'Packages.gz'
+ pkg.append(join(j, 'debian-installer', i, mrgfile))
+ pkg.append(join(j, i, mrgfile))
-#print(pkg)
am = __import__('amprolla_merge')
p = Pool(4)