tpackage.py - amprolla - devuan's apt repo merger
(HTM) git clone https://git.parazyd.org/amprolla
(DIR) Log
(DIR) Files
(DIR) Refs
(DIR) README
(DIR) LICENSE
---
tpackage.py (7166B)
---
1 # See LICENSE file for copyright and license details.
2
3 """
4 Package merging functions and helpers
5 """
6
7 from os import makedirs
8 from os.path import dirname, isfile, join
9 from gzip import open as gzip_open
10 from lzma import open as lzma_open
11 from shutil import copyfile
12
13 import lib.globalvars as globalvars
14 from lib.config import mergedir, packages_keys, sources_keys, spooldir
15 from lib.log import logtofile
16 from lib.parse import parse_dependencies, parse_packages, cmppkgver
17
18
19 def write_packages(packages, filename, sort=True, sources=False):
20 """
21 Writes `packages` to a file (per debian Packages format)
22 If sort=True, the packages are sorted by name.
23 """
24 makedirs(dirname(filename), exist_ok=True)
25
26 # Copy the arch-specific Release file from devuan if it's not there
27 bsnm = 'Packages.gz'
28 if sources:
29 bsnm = 'Sources.gz'
30 rel = filename.replace(bsnm, 'Release')
31 sprl = rel.replace(mergedir, join(spooldir, 'devuan'))
32 if not isfile(rel) and isfile(sprl):
33 copyfile(sprl, rel)
34
35 gzf = gzip_open(filename, 'w')
36 xzf = lzma_open(filename.replace('.gz', '.xz'), 'w')
37 # f = open(filename.replace('.gz', ''), 'wb')
38
39 pkg_items = packages.items()
40 if sort:
41 pkg_items = sorted(pkg_items, key=lambda x: x[0])
42
43 if sources:
44 keylist = sources_keys
45 else:
46 keylist = packages_keys
47
48 for pkg_name, pkg_contents in pkg_items:
49 for key in keylist:
50 if key in pkg_contents:
51 sin = '%s: %s\n' % (key, pkg_contents[key])
52 gzf.write(sin.encode('utf-8'))
53 xzf.write(sin.encode('utf-8'))
54 # f.write(sin.encode('utf-8'))
55 gzf.write(b'\n')
56 xzf.write(b'\n')
57 # f.write(b'\n')
58
59 gzf.close()
60 xzf.close()
61 # f.close()
62
63
64 def load_packages_file(filename):
65 """
66 Load a gzip'd packages file.
67 Returns a dictionary of package name and package key-values.
68 """
69 # TODO: should we skip files like this if they don't exist?
70 if filename is not None and isfile(filename):
71 packages_contents = gzip_open(filename).read()
72 packages_contents = packages_contents.decode('utf-8')
73 return parse_packages(packages_contents)
74
75 return None
76
77
78 def depends_on_all_banned(deps, banned_pkgs):
79 """
80 Gets a list of dicts of dep alternatives and a set of banned packages and
81 returns True if any of the dicts consist exclusively of banned_pkgs.
82 """
83 for dep in deps:
84 alt = set(dep.keys())
85 if len(alt.intersection(banned_pkgs)) == len(alt):
86 # All the alternatives are banned
87 return True
88 return False
89
90
91 def depends_on(deps, package_set):
92 """
93 Gets a list of dicts of dep alternatives and a set of packages and returns
94 True if any of the dicts include at least one of the elements in
95 package_set.
96 """
97 for dep in deps:
98 alt = set(dep.keys())
99 if alt.intersection(package_set):
100 # At least one alternative is in package_set
101 return True
102 return False
103
104
105 def package_banned(pkg, banned_pkgs):
106 """
107 Returns True is the package contains a banned dependency.
108 Currently checks and parses both the 'Depends:' and the 'Pre-Depends'
109 fields of the package.
110 """
111 if pkg.get('Package') in banned_pkgs:
112 logtofile('bannedpackages.txt', '%s,%s\n' % (globalvars.suite,
113 pkg.get('Package')))
114 return True
115
116 depends = parse_dependencies(pkg.get('Depends', ''))
117 pre_depends = parse_dependencies(pkg.get('Pre-Depends', ''))
118
119 depends.extend(pre_depends)
120
121 if depends_on(depends, set(['libsystemd0'])):
122 logtofile('libsystemd.txt', '%s,%s\n' % (globalvars.suite,
123 pkg.get('Package')))
124
125 isbanned = depends_on_all_banned(depends, banned_pkgs)
126 if isbanned:
127 logtofile('bannedpackages.txt', '%s,%s\n' % (globalvars.suite,
128 pkg.get('Package')))
129 return isbanned
130
131
132 def package_newer(pkg1, pkg2):
133 """
134 Checks whether the package of a lower priority has a higher version than
135 the package of the higher priority and returns True if so.
136
137 Ref: https://www.debian.org/doc/debian-policy/#version
138 """
139 # Hardcoded list of packages we don't want to check
140 _skips = []
141 if pkg1.get('Package') in _skips:
142 return False
143
144 if cmppkgver(pkg1.get('Version'), pkg2.get('Version')) < 0:
145 return True
146
147 return False
148
149
150 def merge_packages(pkg1, pkg2, name1, name2, banned_packages=set(),
151 rewriter=None):
152 """
153 Merges two previously loaded/parsed (using load_packages_file) packages
154 dictionaries, preferring `pkg1` over `pkg2`, and optionally discarding any
155 banned packages.
156 """
157 new_pkgs = {}
158 package_names = set(pkg1.keys()).union(set(pkg2.keys()))
159 obsoletepkgs = []
160
161 for pkg in package_names:
162 pkg1_pkg = pkg1.get(pkg)
163 pkg2_pkg = pkg2.get(pkg)
164
165 if pkg1_pkg and pkg2_pkg:
166 if rewriter:
167 pkg1_pkg = rewriter(pkg1_pkg, name1)
168 new_pkgs[pkg] = pkg1_pkg
169 if package_newer(pkg1_pkg, pkg2_pkg):
170 obsoletepkgs.append('%s,%s,%s,%s' % (globalvars.suite,
171 pkg1_pkg.get('Package'),
172 pkg1_pkg.get('Version'),
173 pkg2_pkg.get('Version')))
174 elif pkg1_pkg:
175 if not package_banned(pkg1_pkg, banned_packages):
176 if rewriter:
177 pkg1_pkg = rewriter(pkg1_pkg, name1)
178 new_pkgs[pkg] = pkg1_pkg
179 elif pkg2_pkg:
180 if not package_banned(pkg2_pkg, banned_packages):
181 if rewriter:
182 pkg2_pkg = rewriter(pkg2_pkg, name2)
183 new_pkgs[pkg] = pkg2_pkg
184 else:
185 assert False, 'Impossibru'
186
187 if obsoletepkgs:
188 obsoletepkgs = '\n'.join(obsoletepkgs) + '\n'
189 logtofile('%s-oldpackages.txt' % globalvars.suite, obsoletepkgs,
190 redo=False)
191
192 return new_pkgs
193
194
195 def merge_packages_many(packages, banned_packages=set(), rewriter=None):
196 """
197 Merges two (or more) previously loaded/parsed (using load_packages_file)
198 packages dictionaries, priority is defined by the order of the `packages`
199 list, optionally discarding any banned packages.
200 """
201 if not len(packages) > 1:
202 while not len(packages) > 1:
203 packages.append({'name': '', 'packages': {}})
204
205 new_pkgs = {}
206
207 pkg1 = packages[0]
208 pkg2 = packages[1]
209
210 new_pkgs = merge_packages(pkg1['packages'], pkg2['packages'],
211 pkg1['name'], pkg2['name'],
212 banned_packages=banned_packages,
213 rewriter=rewriter)
214
215 for pkg in packages[2:]:
216 new_pkgs = merge_packages(new_pkgs, pkg['packages'], '', pkg['name'],
217 banned_packages=banned_packages)
218
219 return new_pkgs