tamprolla_merge.py - amprolla - devuan's apt repo merger
(HTM) git clone https://git.parazyd.org/amprolla
(DIR) Log
(DIR) Files
(DIR) Refs
(DIR) README
(DIR) LICENSE
---
tamprolla_merge.py (6528B)
---
1 #!/usr/bin/env python3
2 # See LICENSE file for copyright and license details.
3
4 """
5 Amprolla main module
6 """
7
8 from os.path import basename, join
9 from multiprocessing import Pool
10 from time import time
11
12
13 import lib.globalvars as globalvars
14 from lib.config import (aliases, arches, banpkgs, categories, cpunm, mergedir,
15 mergesubdir, pkgfiles, repos, repo_order, spooldir,
16 srcfiles, suites, skips)
17 from lib.lock import check_lock, free_lock
18 from lib.package import (load_packages_file, merge_packages_many,
19 write_packages)
20 from lib.release import write_release
21 from lib.log import info, die
22
23
24 def prepare_merge_dict():
25 """
26 This function will prepare a dict of lists that contain the repos
27 that need to be merged in an ordered fashion. Orders them using the
28 repo_order list found in lib.config
29 Example output:
30 {'ascii': ['ascii', None, 'stretch']},
31 """
32 merge_dict = {}
33
34 for suite in suites:
35 for i in suites[suite]:
36 merge_dict[i] = []
37
38 for suite in merge_dict:
39 for repo in repo_order:
40 tmpsuite = suite
41 if repos[repo]['aliases'] is True:
42 if tmpsuite in aliases[repos[repo]['name']]:
43 tmpsuite = aliases[repos[repo]['name']][suite]
44 elif repos[repo]['skipmissing'] is True:
45 tmpsuite = None
46 if repo == 'debian' and suite in skips:
47 tmpsuite = None
48 if tmpsuite: # make it a proper path
49 tmpsuite = join(spooldir, repos[repo]['dists'], tmpsuite)
50 merge_dict[suite].append(tmpsuite)
51
52 return merge_dict
53
54
55 def devuan_rewrite(pkg, repo_name):
56 """
57 Function to be called when including a certain package. Allows for changing
58 any attributes.
59 """
60
61 if 'Filename' in pkg and repos[repo_name]['name'] not in pkg['Filename']:
62 pkg['Filename'] = pkg['Filename'].replace('pool/', 'pool/%s/' %
63 repos[repo_name]['name'], 1)
64 if 'Directory' in pkg and repos[repo_name]['name'] not in pkg['Directory']:
65 pkg['Directory'] = pkg['Directory'].replace('pool/', 'pool/%s/' %
66 repos[repo_name]['name'], 1)
67
68 return pkg
69
70
71 def merge(packages_list):
72 """
73 Merges the Packages/Sources files given in the package list
74
75 ['path/to/devuan/Packages.gz', None, 'path/to/debian/Packages.gz']
76 """
77 all_repos = []
78 print('Loading packages: %s' % packages_list)
79
80 for i in range(len(repo_order)):
81 pkgs = load_packages_file(packages_list[i])
82 if pkgs:
83 all_repos.append({'name': repo_order[i], 'packages': pkgs})
84
85 for i in range(len(repo_order)):
86 if packages_list[i]:
87 if basename(packages_list[i]) == 'Packages.gz':
88 print('Merging packages')
89 src = False
90 new_pkgs = merge_packages_many(all_repos,
91 banned_packages=banpkgs,
92 rewriter=devuan_rewrite)
93 elif basename(packages_list[i]) == 'Sources.gz':
94 print('Merging sources')
95 src = True
96 new_pkgs = merge_packages_many(all_repos,
97 rewriter=devuan_rewrite)
98 break
99
100 print('Writing packages')
101 for i in range(len(repo_order)):
102 if packages_list[i]:
103 new_out = packages_list[i].replace(join(spooldir,
104 repos[repo_order[i]]['dists']),
105 join(mergedir, mergesubdir))
106 break
107
108 if src:
109 write_packages(new_pkgs, new_out, sources=True)
110 else:
111 write_packages(new_pkgs, new_out)
112
113
114 def gen_release(suite):
115 """
116 Generates a Release file for a given suite (jessie/ascii/unstable)
117 """
118
119 filelist = []
120 print('\nCrawling %s' % suite)
121 rootdir = join(mergedir, mergesubdir, suite)
122 for cat in categories:
123 for arch in arches:
124 if arch == 'source':
125 flist = srcfiles
126 else:
127 flist = pkgfiles
128 cont = arch.replace('binary', 'Contents')
129 cont_udeb = arch.replace('binary', 'Contents-udeb')
130 filelist.append(join(rootdir, cat, cont+'.gz'))
131 filelist.append(join(rootdir, cat, cont))
132 filelist.append(join(rootdir, cat, cont_udeb+'.gz'))
133 filelist.append(join(rootdir, cat, cont_udeb))
134
135 for i in flist:
136 filelist.append(join(rootdir, cat, arch, i))
137 if arch != 'source':
138 filelist.append(join(rootdir, cat,
139 'debian-installer', arch, i))
140
141 newrfl = join(rootdir, 'Release')
142 oldrfl = newrfl.replace(join(mergedir, mergesubdir),
143 join(spooldir, repos['devuan']['dists']))
144
145 print('Writing Release')
146 write_release(oldrfl, newrfl, filelist, rootdir)
147
148
149 def main_merge(packages_file):
150 """
151 Main function that calls the actual merge
152 """
153 to_merge = prepare_merge_dict()
154
155 for suite in to_merge:
156 globalvars.suite = suite
157 pkg_list = []
158 for rep in to_merge[suite]:
159 if rep:
160 pkg_list.append(join(rep, packages_file))
161 else:
162 pkg_list.append(None)
163
164 merge(pkg_list)
165
166
167 def main():
168 """
169 Crawls the entire directory structure and orchestrates the merge
170 in a queue using multiprocessing
171 """
172 pkg = []
173 for i in arches:
174 for j in categories:
175 if i == 'source':
176 mrgfile = 'Sources.gz'
177 else:
178 mrgfile = 'Packages.gz'
179 pkg.append(join(j, 'debian-installer', i, mrgfile))
180
181 pkg.append(join(j, i, mrgfile))
182
183 mrgpool = Pool(cpunm)
184 mrgpool.map(main_merge, pkg)
185 mrgpool.close()
186
187 rel_list = []
188 for i in suites:
189 for j in suites[i]:
190 rel_list.append(j)
191
192 relpool = Pool(cpunm)
193 relpool.map(gen_release, rel_list)
194 relpool.close()
195
196
197 if __name__ == '__main__':
198 try:
199 t1 = time()
200 check_lock()
201 main()
202 free_lock()
203 t2 = time()
204 info('Total full merge time: %s' % (t2 - t1), tofile=True)
205 except Exception as e:
206 die(e)