tamprolla_init.py - amprolla - devuan's apt repo merger
(HTM) git clone https://git.parazyd.org/amprolla
(DIR) Log
(DIR) Files
(DIR) Refs
(DIR) README
(DIR) LICENSE
---
tamprolla_init.py (3140B)
---
1 #!/usr/bin/env python3
2 # See LICENSE file for copyright and license details.
3
4 """
5 This module will download the initial Release files used to populate
6 the spooldir, along with all the files hashed inside the Release files
7 """
8
9 from os.path import join
10 from multiprocessing import Pool
11 from time import time
12
13 from lib.config import (aliases, arches, categories, cpunm, mainrepofiles,
14 repos, spooldir, suites, skips)
15 from lib.lock import check_lock, free_lock
16 from lib.net import download
17 from lib.parse import parse_release
18 from lib.log import die, info
19
20
21 def pop_dirs(repo):
22 """
23 Crawls through the directories to come up with complete needed
24 directory structure.
25 Returns a list of tuples holding the remote and local locations
26 of the files
27
28 Example:
29 (http://deb.debian.org/debian/dists/jessie/main/binary-all/Packages.gz,
30 ./spool/debian/dists/jessie/main/binary-all/Packages.gz)
31 """
32 repodata = repos[repo]
33
34 urls = []
35
36 for i in suites:
37 for j in suites[i]:
38 baseurl = join(repodata['host'], repodata['dists'])
39 suite = j
40 if repodata['aliases'] is True:
41 if j in aliases[repodata['name']]:
42 suite = aliases[repodata['name']][j]
43 elif repodata['skipmissing'] is True:
44 continue
45 if repo == 'debian' and j in skips:
46 continue
47 pair = (join(baseurl, suite),
48 join(baseurl.replace(repodata['host'],
49 spooldir), suite))
50 urls.append(pair)
51
52 return urls
53
54
55 def main():
56 """
57 Loops through all repositories, and downloads their Release files, along
58 with all the files listed within those Release files.
59 """
60 for dist in repos:
61 print('Downloading %s directory structure' % dist)
62 dlurls = pop_dirs(dist)
63 for url in dlurls:
64 tpl = []
65 for file in mainrepofiles:
66 urls = (join(url[0], file), join(url[1], file))
67 tpl.append(urls)
68 dlpool = Pool(cpunm)
69 dlpool.map(download, tpl)
70 dlpool.close()
71
72 release_contents = open(join(url[1], 'Release')).read()
73 release_contents = parse_release(release_contents)
74 tpl = []
75 for k in release_contents:
76 # if k.endswith('/binary-armhf/Packages.gz'):
77 # for a in arches:
78 # for c in categories:
79 # if a in k and ("/%s/" % c) in k:
80 # urls = (join(url[0], k), join(url[1], k))
81 # tpl.append(urls)
82 urls = (join(url[0], k), join(url[1], k))
83 tpl.append(urls)
84 dlpool = Pool(cpunm)
85 dlpool.map(download, tpl)
86 dlpool.close()
87
88
89 if __name__ == '__main__':
90 try:
91 t1 = time()
92 check_lock()
93 main()
94 free_lock()
95 t2 = time()
96 info('Total init time: %s' % (t2 - t1), tofile=True)
97 except Exception as e:
98 die(e)