tfinalize incremental updates - amprolla - devuan's apt repo merger
(HTM) git clone git://parazyd.org/amprolla.git
(DIR) Log
(DIR) Files
(DIR) Refs
(DIR) README
(DIR) LICENSE
---
(DIR) commit f0914de43726ef280ecde1203421c5721dd22507
(DIR) parent e07a9378a40812ccdfcbb742b8eea1a2905717a7
(HTM) Author: parazyd <parazyd@dyne.org>
Date: Wed, 7 Jun 2017 15:10:45 +0200
finalize incremental updates
Diffstat:
M amprolla_merge.py | 2 ++
M amprolla_update.py | 90 ++++++++++++++++++++------------
2 files changed, 59 insertions(+), 33 deletions(-)
---
(DIR) diff --git a/amprolla_merge.py b/amprolla_merge.py
t@@ -66,6 +66,8 @@ def devuan_rewrite(pkg, repo_name):
def merge(packages_list):
"""
Merges the Packages/Sources files given in the package list
+
+ ['path/to/devuan/Packages.gz', None, 'path/to/debian/Packages.gz']
"""
all_repos = []
print('Loading packages: %s' % packages_list)
(DIR) diff --git a/amprolla_update.py b/amprolla_update.py
t@@ -6,19 +6,19 @@ Perform incremental updates
"""
from os.path import join
-import requests
from multiprocessing import Pool
+import requests
-from amprolla_init import pop_dirs
-from amprolla_merge import prepare_merge_dict
-from lib.config import repos, spooldir, repo_order
+from amprolla_merge import prepare_merge_dict, gen_release, merge
+from lib.config import repos, spooldir, repo_order, aliases
from lib.parse import parse_release, get_time, get_date, compare_dict
from lib.net import download
-from pprint import pprint
-
def remote_is_newer(remote, local):
+ """
+ Checks if a remote Release file holds a newer date, and returns True if so
+ """
rem_date = get_date(remote)
loc_date = get_date(local)
t@@ -33,7 +33,9 @@ def remote_is_newer(remote, local):
def perform_update(suite, paths):
- print('==================================================')
+ """
+ Performs an incremental update and merge of a given suite
+ """
print('Checking for updates in %s' % suite)
print(paths)
t@@ -49,33 +51,33 @@ def perform_update(suite, paths):
print('Working on %s repo' % i)
remote_path = paths[c].replace(spooldir, repos[i]['host'])
remote_rel = requests.get(join(remote_path, 'Release'))
- remote_rel_text = remote_rel.text
- tup = (remote_rel, join(paths[c], 'Release'))
- download(tup)
local_rel_text = open(join(paths[c], 'Release')).read()
- if remote_is_newer(remote_rel_text, local_rel_text):
- remote_parsed = parse_release(remote_rel_text)
- local_parsed = parse_release(local_rel_text)
- diffs = compare_dict(remote_parsed, local_parsed)
- if diffs:
- for k in diffs:
- if k.endswith('Packages.gz') or k.endswith('Sources.gz'):
- needsmerge[i]['mergelist'].append(k)
- rmt = join(paths[c].replace(spooldir, repos[i]['host']), k)
- loc = join(paths[c], k)
- dlf = (rmt, loc)
- needsmerge['downloads'].append(dlf)
+ diffs = {}
+ if remote_is_newer(remote_rel.text, local_rel_text):
+ download((join(remote_path, 'Release'),
+ join(paths[c], 'Release')))
+
+ diffs = compare_dict(parse_release(remote_rel.text),
+ parse_release(local_rel_text))
+ if diffs:
+ for k in diffs:
+ if k.endswith('Packages.gz') or k.endswith('Sources.gz'):
+ needsmerge[i]['mergelist'].append(k)
+ rmt = join(paths[c].replace(spooldir, repos[i]['host']), k)
+ loc = join(paths[c], k)
+ dlf = (rmt, loc)
+ needsmerge['downloads'].append(dlf)
c += 1
# break
# download what needs to be downloaded
- print('Downloading updates...')
- dlpool = Pool(4)
- dlpool.map(download, needsmerge['downloads'])
- dlpool.close
+ if needsmerge['downloads']:
+ print('Downloading updates...')
+ dlpool = Pool(4)
+ dlpool.map(download, needsmerge['downloads'])
# create union of our Packages.gz and Sources.gz files we will merge
uni = []
t@@ -83,17 +85,39 @@ def perform_update(suite, paths):
uni.append(needsmerge[i]['mergelist'])
updpkg_list = set().union(*uni)
+ # make a list of package lists to feed into merge()
+ merge_list = []
+ for i in updpkg_list:
+ pkgs = []
+ for j in repo_order:
+ sui = suite
+ # append valid aliases
+ if repos[j]['aliases']:
+ if suite in aliases[repos[j]['name']]:
+ sui = aliases[repos[j]['name']][suite]
+ elif repos[j]['skipmissing']:
+ sui = None
+ skips = ['jessie-security', 'ascii-security'] # hack
+ if j == 'debian' and suite in skips:
+ sui = None
+
+ if sui:
+ pkgs.append(join(spooldir, repos[j]['dists'], sui, i))
+ else:
+ pkgs.append(None)
+
+ merge_list.append(pkgs)
+
# perform the actual merge
- if updpkg_list:
+ if merge_list:
print('Merging files...')
mrgpool = Pool(4)
- mrgpool.map(merge, updpkg_list)
- mrgpool.close()
-
- print('Generating Release...')
- gen_release(suite)
+ mrgpool.map(merge, merge_list)
- print('==================================================')
+ # generate release files
+ if needsmerge['downloads']:
+ print('Generating Release...')
+ gen_release(suite)
def main():