timplement basics of checking the remote for updates - amprolla - devuan's apt repo merger (HTM) git clone git://parazyd.org/amprolla.git (DIR) Log (DIR) Files (DIR) Refs (DIR) README (DIR) LICENSE --- (DIR) commit 2807e03bd2cd30f25d5cf9dff512729f6df48224 (DIR) parent fe689ce0c36c11c7f5c5e00b645a9ca82f3e4d07 (HTM) Author: parazyd <parazyd@dyne.org> Date: Fri, 26 May 2017 15:44:46 +0200 implement basics of checking the remote for updates Diffstat: M .gitignore | 1 + A amprolla-download | 69 ++++++++++++++++++++++++++++++ M amprolla-merge | 8 +++++--- 3 files changed, 75 insertions(+), 3 deletions(-) --- (DIR) diff --git a/.gitignore b/.gitignore t@@ -1,3 +1,4 @@ *.pyc spool/ +spool.tgz merged/ (DIR) diff --git a/amprolla-download b/amprolla-download t@@ -0,0 +1,69 @@ +#!/usr/bin/env python3 + +import subprocess +from os.path import join +import requests + +from lib.net import download +from lib.parse import parse_release, get_time, get_date + + +roots = { + 'devuan': { + 'local': 'spool/devuan/dists/jessie', + 'remote': 'http://auto.mirror.devuan.org/devuan/dists/jessie', + }, + 'debian': { + 'local': 'spool/debian/dists/jessie', + 'remote': 'http://ftp.debian.org/debian/dists/jessie', + }, + 'debian-sec': { + 'local': 'spool/dists/jessie/updates', + 'remote': 'http://security.debian.org/dists/jessie/updates', + }, +} + +release_file = 'Release' + +def merge_files(repo, relfile): + """ + Loads the local release and call the merge process + """ + print('Loading Release') + rel = join(roots[repo]['local'], relfile) + release_contents = open(rel).read() + + hashes = parse_release(release_contents) + + for k in hashes.keys(): + # if k.endswith('Packages.gz'): + if k.endswith('/binary-armhf/Packages.gz'): + # skip empty files + # TODO: probably best to copy it in place when this occurs + if hashes[k] == 'f61f27bd17de546264aa58f40f3aafaac7021e0ef69c17f6b1b4cd7664a037ec': + print('Skipping %s' % k) + continue + + subprocess.run(['./amprolla-merge', k]) + + +local_rel = join(roots['devuan']['local'], release_file) +remote_rel = join(roots['devuan']['remote'], release_file) + +local_contents = open(local_rel).read() +local_date = get_date(local_contents) + +r = requests.get(remote_rel) +remote_contents = r.text +remote_date = get_date(remote_contents) + +print('Local date: %s' % local_date) +print('Remote date: %s' % remote_date) + +if get_time(remote_date) > get_time(local_date): + # dump new release in place and merge + # NOTE: when testing, watch out because you lose the old Release file in + # spool + print('Remote is newer') + download(remote_rel, local_rel) + merge_files('devuan', local_rel) (DIR) diff --git a/amprolla-merge b/amprolla-merge t@@ -3,6 +3,7 @@ Amprolla main module """ +import sys from os.path import join from time import time t@@ -13,7 +14,7 @@ from lib.config import banpkgs roots = { 'devuan': 'spool/devuan/dists/jessie', 'debian': 'spool/debian/dists/jessie', - 'debian-sec': 'spool/dists/jessie/updates/', + 'debian-sec': 'spool/dists/jessie/updates', } #devuan_release_contents = open(join(roots['devuan'], 'Release')).read() t@@ -23,10 +24,11 @@ roots = { #devuan_files = list(filter(lambda x: x.endswith('Packages.gz') and 'armhf' in x, devuan_release.keys())) #debian_files = list(filter(lambda x: x.endswith('Packages.gz') and 'armhf' in x, debian_release.keys())) -packages_file = 'main/binary-armhf/Packages.gz' +#packages_file = 'main/binary-armhf/Packages.gz' +packages_file = sys.argv[1] t1 = time() -print('Loading packages') +print('Loading packages: %s' % packages_file) devuan = load_packages_file(join(roots['devuan'], packages_file)) debian = load_packages_file(join(roots['debian'], packages_file))