timplement basics of checking the remote for updates - amprolla - devuan's apt repo merger
 (HTM) git clone git://parazyd.org/amprolla.git
 (DIR) Log
 (DIR) Files
 (DIR) Refs
 (DIR) README
 (DIR) LICENSE
       ---
 (DIR) commit 2807e03bd2cd30f25d5cf9dff512729f6df48224
 (DIR) parent fe689ce0c36c11c7f5c5e00b645a9ca82f3e4d07
 (HTM) Author: parazyd <parazyd@dyne.org>
       Date:   Fri, 26 May 2017 15:44:46 +0200
       
       implement basics of checking the remote for updates
       
       Diffstat:
         M .gitignore                          |       1 +
         A amprolla-download                   |      69 ++++++++++++++++++++++++++++++
         M amprolla-merge                      |       8 +++++---
       
       3 files changed, 75 insertions(+), 3 deletions(-)
       ---
 (DIR) diff --git a/.gitignore b/.gitignore
       t@@ -1,3 +1,4 @@
        *.pyc
        spool/
       +spool.tgz
        merged/
 (DIR) diff --git a/amprolla-download b/amprolla-download
       t@@ -0,0 +1,69 @@
       +#!/usr/bin/env python3
       +
       +import subprocess
       +from os.path import join
       +import requests
       +
       +from lib.net import download
       +from lib.parse import parse_release, get_time, get_date
       +
       +
       +roots = {
       +    'devuan': {
       +        'local': 'spool/devuan/dists/jessie',
       +        'remote': 'http://auto.mirror.devuan.org/devuan/dists/jessie',
       +    },
       +    'debian': {
       +        'local': 'spool/debian/dists/jessie',
       +        'remote': 'http://ftp.debian.org/debian/dists/jessie',
       +    },
       +    'debian-sec': {
       +        'local': 'spool/dists/jessie/updates',
       +        'remote': 'http://security.debian.org/dists/jessie/updates',
       +    },
       +}
       +
       +release_file = 'Release'
       +
       +def merge_files(repo, relfile):
       +    """
       +    Loads the local release and call the merge process
       +    """
       +    print('Loading Release')
       +    rel = join(roots[repo]['local'], relfile)
       +    release_contents = open(rel).read()
       +
       +    hashes = parse_release(release_contents)
       +
       +    for k in hashes.keys():
       +        # if k.endswith('Packages.gz'):
       +        if k.endswith('/binary-armhf/Packages.gz'):
       +            # skip empty files
       +            # TODO: probably best to copy it in place when this occurs
       +            if hashes[k] == 'f61f27bd17de546264aa58f40f3aafaac7021e0ef69c17f6b1b4cd7664a037ec':
       +                print('Skipping %s' % k)
       +                continue
       +
       +            subprocess.run(['./amprolla-merge', k])
       +
       +
       +local_rel = join(roots['devuan']['local'], release_file)
       +remote_rel = join(roots['devuan']['remote'], release_file)
       +
       +local_contents = open(local_rel).read()
       +local_date = get_date(local_contents)
       +
       +r = requests.get(remote_rel)
       +remote_contents = r.text
       +remote_date = get_date(remote_contents)
       +
       +print('Local date: %s' % local_date)
       +print('Remote date: %s' % remote_date)
       +
       +if get_time(remote_date) > get_time(local_date):
       +    # dump new release in place and merge
       +    # NOTE: when testing, watch out because you lose the old Release file in
       +    # spool
       +    print('Remote is newer')
       +    download(remote_rel, local_rel)
       +    merge_files('devuan', local_rel)
 (DIR) diff --git a/amprolla-merge b/amprolla-merge
       t@@ -3,6 +3,7 @@
        Amprolla main module
        """
        
       +import sys
        from os.path import join
        from time import time
        
       t@@ -13,7 +14,7 @@ from lib.config import banpkgs
        roots = {
            'devuan': 'spool/devuan/dists/jessie',
            'debian': 'spool/debian/dists/jessie',
       -    'debian-sec': 'spool/dists/jessie/updates/',
       +    'debian-sec': 'spool/dists/jessie/updates',
        }
        
        #devuan_release_contents = open(join(roots['devuan'], 'Release')).read()
       t@@ -23,10 +24,11 @@ roots = {
        #devuan_files = list(filter(lambda x: x.endswith('Packages.gz') and 'armhf' in x, devuan_release.keys()))
        #debian_files = list(filter(lambda x: x.endswith('Packages.gz') and 'armhf' in x, debian_release.keys()))
        
       -packages_file = 'main/binary-armhf/Packages.gz'
       +#packages_file = 'main/binary-armhf/Packages.gz'
       +packages_file = sys.argv[1]
        
        t1 = time()
       -print('Loading packages')
       +print('Loading packages: %s' % packages_file)
        
        devuan = load_packages_file(join(roots['devuan'], packages_file))
        debian = load_packages_file(join(roots['debian'], packages_file))