timplement basics of checking the remote for updates - amprolla - devuan's apt repo merger

git clone git://parazyd.org/amprolla.git

Log

Files

Refs

README

LICENSE

---

commit 2807e03bd2cd30f25d5cf9dff512729f6df48224

parent fe689ce0c36c11c7f5c5e00b645a9ca82f3e4d07

Author: parazyd 

Date:   Fri, 26 May 2017 15:44:46 +0200

implement basics of checking the remote for updates

Diffstat:
  M .gitignore                          |       1 +
  A amprolla-download                   |      69 ++++++++++++++++++++++++++++++
  M amprolla-merge                      |       8 +++++---

3 files changed, 75 insertions(+), 3 deletions(-)
---

diff --git a/.gitignore b/.gitignore

t@@ -1,3 +1,4 @@
 *.pyc
 spool/
+spool.tgz
 merged/

diff --git a/amprolla-download b/amprolla-download

t@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+
+import subprocess
+from os.path import join
+import requests
+
+from lib.net import download
+from lib.parse import parse_release, get_time, get_date
+
+
+roots = {
+    'devuan': {
+        'local': 'spool/devuan/dists/jessie',
+        'remote': 'http://auto.mirror.devuan.org/devuan/dists/jessie',
+    },
+    'debian': {
+        'local': 'spool/debian/dists/jessie',
+        'remote': 'http://ftp.debian.org/debian/dists/jessie',
+    },
+    'debian-sec': {
+        'local': 'spool/dists/jessie/updates',
+        'remote': 'http://security.debian.org/dists/jessie/updates',
+    },
+}
+
+release_file = 'Release'
+
+def merge_files(repo, relfile):
+    """
+    Loads the local release and call the merge process
+    """
+    print('Loading Release')
+    rel = join(roots[repo]['local'], relfile)
+    release_contents = open(rel).read()
+
+    hashes = parse_release(release_contents)
+
+    for k in hashes.keys():
+        # if k.endswith('Packages.gz'):
+        if k.endswith('/binary-armhf/Packages.gz'):
+            # skip empty files
+            # TODO: probably best to copy it in place when this occurs
+            if hashes[k] == 'f61f27bd17de546264aa58f40f3aafaac7021e0ef69c17f6b1b4cd7664a037ec':
+                print('Skipping %s' % k)
+                continue
+
+            subprocess.run(['./amprolla-merge', k])
+
+
+local_rel = join(roots['devuan']['local'], release_file)
+remote_rel = join(roots['devuan']['remote'], release_file)
+
+local_contents = open(local_rel).read()
+local_date = get_date(local_contents)
+
+r = requests.get(remote_rel)
+remote_contents = r.text
+remote_date = get_date(remote_contents)
+
+print('Local date: %s' % local_date)
+print('Remote date: %s' % remote_date)
+
+if get_time(remote_date) > get_time(local_date):
+    # dump new release in place and merge
+    # NOTE: when testing, watch out because you lose the old Release file in
+    # spool
+    print('Remote is newer')
+    download(remote_rel, local_rel)
+    merge_files('devuan', local_rel)

diff --git a/amprolla-merge b/amprolla-merge

t@@ -3,6 +3,7 @@
 Amprolla main module
 """
 
+import sys
 from os.path import join
 from time import time
 
t@@ -13,7 +14,7 @@ from lib.config import banpkgs
 roots = {
     'devuan': 'spool/devuan/dists/jessie',
     'debian': 'spool/debian/dists/jessie',
-    'debian-sec': 'spool/dists/jessie/updates/',
+    'debian-sec': 'spool/dists/jessie/updates',
 }
 
 #devuan_release_contents = open(join(roots['devuan'], 'Release')).read()
t@@ -23,10 +24,11 @@ roots = {
 #devuan_files = list(filter(lambda x: x.endswith('Packages.gz') and 'armhf' in x, devuan_release.keys()))
 #debian_files = list(filter(lambda x: x.endswith('Packages.gz') and 'armhf' in x, debian_release.keys()))
 
-packages_file = 'main/binary-armhf/Packages.gz'
+#packages_file = 'main/binary-armhf/Packages.gz'
+packages_file = sys.argv[1]
 
 t1 = time()
-print('Loading packages')
+print('Loading packages: %s' % packages_file)
 
 devuan = load_packages_file(join(roots['devuan'], packages_file))
 debian = load_packages_file(join(roots['debian'], packages_file))