From 0fc80b256bc38fb244a70e7364a2400360424e68 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Mon, 18 Sep 2017 21:17:13 +0200 Subject: [PATCH 01/25] [mod] constant needs to be upper cases --- src/yunohost/dyndns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/yunohost/dyndns.py b/src/yunohost/dyndns.py index 621043c3e..7bbf5e568 100644 --- a/src/yunohost/dyndns.py +++ b/src/yunohost/dyndns.py @@ -63,7 +63,7 @@ class IPRouteLine(object): for k, v in self.m.groupdict().items(): setattr(self, k, v) -re_dyndns_private_key = re.compile( +RE_DYNDNS_PRIVATE_KEY = re.compile( r'.*/K(?P[^\s\+]+)\.\+157.+\.private$' ) @@ -176,7 +176,7 @@ def dyndns_update(dyn_host="dyndns.yunohost.org", domain=None, key=None, if domain is None: # Retrieve the first registered domain for path in glob.iglob('/etc/yunohost/dyndns/K*.private'): - match = re_dyndns_private_key.match(path) + match = RE_DYNDNS_PRIVATE_KEY.match(path) if not match: continue _domain = match.group('domain') From 622bed116935fcb9740b3c3551eb7fadfe05776f Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Mon, 18 Sep 2017 21:29:01 +0200 Subject: [PATCH 02/25] [mod] introduce regex for SHA512 --- src/yunohost/dyndns.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/yunohost/dyndns.py b/src/yunohost/dyndns.py index 7bbf5e568..842fdb841 100644 --- a/src/yunohost/dyndns.py +++ b/src/yunohost/dyndns.py @@ -63,11 +63,17 @@ class IPRouteLine(object): for k, v in self.m.groupdict().items(): setattr(self, k, v) -RE_DYNDNS_PRIVATE_KEY = re.compile( + +RE_DYNDNS_PRIVATE_KEY_MD5 = re.compile( r'.*/K(?P[^\s\+]+)\.\+157.+\.private$' ) +RE_DYNDNS_PRIVATE_KEY_SHA512 = re.compile( + r'.*/K(?P[^\s\+]+)\.\+163.+\.private$' +) + + def dyndns_subscribe(subscribe_host="dyndns.yunohost.org", domain=None, key=None): """ Subscribe to a DynDNS service @@ -176,7 +182,7 @@ def dyndns_update(dyn_host="dyndns.yunohost.org", domain=None, key=None, if domain is None: # Retrieve the first registered domain for path in glob.iglob('/etc/yunohost/dyndns/K*.private'): - match = RE_DYNDNS_PRIVATE_KEY.match(path) + match = RE_DYNDNS_PRIVATE_KEY_MD5.match(path) if not match: continue _domain = match.group('domain') From b53b05eabda62cefb0a9c56e60f30995fb4cb406 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Tue, 19 Sep 2017 00:08:56 +0200 Subject: [PATCH 03/25] [enh] register tsig as hmac-sha512 --- src/yunohost/dyndns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/yunohost/dyndns.py b/src/yunohost/dyndns.py index 842fdb841..e46598fbd 100644 --- a/src/yunohost/dyndns.py +++ b/src/yunohost/dyndns.py @@ -101,7 +101,7 @@ def dyndns_subscribe(subscribe_host="dyndns.yunohost.org", domain=None, key=None logger.info(m18n.n('dyndns_key_generating')) os.system('cd /etc/yunohost/dyndns && ' - 'dnssec-keygen -a hmac-md5 -b 128 -r /dev/urandom -n USER %s' % domain) + 'dnssec-keygen -a hmac-sha512 -b 512 -r /dev/urandom -n USER %s' % domain) os.system('chmod 600 /etc/yunohost/dyndns/*.key /etc/yunohost/dyndns/*.private') key_file = glob.glob('/etc/yunohost/dyndns/*.key')[0] @@ -110,7 +110,7 @@ def dyndns_subscribe(subscribe_host="dyndns.yunohost.org", domain=None, key=None # Send subscription try: - r = requests.post('https://%s/key/%s' % (subscribe_host, base64.b64encode(key)), data={'subdomain': domain}) + r = requests.post('https://%s/key/%s?key_algo=hmac-sha512' % (subscribe_host, base64.b64encode(key)), data={'subdomain': domain}) except requests.ConnectionError: raise MoulinetteError(errno.ENETUNREACH, m18n.n('no_internet_connection')) if r.status_code != 201: From 2c73dd1a2f7529aa85989175715c2f7cbf2feecd Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Tue, 19 Sep 2017 00:51:11 +0200 Subject: [PATCH 04/25] [fix] 163 was for sha256 --- src/yunohost/dyndns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/dyndns.py b/src/yunohost/dyndns.py index e46598fbd..516ac59de 100644 --- a/src/yunohost/dyndns.py +++ b/src/yunohost/dyndns.py @@ -70,7 +70,7 @@ RE_DYNDNS_PRIVATE_KEY_MD5 = re.compile( RE_DYNDNS_PRIVATE_KEY_SHA512 = re.compile( - r'.*/K(?P[^\s\+]+)\.\+163.+\.private$' + r'.*/K(?P[^\s\+]+)\.\+165.+\.private$' ) From f5f813badb326a5fcbfb29ae5aa816ab2a4ff217 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Thu, 21 Sep 2017 05:52:59 +0200 Subject: [PATCH 05/25] [fix] hmac-sha512 key can have space in them because why the fuck not --- src/yunohost/dyndns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/dyndns.py b/src/yunohost/dyndns.py index 516ac59de..c3e216071 100644 --- a/src/yunohost/dyndns.py +++ b/src/yunohost/dyndns.py @@ -106,7 +106,7 @@ def dyndns_subscribe(subscribe_host="dyndns.yunohost.org", domain=None, key=None key_file = glob.glob('/etc/yunohost/dyndns/*.key')[0] with open(key_file) as f: - key = f.readline().strip().split(' ')[-1] + key = f.readline().strip().split(' ', 6)[-1] # Send subscription try: From 87a5759ca427c5e80ddc58d5d53fdb9eb8ecb91e Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Thu, 21 Sep 2017 06:56:11 +0200 Subject: [PATCH 06/25] [enh] automatically migrate tsig done using md5 to sha512 when doing a dyndns update --- src/yunohost/dyndns.py | 46 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/src/yunohost/dyndns.py b/src/yunohost/dyndns.py index c3e216071..be37072f9 100644 --- a/src/yunohost/dyndns.py +++ b/src/yunohost/dyndns.py @@ -182,9 +182,11 @@ def dyndns_update(dyn_host="dyndns.yunohost.org", domain=None, key=None, if domain is None: # Retrieve the first registered domain for path in glob.iglob('/etc/yunohost/dyndns/K*.private'): - match = RE_DYNDNS_PRIVATE_KEY_MD5.match(path) + match = RE_DYNDNS_PRIVATE_KEY_SHA512.match(path) if not match: - continue + match = RE_DYNDNS_PRIVATE_KEY_MD5.match(path) + if not match: + continue _domain = match.group('domain') try: @@ -213,6 +215,11 @@ def dyndns_update(dyn_host="dyndns.yunohost.org", domain=None, key=None, key = keys[0] + # this mean that hmac-md5 is used + if "+157" in key: + print "detecting md5 key" + key = _migrate_from_md5_tsig_to_sha512_tsig(key, domain, dyn_host) + host = domain.split('.')[1:] host = '.'.join(host) @@ -269,6 +276,41 @@ def dyndns_update(dyn_host="dyndns.yunohost.org", domain=None, key=None, f.write(ipv6) +def _migrate_from_md5_tsig_to_sha512_tsig(private_key_path, domain, dyn_host): + public_key_path = private_key_path.rsplit(".private", 1)[0] + ".key" + public_key_md5 = open(public_key_path).read().strip().split(' ')[-1] + + os.system('cd /etc/yunohost/dyndns && ' + 'dnssec-keygen -a hmac-sha512 -b 512 -r /dev/urandom -n USER %s' % domain) + os.system('chmod 600 /etc/yunohost/dyndns/*.key /etc/yunohost/dyndns/*.private') + + # +165 means that this file store a hmac-sha512 key + new_key_path = glob.glob('/etc/yunohost/dyndns/*+165*.key')[0] + public_key_sha512 = open(new_key_path).read().strip().split(' ', 6)[-1] + + try: + r = requests.put('https://%s/migrate_key_to_sha512/' % (dyn_host), + data={ + 'public_key_md5': base64.b64encode(public_key_md5), + 'public_key_sha512': base64.b64encode(public_key_sha512), + }) + except requests.ConnectionError: + raise MoulinetteError(errno.ENETUNREACH, m18n.n('no_internet_connection')) + + if r.status_code != 201: + print r.text + error = json.loads(r.text)['error'] + print "ERROR:", error + # raise MoulinetteError(errno.EPERM, + # m18n.n('dyndns_registration_failed', error=error)) + # XXX print warning + os.system("mv /etc/yunohost/dyndns/*+165* /tmp") + return public_key_path + + os.system("mv /etc/yunohost/dyndns/*+157* /tmp") + return new_key_path.rsplit(".key", 1)[0] + ".private" + + def dyndns_installcron(): """ Install IP update cron From fbdf6842542432be2012b81530136271a178b39d Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Mon, 9 Oct 2017 12:12:08 +0200 Subject: [PATCH 07/25] [mod] add failure handling mechanism for json decoding --- src/yunohost/dyndns.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/yunohost/dyndns.py b/src/yunohost/dyndns.py index be37072f9..d596e2525 100644 --- a/src/yunohost/dyndns.py +++ b/src/yunohost/dyndns.py @@ -299,8 +299,15 @@ def _migrate_from_md5_tsig_to_sha512_tsig(private_key_path, domain, dyn_host): if r.status_code != 201: print r.text - error = json.loads(r.text)['error'] - print "ERROR:", error + try: + error = json.loads(r.text)['error'] + print "ERROR:", error + except Exception as e: + import traceback + traceback.print_exc() + print e + error = r.text + # raise MoulinetteError(errno.EPERM, # m18n.n('dyndns_registration_failed', error=error)) # XXX print warning From a32359c76a6005d3a46b8a73aea2d5c9766116dd Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Mon, 9 Oct 2017 12:12:41 +0200 Subject: [PATCH 08/25] [mod] wait locally for the dynette cron to have runned during tsig migration --- src/yunohost/dyndns.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/yunohost/dyndns.py b/src/yunohost/dyndns.py index d596e2525..ff6a5d42c 100644 --- a/src/yunohost/dyndns.py +++ b/src/yunohost/dyndns.py @@ -27,6 +27,7 @@ import os import re import json import glob +import time import base64 import errno import requests @@ -315,6 +316,10 @@ def _migrate_from_md5_tsig_to_sha512_tsig(private_key_path, domain, dyn_host): return public_key_path os.system("mv /etc/yunohost/dyndns/*+157* /tmp") + + # sleep to wait for dyndns cache invalidation + time.sleep(180) + return new_key_path.rsplit(".key", 1)[0] + ".private" From 50867079836553c98c6faac05b524dafd8fdbd89 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Mon, 9 Oct 2017 12:14:14 +0200 Subject: [PATCH 09/25] [mod] add some documentation comment --- src/yunohost/dyndns.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/yunohost/dyndns.py b/src/yunohost/dyndns.py index ff6a5d42c..ccbfdaffb 100644 --- a/src/yunohost/dyndns.py +++ b/src/yunohost/dyndns.py @@ -315,6 +315,7 @@ def _migrate_from_md5_tsig_to_sha512_tsig(private_key_path, domain, dyn_host): os.system("mv /etc/yunohost/dyndns/*+165* /tmp") return public_key_path + # remove old certificates os.system("mv /etc/yunohost/dyndns/*+157* /tmp") # sleep to wait for dyndns cache invalidation From 044b2406d3c1a0f11e246dc1f2827a91e8a77212 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Wed, 3 Jan 2018 18:45:18 +0100 Subject: [PATCH 10/25] [enh] better logging during key migration --- locales/en.json | 4 ++++ src/yunohost/dyndns.py | 11 ++++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/locales/en.json b/locales/en.json index 80ff22655..421a04f56 100644 --- a/locales/en.json +++ b/locales/en.json @@ -207,6 +207,10 @@ "mailbox_used_space_dovecot_down": "Dovecot mailbox service need to be up, if you want to get mailbox used space", "maindomain_change_failed": "Unable to change the main domain", "maindomain_changed": "The main domain has been changed", + "migrate_tsig_end": "Migration to hmac-sha512 finished", + "migrate_tsig_failed": "Migrating the dyndns domain {domain} to hmac-sha512 failed, rolling back. Error: {error_code} - {error}", + "migrate_tsig_start": "Not secure enough key algorithm detected for TSIG signature of domain '{domain}', initiating migration to the more secure one hmac-sha512", + "migrate_tsig_wait": "Let's wait 3min for the dyndns server to take the new key into account...", "migrations_backward": "Migrating backward.", "migrations_bad_value_for_target": "Invalide number for target argument, available migrations numbers are 0 or {}", "migrations_cant_reach_migration_file": "Can't access migrations files at path %s", diff --git a/src/yunohost/dyndns.py b/src/yunohost/dyndns.py index ccbfdaffb..459a1e04e 100644 --- a/src/yunohost/dyndns.py +++ b/src/yunohost/dyndns.py @@ -278,6 +278,7 @@ def dyndns_update(dyn_host="dyndns.yunohost.org", domain=None, key=None, def _migrate_from_md5_tsig_to_sha512_tsig(private_key_path, domain, dyn_host): + logger.warning(m18n.n('migrate_tsig_start', domain=domain)) public_key_path = private_key_path.rsplit(".private", 1)[0] + ".key" public_key_md5 = open(public_key_path).read().strip().split(' ')[-1] @@ -299,19 +300,17 @@ def _migrate_from_md5_tsig_to_sha512_tsig(private_key_path, domain, dyn_host): raise MoulinetteError(errno.ENETUNREACH, m18n.n('no_internet_connection')) if r.status_code != 201: - print r.text try: error = json.loads(r.text)['error'] - print "ERROR:", error except Exception as e: import traceback traceback.print_exc() print e error = r.text - # raise MoulinetteError(errno.EPERM, - # m18n.n('dyndns_registration_failed', error=error)) - # XXX print warning + logger.warning(m18n.n('migrate_tsig_failed', domain=domain, + error_code=str(r.status_code), error=error)) + os.system("mv /etc/yunohost/dyndns/*+165* /tmp") return public_key_path @@ -319,8 +318,10 @@ def _migrate_from_md5_tsig_to_sha512_tsig(private_key_path, domain, dyn_host): os.system("mv /etc/yunohost/dyndns/*+157* /tmp") # sleep to wait for dyndns cache invalidation + logger.warning(m18n.n('migrate_tsig_wait')) time.sleep(180) + logger.warning(m18n.n('migrate_tsig_end')) return new_key_path.rsplit(".key", 1)[0] + ".private" From 299cba623b8ac9d4cfc5edbb360826e052681ec7 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Wed, 3 Jan 2018 18:52:01 +0100 Subject: [PATCH 11/25] [fix] avoid stupid mistake --- src/yunohost/dyndns.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/yunohost/dyndns.py b/src/yunohost/dyndns.py index 459a1e04e..36f30bc03 100644 --- a/src/yunohost/dyndns.py +++ b/src/yunohost/dyndns.py @@ -97,7 +97,8 @@ def dyndns_subscribe(subscribe_host="dyndns.yunohost.org", domain=None, key=None if key is None: if len(glob.glob('/etc/yunohost/dyndns/*.key')) == 0: - os.makedirs('/etc/yunohost/dyndns') + if not os.path.exists('/etc/yunohost/dyndns'): + os.makedirs('/etc/yunohost/dyndns') logger.info(m18n.n('dyndns_key_generating')) From 544ffb273e4977a421660a8b4b0d282da48087c4 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Wed, 3 Jan 2018 19:00:16 +0100 Subject: [PATCH 12/25] [mod] improve waiting time UX --- locales/en.json | 3 +++ src/yunohost/dyndns.py | 8 +++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/locales/en.json b/locales/en.json index 421a04f56..3a8c2b13c 100644 --- a/locales/en.json +++ b/locales/en.json @@ -211,6 +211,9 @@ "migrate_tsig_failed": "Migrating the dyndns domain {domain} to hmac-sha512 failed, rolling back. Error: {error_code} - {error}", "migrate_tsig_start": "Not secure enough key algorithm detected for TSIG signature of domain '{domain}', initiating migration to the more secure one hmac-sha512", "migrate_tsig_wait": "Let's wait 3min for the dyndns server to take the new key into account...", + "migrate_tsig_wait_2": "2min...", + "migrate_tsig_wait_3": "1min...", + "migrate_tsig_wait_3": "30 secondes...", "migrations_backward": "Migrating backward.", "migrations_bad_value_for_target": "Invalide number for target argument, available migrations numbers are 0 or {}", "migrations_cant_reach_migration_file": "Can't access migrations files at path %s", diff --git a/src/yunohost/dyndns.py b/src/yunohost/dyndns.py index 36f30bc03..d4313dc8b 100644 --- a/src/yunohost/dyndns.py +++ b/src/yunohost/dyndns.py @@ -320,7 +320,13 @@ def _migrate_from_md5_tsig_to_sha512_tsig(private_key_path, domain, dyn_host): # sleep to wait for dyndns cache invalidation logger.warning(m18n.n('migrate_tsig_wait')) - time.sleep(180) + time.sleep(60) + logger.warning(m18n.n('migrate_tsig_wait_2')) + time.sleep(60) + logger.warning(m18n.n('migrate_tsig_wait_3')) + time.sleep(30) + logger.warning(m18n.n('migrate_tsig_wait_4')) + time.sleep(30) logger.warning(m18n.n('migrate_tsig_end')) return new_key_path.rsplit(".key", 1)[0] + ".private" From 89358173387ddfd88e458f8e803c90e0f16cbbc1 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Wed, 3 Jan 2018 19:21:53 +0100 Subject: [PATCH 13/25] [fix] add timeout on requests --- src/yunohost/dyndns.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/yunohost/dyndns.py b/src/yunohost/dyndns.py index d4313dc8b..aabc191a1 100644 --- a/src/yunohost/dyndns.py +++ b/src/yunohost/dyndns.py @@ -90,7 +90,7 @@ def dyndns_subscribe(subscribe_host="dyndns.yunohost.org", domain=None, key=None # Verify if domain is available try: - if requests.get('https://%s/test/%s' % (subscribe_host, domain)).status_code != 200: + if requests.get('https://%s/test/%s' % (subscribe_host, domain), timeout=30).status_code != 200: raise MoulinetteError(errno.EEXIST, m18n.n('dyndns_unavailable')) except requests.ConnectionError: raise MoulinetteError(errno.ENETUNREACH, m18n.n('no_internet_connection')) @@ -112,7 +112,7 @@ def dyndns_subscribe(subscribe_host="dyndns.yunohost.org", domain=None, key=None # Send subscription try: - r = requests.post('https://%s/key/%s?key_algo=hmac-sha512' % (subscribe_host, base64.b64encode(key)), data={'subdomain': domain}) + r = requests.post('https://%s/key/%s' % (subscribe_host, base64.b64encode(key)), data={'subdomain': domain}, timeout=30) except requests.ConnectionError: raise MoulinetteError(errno.ENETUNREACH, m18n.n('no_internet_connection')) if r.status_code != 201: @@ -296,7 +296,7 @@ def _migrate_from_md5_tsig_to_sha512_tsig(private_key_path, domain, dyn_host): data={ 'public_key_md5': base64.b64encode(public_key_md5), 'public_key_sha512': base64.b64encode(public_key_sha512), - }) + }, timeout=30) except requests.ConnectionError: raise MoulinetteError(errno.ENETUNREACH, m18n.n('no_internet_connection')) From 28b44401f5f97425df42c5d1a7ab7daf0aaccdb8 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Wed, 3 Jan 2018 19:22:10 +0100 Subject: [PATCH 14/25] [mod] remove debug print --- src/yunohost/dyndns.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/yunohost/dyndns.py b/src/yunohost/dyndns.py index aabc191a1..d2137f91a 100644 --- a/src/yunohost/dyndns.py +++ b/src/yunohost/dyndns.py @@ -219,7 +219,6 @@ def dyndns_update(dyn_host="dyndns.yunohost.org", domain=None, key=None, # this mean that hmac-md5 is used if "+157" in key: - print "detecting md5 key" key = _migrate_from_md5_tsig_to_sha512_tsig(key, domain, dyn_host) host = domain.split('.')[1:] From 57665b6f11968dc60a65fa74a891004e135d3eb9 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Wed, 3 Jan 2018 19:23:47 +0100 Subject: [PATCH 15/25] [mod] uses builtin function to show traceback --- src/yunohost/dyndns.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/yunohost/dyndns.py b/src/yunohost/dyndns.py index d2137f91a..d65f5f7ed 100644 --- a/src/yunohost/dyndns.py +++ b/src/yunohost/dyndns.py @@ -302,14 +302,15 @@ def _migrate_from_md5_tsig_to_sha512_tsig(private_key_path, domain, dyn_host): if r.status_code != 201: try: error = json.loads(r.text)['error'] - except Exception as e: - import traceback - traceback.print_exc() - print e + show_traceback = 0 + except Exception: + # failed to decode json error = r.text + show_traceback = 1 logger.warning(m18n.n('migrate_tsig_failed', domain=domain, - error_code=str(r.status_code), error=error)) + error_code=str(r.status_code), error=error), + exc_info=show_traceback) os.system("mv /etc/yunohost/dyndns/*+165* /tmp") return public_key_path From a9eea61a26e2ed78a9e2496b20150f1d8c5bcb27 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Wed, 3 Jan 2018 20:07:32 +0100 Subject: [PATCH 16/25] [fix] overwritting key --- locales/en.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/locales/en.json b/locales/en.json index b2d7c847a..a2df29a54 100644 --- a/locales/en.json +++ b/locales/en.json @@ -216,7 +216,7 @@ "migrate_tsig_wait": "Let's wait 3min for the dyndns server to take the new key into account...", "migrate_tsig_wait_2": "2min...", "migrate_tsig_wait_3": "1min...", - "migrate_tsig_wait_3": "30 secondes...", + "migrate_tsig_wait_4": "30 secondes...", "migrations_backward": "Migrating backward.", "migrations_bad_value_for_target": "Invalide number for target argument, available migrations numbers are 0 or {}", "migrations_cant_reach_migration_file": "Can't access migrations files at path %s", From 7c63c0a34de6013b9e528437d7c705902e6ce322 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Wed, 3 Jan 2018 20:22:46 +0100 Subject: [PATCH 17/25] [enh] subscribe has hmac-sha512 --- src/yunohost/dyndns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/dyndns.py b/src/yunohost/dyndns.py index c11d0a067..6ff73d1e2 100644 --- a/src/yunohost/dyndns.py +++ b/src/yunohost/dyndns.py @@ -153,7 +153,7 @@ def dyndns_subscribe(subscribe_host="dyndns.yunohost.org", domain=None, key=None # Send subscription try: - r = requests.post('https://%s/key/%s' % (subscribe_host, base64.b64encode(key)), data={'subdomain': domain}, timeout=30) + r = requests.post('https://%s/key/%s?key_algo=hmac-sha512' % (subscribe_host, base64.b64encode(key)), data={'subdomain': domain}, timeout=30) except requests.ConnectionError: raise MoulinetteError(errno.ENETUNREACH, m18n.n('no_internet_connection')) if r.status_code != 201: From beb432bc6fca34ad11141083a3bef115197b6c9c Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 6 Jan 2018 18:05:01 +0100 Subject: [PATCH 18/25] Add a draft migration for tsig_sha256 based on existing stuff in dyndns.py --- .../0002_migrate_to_tsig_sha256.py | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py diff --git a/src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py b/src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py new file mode 100644 index 000000000..df27a1f9f --- /dev/null +++ b/src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py @@ -0,0 +1,79 @@ +import glob + +from yunohost.tools import Migration +from moulinette.utils.log import getActionLogger +logger = getActionLogger('yunohost.migration') + + +class MigrateToTsigSha512(Migration): + "Migrate Dyndns stuff from MD5 TSIG to SHA512 TSIG": + + + def backward(self): + # Not possible because that's a non-reversible operation ? + pass + + + def forward(self): + + dyn_host="dyndns.yunohost.org" + + try: + (domain, private_key_path) = _guess_current_dyndns_domain(dyn_host) + except MoulinetteError: + logger.warning("migrate_tsig_not_needed") + + logger.warning(m18n.n('migrate_tsig_start', domain=domain)) + public_key_path = private_key_path.rsplit(".private", 1)[0] + ".key" + public_key_md5 = open(public_key_path).read().strip().split(' ')[-1] + + os.system('cd /etc/yunohost/dyndns && ' + 'dnssec-keygen -a hmac-sha512 -b 512 -r /dev/urandom -n USER %s' % domain) + os.system('chmod 600 /etc/yunohost/dyndns/*.key /etc/yunohost/dyndns/*.private') + + # +165 means that this file store a hmac-sha512 key + new_key_path = glob.glob('/etc/yunohost/dyndns/*+165*.key')[0] + public_key_sha512 = open(new_key_path).read().strip().split(' ', 6)[-1] + + try: + r = requests.put('https://%s/migrate_key_to_sha512/' % (dyn_host), + data={ + 'public_key_md5': base64.b64encode(public_key_md5), + 'public_key_sha512': base64.b64encode(public_key_sha512), + }, timeout=30) + except requests.ConnectionError: + raise MoulinetteError(errno.ENETUNREACH, m18n.n('no_internet_connection')) + + if r.status_code != 201: + try: + error = json.loads(r.text)['error'] + show_traceback = 0 + except Exception: + # failed to decode json + error = r.text + show_traceback = 1 + + logger.warning(m18n.n('migrate_tsig_failed', domain=domain, + error_code=str(r.status_code), error=error), + exc_info=show_traceback) + + os.system("mv /etc/yunohost/dyndns/*+165* /tmp") + return public_key_path + + # remove old certificates + os.system("mv /etc/yunohost/dyndns/*+157* /tmp") + + # sleep to wait for dyndns cache invalidation + logger.warning(m18n.n('migrate_tsig_wait')) + time.sleep(60) + logger.warning(m18n.n('migrate_tsig_wait_2')) + time.sleep(60) + logger.warning(m18n.n('migrate_tsig_wait_3')) + time.sleep(30) + logger.warning(m18n.n('migrate_tsig_wait_4')) + time.sleep(30) + + logger.warning(m18n.n('migrate_tsig_end')) + return new_key_path.rsplit(".key", 1)[0] + ".private" + + From 6f8912f0d431df79ccfe1ed0811e1d80911815e5 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 6 Jan 2018 18:50:30 +0100 Subject: [PATCH 19/25] Fix a few things following tests --- .../0002_migrate_to_tsig_sha256.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py b/src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py index df27a1f9f..26ea3a8b8 100644 --- a/src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py +++ b/src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py @@ -1,13 +1,21 @@ import glob +import os +import requests +import base64 +import time + +from moulinette import m18n +from moulinette.core import MoulinetteError +from moulinette.utils.log import getActionLogger from yunohost.tools import Migration -from moulinette.utils.log import getActionLogger +from yunohost.dyndns import _guess_current_dyndns_domain + logger = getActionLogger('yunohost.migration') -class MigrateToTsigSha512(Migration): - "Migrate Dyndns stuff from MD5 TSIG to SHA512 TSIG": - +class MyMigration(Migration): + "Migrate Dyndns stuff from MD5 TSIG to SHA512 TSIG" def backward(self): # Not possible because that's a non-reversible operation ? @@ -22,6 +30,7 @@ class MigrateToTsigSha512(Migration): (domain, private_key_path) = _guess_current_dyndns_domain(dyn_host) except MoulinetteError: logger.warning("migrate_tsig_not_needed") + return logger.warning(m18n.n('migrate_tsig_start', domain=domain)) public_key_path = private_key_path.rsplit(".private", 1)[0] + ".key" From 465aff4581cac68bd8776ab6a59c9a7fd868bf39 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 6 Jan 2018 20:51:20 +0100 Subject: [PATCH 20/25] Be able to fetch a single migration by its name --- src/yunohost/tools.py | 63 ++++++++++++++++++++++++++++--------------- 1 file changed, 42 insertions(+), 21 deletions(-) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 042671125..024ae0da4 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -717,30 +717,10 @@ def tools_migrations_migrate(target=None, skip=False): # loading all migrations for migration in tools_migrations_list()["migrations"]: - logger.debug(m18n.n('migrations_loading_migration', - number=migration["number"], - name=migration["name"], - )) - - try: - # this is python builtin method to import a module using a name, we - # use that to import the migration as a python object so we'll be - # able to run it in the next loop - module = import_module("yunohost.data_migrations.{file_name}".format(**migration)) - except Exception: - import traceback - traceback.print_exc() - - raise MoulinetteError(errno.EINVAL, m18n.n('migrations_error_failed_to_load_migration', - number=migration["number"], - name=migration["name"], - )) - break - migrations.append({ "number": migration["number"], "name": migration["name"], - "module": module, + "module": _get_migration_module(migration), }) migrations = sorted(migrations, key=lambda x: x["number"]) @@ -877,6 +857,47 @@ def _get_migrations_list(): return sorted(migrations) +def _get_migration_by_name(migration_name, with_module=True): + """ + Low-level / "private" function to find a migration by its name + """ + + migrations = tools_migrations_list()["migrations"] + + matches = [ m for m in migrations if m["name"] == migration_name ] + + assert len(matches) == 1, "Unable to find migration with name %s" % migration_name + + migration = matches[0] + + if with_module: + migration["module"] = _get_migration_module(migration) + + return migration + + +def _get_migration_module(migration): + + logger.debug(m18n.n('migrations_loading_migration', + number=migration["number"], + name=migration["name"], + )) + + try: + # this is python builtin method to import a module using a name, we + # use that to import the migration as a python object so we'll be + # able to run it in the next loop + return import_module("yunohost.data_migrations.{file_name}".format(**migration)) + except Exception: + import traceback + traceback.print_exc() + + raise MoulinetteError(errno.EINVAL, m18n.n('migrations_error_failed_to_load_migration', + number=migration["number"], + name=migration["name"], + )) + + class Migration(object): def migrate(self): From 7e02e355d51dc819b183d697d9acc0e8c731248d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 6 Jan 2018 21:01:10 +0100 Subject: [PATCH 21/25] Call the new migration from dyndns.py when MD5 is detected --- .../0002_migrate_to_tsig_sha256.py | 16 ++--- src/yunohost/dyndns.py | 68 +++---------------- 2 files changed, 19 insertions(+), 65 deletions(-) diff --git a/src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py b/src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py index 26ea3a8b8..257f3525a 100644 --- a/src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py +++ b/src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py @@ -22,15 +22,15 @@ class MyMigration(Migration): pass - def forward(self): + def forward(self, dyn_host="dyndns.yunohost.org", domain=None, private_key_path=None): - dyn_host="dyndns.yunohost.org" - - try: - (domain, private_key_path) = _guess_current_dyndns_domain(dyn_host) - except MoulinetteError: - logger.warning("migrate_tsig_not_needed") - return + if domain in None or private_key_path is None: + try: + (domain, private_key_path) = _guess_current_dyndns_domain(dyn_host) + assert "+157" in private_key_path + except MoulinetteError: + logger.warning("migrate_tsig_not_needed") + return logger.warning(m18n.n('migrate_tsig_start', domain=domain)) public_key_path = private_key_path.rsplit(".private", 1)[0] + ".key" diff --git a/src/yunohost/dyndns.py b/src/yunohost/dyndns.py index 6ff73d1e2..851d04f45 100644 --- a/src/yunohost/dyndns.py +++ b/src/yunohost/dyndns.py @@ -223,9 +223,18 @@ def dyndns_update(dyn_host="dyndns.yunohost.org", domain=None, key=None, key = keys[0] - # this mean that hmac-md5 is used + # This mean that hmac-md5 is used + # (Re?)Trigger the migration to sha256 and return immediately. + # The actual update will be done in next run. if "+157" in key: - key = _migrate_from_md5_tsig_to_sha512_tsig(key, domain, dyn_host) + from yunohost.tools import _get_migration_by_name + migration = _get_migration_by_name("migrate_to_tsig_sha256") + try: + migration["module"].MyMigration().migrate(dyn_host, domain, key) + except Exception as e: + logger.error(m18n.n('migrations_migration_has_failed', exception=e, **migration), exc_info=1) + + return # Extract 'host', e.g. 'nohost.me' from 'foo.nohost.me' host = domain.split('.')[1:] @@ -292,61 +301,6 @@ def dyndns_update(dyn_host="dyndns.yunohost.org", domain=None, key=None, write_to_file(OLD_IPV6_FILE, ipv6) -def _migrate_from_md5_tsig_to_sha512_tsig(private_key_path, domain, dyn_host): - logger.warning(m18n.n('migrate_tsig_start', domain=domain)) - public_key_path = private_key_path.rsplit(".private", 1)[0] + ".key" - public_key_md5 = open(public_key_path).read().strip().split(' ')[-1] - - os.system('cd /etc/yunohost/dyndns && ' - 'dnssec-keygen -a hmac-sha512 -b 512 -r /dev/urandom -n USER %s' % domain) - os.system('chmod 600 /etc/yunohost/dyndns/*.key /etc/yunohost/dyndns/*.private') - - # +165 means that this file store a hmac-sha512 key - new_key_path = glob.glob('/etc/yunohost/dyndns/*+165*.key')[0] - public_key_sha512 = open(new_key_path).read().strip().split(' ', 6)[-1] - - try: - r = requests.put('https://%s/migrate_key_to_sha512/' % (dyn_host), - data={ - 'public_key_md5': base64.b64encode(public_key_md5), - 'public_key_sha512': base64.b64encode(public_key_sha512), - }, timeout=30) - except requests.ConnectionError: - raise MoulinetteError(errno.ENETUNREACH, m18n.n('no_internet_connection')) - - if r.status_code != 201: - try: - error = json.loads(r.text)['error'] - show_traceback = 0 - except Exception: - # failed to decode json - error = r.text - show_traceback = 1 - - logger.warning(m18n.n('migrate_tsig_failed', domain=domain, - error_code=str(r.status_code), error=error), - exc_info=show_traceback) - - os.system("mv /etc/yunohost/dyndns/*+165* /tmp") - return public_key_path - - # remove old certificates - os.system("mv /etc/yunohost/dyndns/*+157* /tmp") - - # sleep to wait for dyndns cache invalidation - logger.warning(m18n.n('migrate_tsig_wait')) - time.sleep(60) - logger.warning(m18n.n('migrate_tsig_wait_2')) - time.sleep(60) - logger.warning(m18n.n('migrate_tsig_wait_3')) - time.sleep(30) - logger.warning(m18n.n('migrate_tsig_wait_4')) - time.sleep(30) - - logger.warning(m18n.n('migrate_tsig_end')) - return new_key_path.rsplit(".key", 1)[0] + ".private" - - def dyndns_installcron(): """ Install IP update cron From 6bf80877af7ae0b0d3ad2530ea66e617ae8e2c72 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 6 Jan 2018 21:42:47 +0100 Subject: [PATCH 22/25] Fixing a few stuff after tests.. --- .../0002_migrate_to_tsig_sha256.py | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py b/src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py index 257f3525a..98c591716 100644 --- a/src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py +++ b/src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py @@ -3,6 +3,7 @@ import os import requests import base64 import time +import json from moulinette import m18n from moulinette.core import MoulinetteError @@ -21,14 +22,13 @@ class MyMigration(Migration): # Not possible because that's a non-reversible operation ? pass + def migrate(self, dyn_host="dyndns.yunohost.org", domain=None, private_key_path=None): - def forward(self, dyn_host="dyndns.yunohost.org", domain=None, private_key_path=None): - - if domain in None or private_key_path is None: + if domain is None or private_key_path is None: try: (domain, private_key_path) = _guess_current_dyndns_domain(dyn_host) - assert "+157" in private_key_path - except MoulinetteError: + #assert "+157" in private_key_path + except (MoulinetteError, AssertionError): logger.warning("migrate_tsig_not_needed") return @@ -56,18 +56,21 @@ class MyMigration(Migration): if r.status_code != 201: try: error = json.loads(r.text)['error'] - show_traceback = 0 except Exception: # failed to decode json error = r.text - show_traceback = 1 - logger.warning(m18n.n('migrate_tsig_failed', domain=domain, - error_code=str(r.status_code), error=error), - exc_info=show_traceback) + import traceback + from StringIO import StringIO + stack = StringIO() + traceback.print_exc(file=stack) + logger.error(stack.getvalue()) + # Migration didn't succeed, so we rollback and raise an exception os.system("mv /etc/yunohost/dyndns/*+165* /tmp") - return public_key_path + + raise MoulinetteError(m18n.n('migrate_tsig_failed', domain=domain, + error_code=str(r.status_code), error=error)) # remove old certificates os.system("mv /etc/yunohost/dyndns/*+157* /tmp") @@ -83,6 +86,5 @@ class MyMigration(Migration): time.sleep(30) logger.warning(m18n.n('migrate_tsig_end')) - return new_key_path.rsplit(".key", 1)[0] + ".private" - + return From 08caf2e07ff380f0613668c1e93e85d5025116b9 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 6 Jan 2018 22:42:37 +0100 Subject: [PATCH 23/25] Fixing a few stuff after tests.. --- src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py b/src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py index 98c591716..15f092b75 100644 --- a/src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py +++ b/src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py @@ -27,7 +27,7 @@ class MyMigration(Migration): if domain is None or private_key_path is None: try: (domain, private_key_path) = _guess_current_dyndns_domain(dyn_host) - #assert "+157" in private_key_path + assert "+157" in private_key_path except (MoulinetteError, AssertionError): logger.warning("migrate_tsig_not_needed") return @@ -63,7 +63,7 @@ class MyMigration(Migration): import traceback from StringIO import StringIO stack = StringIO() - traceback.print_exc(file=stack) + traceback.print_stack(file=stack) logger.error(stack.getvalue()) # Migration didn't succeed, so we rollback and raise an exception From 6ccf054d5c2b265a37b4e495ec476ed9dff301ab Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 14 Jan 2018 00:40:44 +0100 Subject: [PATCH 24/25] Forgot to import errno --- src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py b/src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py index 15f092b75..9cfd64842 100644 --- a/src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py +++ b/src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py @@ -4,6 +4,7 @@ import requests import base64 import time import json +import errno from moulinette import m18n from moulinette.core import MoulinetteError From d8435eaccc6627b02a2ded47f80a755ef5dbece9 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 14 Jan 2018 00:59:48 +0100 Subject: [PATCH 25/25] Add missing translation --- locales/en.json | 1 + src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/locales/en.json b/locales/en.json index a2df29a54..ffb3233d4 100644 --- a/locales/en.json +++ b/locales/en.json @@ -217,6 +217,7 @@ "migrate_tsig_wait_2": "2min...", "migrate_tsig_wait_3": "1min...", "migrate_tsig_wait_4": "30 secondes...", + "migrate_tsig_not_needed": "You do not appear to use a dyndns domain, so no migration is needed !", "migrations_backward": "Migrating backward.", "migrations_bad_value_for_target": "Invalide number for target argument, available migrations numbers are 0 or {}", "migrations_cant_reach_migration_file": "Can't access migrations files at path %s", diff --git a/src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py b/src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py index 9cfd64842..5d495b06c 100644 --- a/src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py +++ b/src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py @@ -30,7 +30,7 @@ class MyMigration(Migration): (domain, private_key_path) = _guess_current_dyndns_domain(dyn_host) assert "+157" in private_key_path except (MoulinetteError, AssertionError): - logger.warning("migrate_tsig_not_needed") + logger.warning(m18n.n("migrate_tsig_not_needed")) return logger.warning(m18n.n('migrate_tsig_start', domain=domain))