From 48009625eb52fa75fd4823a2daba9c37fa8e2393 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Mon, 9 Apr 2018 18:22:31 +0200 Subject: [PATCH 1/7] [enh] queue/worker pattern to avoid hammering the server with ios --- dynette.cron.py | 191 +++++++++++++++++++++++++++--------------------- dynette.rb | 30 ++++++-- 2 files changed, 134 insertions(+), 87 deletions(-) diff --git a/dynette.cron.py b/dynette.cron.py index 77dfe74..261635f 100755 --- a/dynette.cron.py +++ b/dynette.cron.py @@ -2,6 +2,7 @@ ### Configuration ### +postgresql_dsn = "dbname=dynette user=dynette password=myPassword" conf_file = '/etc/bind/named.conf.local' # Include this filename in '/etc/bind/named.conf' zone_dir = '/var/lib/bind/' # Do not forget the trailing '/' subs_urls = ['https://dyndns.yunohost.org'] # 127.0.0.1 if you install subscribe server locally @@ -26,8 +27,10 @@ allowed_operations = { import os import json +import psycopg2 from urllib import urlopen + # Get master key master_key_path = os.path.join(os.path.dirname(__file__), 'master.key') master_key = open(master_key_path).read().rstrip() @@ -35,93 +38,117 @@ master_key = open(master_key_path).read().rstrip() # Bind configuration lines = ['// Generated by Dynette CRON'] -# Loop through Dynette servers -for url in subs_urls: +with psycopg2.connect(postgresql_dsn) as postgresql_connection: + with postgresql_connection.cursor() as psql: + # Loop through Dynette servers + for url in subs_urls: - lines.extend([ - 'key dynette. {', - ' algorithm hmac-md5;', - ' secret "'+ master_key +'";', - '};', - ]) - - # Get available DynDNS domains - domains = json.loads(str(urlopen(url +'/domains').read())) - for domain in domains: - - # Create zone database if not present - if not os.path.exists(zone_dir + domain +'.db'): - db_lines = [ - '$ORIGIN .', - '$TTL 10 ; 10 seconds', - domain+'. IN SOA '+ ns0 +'. '+ rname +'. (', - ' 18 ; serial', - ' 10800 ; refresh (3 hours)', - ' 3600 ; retry (1 hour)', - ' 604800 ; expire (1 week)', - ' 10 ; minimum (10 seconds)', - ' )', - '$TTL 3600 ; 1 hour', - ' NS '+ ns0 +'.', - ' NS '+ ns1 +'.', - '', - '$ORIGIN '+ domain +'.', - ] - with open(zone_dir + domain +'.db', 'w') as zone: - for line in db_lines: - zone.write(line + '\n') - - lines.extend([ - 'zone "'+ domain +'" {', - ' type master;', - ' file "'+ zone_dir + domain +'.db"; ', - ' update-policy {', - ' grant dynette. wildcard *.'+ domain +'. ANY;', - ]) - - # Get registered sub-domains - result = json.loads(str(urlopen(url +'/all/'+ domain).read())) - for entry in result: - for subd, type in allowed_operations.items(): - if subd == '.': subd = '' - lines.append(' grant '+ entry['subdomain'] +'. name '+ subd + entry['subdomain'] +'. ' + ' '.join(type) +';') - - lines.extend([ - ' };', - '};' - '', - ]) - - for entry in result: lines.extend([ - 'key '+ entry['subdomain'] +'. {', - ' algorithm ' + entry['key_algo'] + ';', - ' secret "'+ entry['public_key'] +'";', - '};', + 'key dynette. {', + ' algorithm hmac-md5;', + ' secret "'+ master_key +'";', + '};', ]) -# Backup old Bind configuration file. -os.system('cp '+ conf_file +' '+ conf_file +'.back') + # Get available DynDNS domains + domains = json.loads(str(urlopen(url +'/domains').read())) + for domain in domains: -# Write Bind configuration file. -with open(conf_file, 'w') as zone: - zone.write('\n'.join(lines) + '\n') + # Create zone database if not present + if not os.path.exists(zone_dir + domain +'.db'): + db_lines = [ + '$ORIGIN .', + '$TTL 10 ; 10 seconds', + domain+'. IN SOA '+ ns0 +'. '+ rname +'. (', + ' 18 ; serial', + ' 10800 ; refresh (3 hours)', + ' 3600 ; retry (1 hour)', + ' 604800 ; expire (1 week)', + ' 10 ; minimum (10 seconds)', + ' )', + '$TTL 3600 ; 1 hour', + ' NS '+ ns0 +'.', + ' NS '+ ns1 +'.', + '', + '$ORIGIN '+ domain +'.', + ] + with open(zone_dir + domain +'.db', 'w') as zone: + for line in db_lines: + zone.write(line + '\n') -# Restore ownership -os.system('chown -R bind:bind '+ zone_dir +' '+ conf_file) + lines.extend([ + 'zone "'+ domain +'" {', + ' type master;', + ' file "'+ zone_dir + domain +'.db"; ', + ' update-policy {', + ' grant dynette. wildcard *.'+ domain +'. ANY;', + ]) -# Reload Bind -if os.system('/usr/sbin/rndc reload') == 0: - exit(0) -else: - os.system('cp '+ conf_file +' '+ conf_file +'.bad') - os.system('cp '+ conf_file +'.back '+ conf_file) - os.system('/usr/sbin/rndc reload') - print("An error occured ! Please check daemon.log and your conf.bad") - exit(1) + # Get registered sub-domains + result = json.loads(str(urlopen(url +'/all/'+ domain).read())) + for entry in result: + for subd, type in allowed_operations.items(): + if subd == '.': subd = '' + lines.append(' grant '+ entry['subdomain'] +'. name '+ subd + entry['subdomain'] +'. ' + ' '.join(type) +';') -# mein got this is so awful -if os.path.exists('/tmp/dynette_flush_bind_cache'): - os.system('/usr/sbin/rndc flush') - os.system('/usr/sbin/rndc reload') - os.system('rm /tmp/dynette_flush_bind_cache') + lines.extend([ + ' };', + '};' + '', + ]) + + for entry in result: + lines.extend([ + 'key '+ entry['subdomain'] +'. {', + ' algorithm ' + entry['key_algo'] + ';', + ' secret "'+ entry['public_key'] +'";', + '};', + ]) + + # look in the job queue if we have tasks to handle + need_rewrite = False + need_bind9_cache_flush = False + + # DataMapper convert table names to lower cases and add a "s" at the + # end + # consume all available tasks at once to merge them and avoir doing + # useless jobs + for task in psql.execute("SELECT task FROM jobqueues ORDER BY id ASC;"): + task = task[0] + if task == "conf_rewrite": + need_rewrite = True + elif task == "bind9_cache_flush": + need_bind9_cache_flush = True + + # we have consume all the jobs, flush it + # because we are in a SQL transaction we won't have situation where a + # job could be added just after we read them all + psql.execute("DELETE FROM jobqueues;") + + # update bind9 zone + if need_rewrite: + # Backup old Bind configuration file. + os.system('cp '+ conf_file +' '+ conf_file +'.back') + + # Write Bind configuration file. + with open(conf_file, 'w') as zone: + zone.write('\n'.join(lines) + '\n') + + # Restore ownership + os.system('chown -R bind:bind '+ zone_dir +' '+ conf_file) + + # Reload Bind + if os.system('/usr/sbin/rndc reload') == 0: + exit(0) + else: + os.system('cp '+ conf_file +' '+ conf_file +'.bad') + os.system('cp '+ conf_file +'.back '+ conf_file) + os.system('/usr/sbin/rndc reload') + print("An error occured ! Please check daemon.log and your conf.bad") + exit(1) + + # flush bind9 cache (mostly because we got a hmac-sha512 key migration + if need_bind9_cache_flush: + os.system('/usr/sbin/rndc flush') + os.system('/usr/sbin/rndc reload') + os.system('rm /tmp/dynette_flush_bind_cache') diff --git a/dynette.rb b/dynette.rb index b4703d5..c6d5d37 100755 --- a/dynette.rb +++ b/dynette.rb @@ -68,6 +68,25 @@ class Ipban property :ip_addr, String, :key => true end +################ +### JobQueue ### +################ + +# JobQueue to communicate with the conf updater +class Jobqueue + include DataMapper::Resource + + property :id, Serial, :key => true + property :task, String +end + +def schedule_conf_rewrite(task) + Jobqueue.create(:task => "conf_rewrite") +end + +def schedule_bind9_cache_flush(task) + Jobqueue.create(:task => "bind9_cache_flush") +end ################ ### Handlers ### @@ -198,6 +217,7 @@ post '/key/:public_key' do entry.ips << Ip.create(:ip_addr => request.ip) if entry.save + schedule_conf_rewrite halt 201, { :public_key => entry.public_key, :subdomain => entry.subdomain, :current_ip => entry.current_ip }.to_json else halt 412, { :error => "A problem occured during DNS registration" }.to_json @@ -237,11 +257,8 @@ put '/migrate_key_to_sha512/' do halt 412, { :error => "A problem occured during key algo migration" }.to_json end - # I don't have any other way of communicating with this dynette.cron.py - # this is awful - File.open("/tmp/dynette_flush_bind_cache", "w").close - # let's try flusing here, hope that could help ... (this design is so awful) - `/usr/sbin/rndc flush` + schedule_conf_rewrite + schedule_bind9_cache_flush halt 201, { :public_key => entry.public_key, :subdomain => entry.subdomain, :current_ip => entry.current_ip }.to_json end @@ -255,6 +272,7 @@ put '/key/:public_key' do end entry.current_ip = request.ip if entry.save + schedule_conf_rewrite halt 201, { :public_key => entry.public_key, :subdomain => entry.subdomain, :current_ip => entry.current_ip }.to_json else halt 412, { :error => "A problem occured during DNS update" }.to_json @@ -270,6 +288,7 @@ delete '/key/:public_key' do if entry = Entry.first(:public_key => params[:public_key]) Ip.first(:entry_id => entry.id).destroy if entry.destroy + schedule_conf_rewrite halt 200, "OK".to_json else halt 412, { :error => "A problem occured during DNS deletion" }.to_json @@ -296,6 +315,7 @@ delete '/domains/:subdomain' do Ip.first(:entry_id => entry.id).destroy if entry.destroy + schedule_conf_rewrite halt 200, "OK".to_json else halt 412, { :error => "A problem occured during DNS deletion" }.to_json From 8f3aa7cf0e84378bcca962e9cf18bb656a6863b2 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Wed, 18 Apr 2018 03:35:18 +0200 Subject: [PATCH 2/7] [mod] we don't need to do that anymore --- dynette.cron.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dynette.cron.py b/dynette.cron.py index 261635f..84c505d 100755 --- a/dynette.cron.py +++ b/dynette.cron.py @@ -151,4 +151,3 @@ with psycopg2.connect(postgresql_dsn) as postgresql_connection: if need_bind9_cache_flush: os.system('/usr/sbin/rndc flush') os.system('/usr/sbin/rndc reload') - os.system('rm /tmp/dynette_flush_bind_cache') From af45f67cc5239a9c86dc8b75e23a512c1bc238a6 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Wed, 18 Apr 2018 03:48:46 +0200 Subject: [PATCH 3/7] [mod] avoid useless get --- dynette.cron.py | 44 ++++++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/dynette.cron.py b/dynette.cron.py index 84c505d..80e40d3 100755 --- a/dynette.cron.py +++ b/dynette.cron.py @@ -26,6 +26,7 @@ allowed_operations = { ### Script ### import os +import sys import json import psycopg2 from urllib import urlopen @@ -40,6 +41,29 @@ lines = ['// Generated by Dynette CRON'] with psycopg2.connect(postgresql_dsn) as postgresql_connection: with postgresql_connection.cursor() as psql: + # look in the job queue if we have tasks to handle + need_rewrite = False + need_bind9_cache_flush = False + + # DataMapper convert table names to lower cases and add a "s" at the + # end + # consume all available tasks at once to merge them and avoir doing + # useless jobs + for task in psql.execute("SELECT task FROM jobqueues ORDER BY id ASC;"): + task = task[0] + if task == "conf_rewrite": + need_rewrite = True + elif task == "bind9_cache_flush": + need_bind9_cache_flush = True + + if not need_rewrite and not need_bind9_cache_flush: + sys.exit(0) + + # we have consume all the jobs, flush it + # because we are in a SQL transaction we won't have situation where a + # job could be added just after we read them all + psql.execute("DELETE FROM jobqueues;") + # Loop through Dynette servers for url in subs_urls: @@ -105,26 +129,6 @@ with psycopg2.connect(postgresql_dsn) as postgresql_connection: '};', ]) - # look in the job queue if we have tasks to handle - need_rewrite = False - need_bind9_cache_flush = False - - # DataMapper convert table names to lower cases and add a "s" at the - # end - # consume all available tasks at once to merge them and avoir doing - # useless jobs - for task in psql.execute("SELECT task FROM jobqueues ORDER BY id ASC;"): - task = task[0] - if task == "conf_rewrite": - need_rewrite = True - elif task == "bind9_cache_flush": - need_bind9_cache_flush = True - - # we have consume all the jobs, flush it - # because we are in a SQL transaction we won't have situation where a - # job could be added just after we read them all - psql.execute("DELETE FROM jobqueues;") - # update bind9 zone if need_rewrite: # Backup old Bind configuration file. From eb0628d3b7f6a114b53f1f9a9ef33154b268a9cb Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Wed, 18 Apr 2018 03:49:53 +0200 Subject: [PATCH 4/7] [fix] psycopg2 is boring --- dynette.cron.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dynette.cron.py b/dynette.cron.py index 80e40d3..6ede07e 100755 --- a/dynette.cron.py +++ b/dynette.cron.py @@ -49,7 +49,9 @@ with psycopg2.connect(postgresql_dsn) as postgresql_connection: # end # consume all available tasks at once to merge them and avoir doing # useless jobs - for task in psql.execute("SELECT task FROM jobqueues ORDER BY id ASC;"): + psql.execute("SELECT task FROM jobqueues ORDER BY id ASC;") + + for task in psql.fetchall(): task = task[0] if task == "conf_rewrite": need_rewrite = True From c291ed5a2bb25631ba5b337ea70015e4e975943d Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Wed, 18 Apr 2018 04:25:54 +0200 Subject: [PATCH 5/7] [fix] no arguments needed --- dynette.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dynette.rb b/dynette.rb index c6d5d37..b8616d7 100755 --- a/dynette.rb +++ b/dynette.rb @@ -80,11 +80,11 @@ class Jobqueue property :task, String end -def schedule_conf_rewrite(task) +def schedule_conf_rewrite() Jobqueue.create(:task => "conf_rewrite") end -def schedule_bind9_cache_flush(task) +def schedule_bind9_cache_flush() Jobqueue.create(:task => "bind9_cache_flush") end From e2120ed8f06ecadb84d2e46114c9a0d0b485a770 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Wed, 18 Apr 2018 04:50:54 +0200 Subject: [PATCH 6/7] [mod] exit is from sys --- dynette.cron.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dynette.cron.py b/dynette.cron.py index 6ede07e..21dda87 100755 --- a/dynette.cron.py +++ b/dynette.cron.py @@ -145,13 +145,13 @@ with psycopg2.connect(postgresql_dsn) as postgresql_connection: # Reload Bind if os.system('/usr/sbin/rndc reload') == 0: - exit(0) + sys.exit(0) else: os.system('cp '+ conf_file +' '+ conf_file +'.bad') os.system('cp '+ conf_file +'.back '+ conf_file) os.system('/usr/sbin/rndc reload') print("An error occured ! Please check daemon.log and your conf.bad") - exit(1) + sys.exit(1) # flush bind9 cache (mostly because we got a hmac-sha512 key migration if need_bind9_cache_flush: From 2cc808333fd005b00542c29ece16a7e2a1d033ff Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Wed, 18 Apr 2018 04:51:12 +0200 Subject: [PATCH 7/7] [fix] do not exit on success bind reload --- dynette.cron.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dynette.cron.py b/dynette.cron.py index 21dda87..8205f98 100755 --- a/dynette.cron.py +++ b/dynette.cron.py @@ -144,9 +144,7 @@ with psycopg2.connect(postgresql_dsn) as postgresql_connection: os.system('chown -R bind:bind '+ zone_dir +' '+ conf_file) # Reload Bind - if os.system('/usr/sbin/rndc reload') == 0: - sys.exit(0) - else: + if os.system('/usr/sbin/rndc reload') != 0: os.system('cp '+ conf_file +' '+ conf_file +'.bad') os.system('cp '+ conf_file +'.back '+ conf_file) os.system('/usr/sbin/rndc reload')