[enh] queue/worker pattern to avoid hammering the server with ios

This commit is contained in:
Laurent Peuch 2018-04-09 18:22:31 +02:00
parent 16dedb609a
commit 48009625eb
2 changed files with 134 additions and 87 deletions

View file

@ -2,6 +2,7 @@
### Configuration ###
postgresql_dsn = "dbname=dynette user=dynette password=myPassword"
conf_file = '/etc/bind/named.conf.local' # Include this filename in '/etc/bind/named.conf'
zone_dir = '/var/lib/bind/' # Do not forget the trailing '/'
subs_urls = ['https://dyndns.yunohost.org'] # 127.0.0.1 if you install subscribe server locally
@ -26,8 +27,10 @@ allowed_operations = {
import os
import json
import psycopg2
from urllib import urlopen
# Get master key
master_key_path = os.path.join(os.path.dirname(__file__), 'master.key')
master_key = open(master_key_path).read().rstrip()
@ -35,6 +38,8 @@ master_key = open(master_key_path).read().rstrip()
# Bind configuration
lines = ['// Generated by Dynette CRON']
with psycopg2.connect(postgresql_dsn) as postgresql_connection:
with postgresql_connection.cursor() as psql:
# Loop through Dynette servers
for url in subs_urls:
@ -100,6 +105,28 @@ for url in subs_urls:
'};',
])
# look in the job queue if we have tasks to handle
need_rewrite = False
need_bind9_cache_flush = False
# DataMapper convert table names to lower cases and add a "s" at the
# end
# consume all available tasks at once to merge them and avoir doing
# useless jobs
for task in psql.execute("SELECT task FROM jobqueues ORDER BY id ASC;"):
task = task[0]
if task == "conf_rewrite":
need_rewrite = True
elif task == "bind9_cache_flush":
need_bind9_cache_flush = True
# we have consume all the jobs, flush it
# because we are in a SQL transaction we won't have situation where a
# job could be added just after we read them all
psql.execute("DELETE FROM jobqueues;")
# update bind9 zone
if need_rewrite:
# Backup old Bind configuration file.
os.system('cp '+ conf_file +' '+ conf_file +'.back')
@ -120,8 +147,8 @@ else:
print("An error occured ! Please check daemon.log and your conf.bad")
exit(1)
# mein got this is so awful
if os.path.exists('/tmp/dynette_flush_bind_cache'):
# flush bind9 cache (mostly because we got a hmac-sha512 key migration
if need_bind9_cache_flush:
os.system('/usr/sbin/rndc flush')
os.system('/usr/sbin/rndc reload')
os.system('rm /tmp/dynette_flush_bind_cache')

View file

@ -68,6 +68,25 @@ class Ipban
property :ip_addr, String, :key => true
end
################
### JobQueue ###
################
# JobQueue to communicate with the conf updater
class Jobqueue
include DataMapper::Resource
property :id, Serial, :key => true
property :task, String
end
def schedule_conf_rewrite(task)
Jobqueue.create(:task => "conf_rewrite")
end
def schedule_bind9_cache_flush(task)
Jobqueue.create(:task => "bind9_cache_flush")
end
################
### Handlers ###
@ -198,6 +217,7 @@ post '/key/:public_key' do
entry.ips << Ip.create(:ip_addr => request.ip)
if entry.save
schedule_conf_rewrite
halt 201, { :public_key => entry.public_key, :subdomain => entry.subdomain, :current_ip => entry.current_ip }.to_json
else
halt 412, { :error => "A problem occured during DNS registration" }.to_json
@ -237,11 +257,8 @@ put '/migrate_key_to_sha512/' do
halt 412, { :error => "A problem occured during key algo migration" }.to_json
end
# I don't have any other way of communicating with this dynette.cron.py
# this is awful
File.open("/tmp/dynette_flush_bind_cache", "w").close
# let's try flusing here, hope that could help ... (this design is so awful)
`/usr/sbin/rndc flush`
schedule_conf_rewrite
schedule_bind9_cache_flush
halt 201, { :public_key => entry.public_key, :subdomain => entry.subdomain, :current_ip => entry.current_ip }.to_json
end
@ -255,6 +272,7 @@ put '/key/:public_key' do
end
entry.current_ip = request.ip
if entry.save
schedule_conf_rewrite
halt 201, { :public_key => entry.public_key, :subdomain => entry.subdomain, :current_ip => entry.current_ip }.to_json
else
halt 412, { :error => "A problem occured during DNS update" }.to_json
@ -270,6 +288,7 @@ delete '/key/:public_key' do
if entry = Entry.first(:public_key => params[:public_key])
Ip.first(:entry_id => entry.id).destroy
if entry.destroy
schedule_conf_rewrite
halt 200, "OK".to_json
else
halt 412, { :error => "A problem occured during DNS deletion" }.to_json
@ -296,6 +315,7 @@ delete '/domains/:subdomain' do
Ip.first(:entry_id => entry.id).destroy
if entry.destroy
schedule_conf_rewrite
halt 200, "OK".to_json
else
halt 412, { :error => "A problem occured during DNS deletion" }.to_json