From c2a0a51646d40bb5123296838152f48d71adeecf Mon Sep 17 00:00:00 2001 From: Kay0u Date: Wed, 16 Jun 2021 18:53:59 +0200 Subject: [PATCH] Upgrade to Sanic 21 --- requirements-frozen.txt | 19 ++++++----- requirements.txt | 2 +- run.py | 72 +++++++++++++++++++++++------------------ static/js/app.js | 2 +- templates/app.html | 2 +- templates/job.html | 2 +- 6 files changed, 55 insertions(+), 44 deletions(-) diff --git a/requirements-frozen.txt b/requirements-frozen.txt index 1b8f4d0..d5b8d3a 100644 --- a/requirements-frozen.txt +++ b/requirements-frozen.txt @@ -1,5 +1,5 @@ -aiofiles==0.4.0 -aiohttp==3.4.4 +aiofiles==0.7.0 +aiohttp==3.7.4.post0 argh==0.26.2 async-timeout==3.0.1 attrs==18.2.0 @@ -10,13 +10,16 @@ idna==2.8 idna-ssl==1.1.0 Jinja2==2.10.1 MarkupSafe==1.1.0 -multidict==4.5.2 -peewee==3.7.1 -requests==2.21.0 -sanic==19.3.1 -sanic-jinja2==0.7.2 +multidict==5.1.0 +peewee==3.14.4 +pkg-resources==0.0.0 +requests==2.25.1 +sanic==21.3.4 +sanic-jinja2==0.10.0 +sanic-routing==0.6.2 +typing-extensions==3.10.0.0 ujson==1.35 urllib3==1.24.2 uvloop==0.11.3 -websockets==6.0 +websockets==8.1 yarl==1.3.0 diff --git a/requirements.txt b/requirements.txt index 30f1f0b..ff70ef1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ aiohttp aiofiles peewee sanic-jinja2 -websockets<6.0 +websockets # cli argh diff --git a/run.py b/run.py index 41fd959..1d59ba4 100644 --- a/run.py +++ b/run.py @@ -20,7 +20,7 @@ from functools import wraps from concurrent.futures._base import CancelledError from asyncio import Task -import ujson +import json import aiohttp import aiofiles @@ -30,8 +30,8 @@ from websockets import WebSocketCommonProtocol from sanic import Sanic, response from sanic.exceptions import NotFound, abort from sanic.log import LOGGING_CONFIG_DEFAULTS -from sanic.response import json +from jinja2 import FileSystemLoader from sanic_jinja2 import SanicJinja2 from peewee import fn @@ -80,10 +80,11 @@ LOGGING_CONFIG_DEFAULTS["formatters"] = { task_logger = logging.getLogger("task") api_logger = logging.getLogger("api") -app = Sanic() +app = Sanic(__name__) app.static('/static', './static/') -jinja = SanicJinja2(app) +loader = FileSystemLoader(os.path.abspath(os.path.dirname(__file__)) + '/templates', encoding='utf8') +jinja = SanicJinja2(app, loader=loader) # to avoid conflict with vue.js jinja.env.block_start_string = '<%' @@ -104,6 +105,11 @@ subscriptions = defaultdict(list) jobs_in_memory_state = {} +def datetime_to_epoch_json_converter(o): + if isinstance(o, datetime): + return o.strftime('%s') + + @asyncio.coroutine def wait_closed(self): """ @@ -464,16 +470,12 @@ async def broadcast(message, channels): for channel in channels: ws_list = subscriptions[channel] - dead_ws = [] for ws in ws_list: try: - await ws.send(ujson.dumps(message)) + await ws.send(json.dumps(message, default=datetime_to_epoch_json_converter)) except ConnectionClosed: - dead_ws.append(ws) - - for to_remove in dead_ws: - ws_list.remove(to_remove) + ws_list.remove(ws) def subscribe(ws, channel): @@ -518,6 +520,7 @@ def chunks(l, n): yield chunk + @app.websocket('/index-ws') @clean_websocket async def ws_index(request, websocket): @@ -560,21 +563,21 @@ async def ws_index(request, websocket): first_chunck = next(data) - await websocket.send(ujson.dumps({ + await websocket.send(json.dumps({ "action": "init_jobs", "data": first_chunck, # send first chunk - })) + }, default=datetime_to_epoch_json_converter)) for chunk in data: - await websocket.send(ujson.dumps({ + await websocket.send(json.dumps({ "action": "init_jobs_stream", "data": chunk, - })) + }, default=datetime_to_epoch_json_converter)) await websocket.wait_closed() -@app.websocket('/job--ws') +@app.websocket('/job-ws/') @clean_websocket async def ws_job(request, websocket, job_id): job = Job.select().where(Job.id == job_id) @@ -586,10 +589,10 @@ async def ws_job(request, websocket, job_id): subscribe(websocket, f"job-{job.id}") - await websocket.send(ujson.dumps({ + await websocket.send(json.dumps({ "action": "init_job", "data": model_to_dict(job), - })) + }, default=datetime_to_epoch_json_converter)) await websocket.wait_closed() @@ -686,15 +689,15 @@ async def ws_apps(request, websocket): repos = sorted(repos, key=lambda x: x["name"]) - await websocket.send(ujson.dumps({ + await websocket.send(json.dumps({ "action": "init_apps", "data": repos, - })) + }, default=datetime_to_epoch_json_converter)) await websocket.wait_closed() -@app.websocket('/app--ws') +@app.websocket('/app-ws/') @clean_websocket async def ws_app(request, websocket, app_name): # XXX I don't check if the app exists because this websocket is supposed to @@ -703,11 +706,11 @@ async def ws_app(request, websocket, app_name): subscribe(websocket, f"app-jobs-{app.url}") - await websocket.send(ujson.dumps({ + await websocket.send(json.dumps({ "action": "init_jobs", "data": Job.select().where(Job.url_or_path == app.url).order_by(-Job.id), - })) + }, default=datetime_to_epoch_json_converter)) await websocket.wait_closed() @@ -807,9 +810,7 @@ async def api_delete_job(request, job_id): return response.text("ok") -@app.route("/api/job//stop", methods=['POST']) -async def api_stop_job(request, job_id): - # TODO auth or some kind +async def stop_job(job_id): job = Job.select().where(Job.id == job_id) if job.count() == 0: @@ -862,10 +863,17 @@ async def api_stop_job(request, job_id): f"{job.state}") +@app.route("/api/job//stop", methods=['POST']) +async def api_stop_job(request, job_id): + # TODO auth or some kind + await stop_job(job_id) + + @app.route("/api/job//restart", methods=['POST']) async def api_restart_job(request, job_id): api_logger.info(f"Request to restart job {job_id}") - await api_stop_job(request, job_id) + # Calling a route (eg api_stop_job) doesn't work anymore + await stop_job(job_id) # no need to check if job existss, api_stop_job will do it for us job = Job.select().where(Job.id == job_id)[0] @@ -930,7 +938,7 @@ async def html_job(request, job_id): } -@app.route('/apps/') +@app.route('/apps/', strict_slashes=True) # To avoid reaching the route "/apps//" with an empty string @jinja.template('apps.html') async def html_apps(request): return {'relative_path_to_root': '../', 'path': request.path} @@ -965,7 +973,7 @@ async def monitor(request): tasks = Task.all_tasks() - return json({ + return response.json({ "top_20_trace": [str(x) for x in top_stats[:20]], "tasks": { "number": len(tasks), @@ -1075,7 +1083,7 @@ async def github(request): token = open("./github_bot_token").read().strip() async with aiohttp.ClientSession(headers={"Authorization": f"token {token}"}) as session: - async with session.post(comments_url, data=ujson.dumps({"body": body})) as resp: + async with session.post(comments_url, data=json.dumps({"body": body}, default=datetime_to_epoch_json_converter)) as resp: api_logger.info("Added comment %s" % resp.json()["html_url"]) catchphrases = ["Alrighty!", "Fingers crossed!", "May the CI gods be with you!", ":carousel_horse:", ":rocket:", ":sunflower:", "Meow :cat2:", ":v:", ":stuck_out_tongue_winking_eye:" ] @@ -1130,11 +1138,11 @@ def main(config="./config.py"): "MONTHLY_JOBS": False, } - app.config.from_object(default_config) - app.config.from_pyfile(config) + app.config.update_config(default_config) + app.config.update_config(config) if not os.path.exists(app.config.PATH_TO_ANALYZER): - print(f"Error: analyzer script doesn't exist at '{PATH_TO_ANALYZER}'. Please fix the configuration in {config}") + print(f"Error: analyzer script doesn't exist at '{app.config.PATH_TO_ANALYZER}'. Please fix the configuration in {config}") sys.exit(1) reset_pending_jobs() diff --git a/static/js/app.js b/static/js/app.js index c22d5ed..d46e1df 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -339,7 +339,7 @@ var AnsiUp = (function () { }; return AnsiUp; }()); -//# sourceMappingURL=ansi_up.js.map +// sourceMappingURL=ansi_up.js.map Object.defineProperty(exports, "__esModule", { value: true }); exports.default = AnsiUp; })); diff --git a/templates/app.html b/templates/app.html index 039d9bd..1073b2d 100644 --- a/templates/app.html +++ b/templates/app.html @@ -44,7 +44,7 @@ } }) - ws = new ReconnectingWebSocket(websocketPrefix() + '://' + document.domain + ':' + location.port + websocketRelativePath('<{ path }>') + '/app-<{ app.name }>-ws'); + ws = new ReconnectingWebSocket(websocketPrefix() + '://' + document.domain + ':' + location.port + websocketRelativePath('<{ path }>') + '/app-ws/<{ app.name }>'); ws.onmessage = function (event) { var message = JSON.parse(event.data); diff --git a/templates/job.html b/templates/job.html index d19f296..e50a5bb 100644 --- a/templates/job.html +++ b/templates/job.html @@ -67,7 +67,7 @@ } }) - ws = new ReconnectingWebSocket(websocketPrefix() + '://' + document.domain + ':' + location.port + websocketRelativePath('<{ path }>') + '/job-<{ job.id }>-ws'); + ws = new ReconnectingWebSocket(websocketPrefix() + '://' + document.domain + ':' + location.port + websocketRelativePath('<{ path }>') + '/job-ws/<{ job.id }>'); ws.onmessage = function (event) { var message = JSON.parse(event.data);