From c2a0a51646d40bb5123296838152f48d71adeecf Mon Sep 17 00:00:00 2001 From: Kay0u Date: Wed, 16 Jun 2021 18:53:59 +0200 Subject: [PATCH 01/11] Upgrade to Sanic 21 --- requirements-frozen.txt | 19 ++++++----- requirements.txt | 2 +- run.py | 72 +++++++++++++++++++++++------------------ static/js/app.js | 2 +- templates/app.html | 2 +- templates/job.html | 2 +- 6 files changed, 55 insertions(+), 44 deletions(-) diff --git a/requirements-frozen.txt b/requirements-frozen.txt index 1b8f4d0..d5b8d3a 100644 --- a/requirements-frozen.txt +++ b/requirements-frozen.txt @@ -1,5 +1,5 @@ -aiofiles==0.4.0 -aiohttp==3.4.4 +aiofiles==0.7.0 +aiohttp==3.7.4.post0 argh==0.26.2 async-timeout==3.0.1 attrs==18.2.0 @@ -10,13 +10,16 @@ idna==2.8 idna-ssl==1.1.0 Jinja2==2.10.1 MarkupSafe==1.1.0 -multidict==4.5.2 -peewee==3.7.1 -requests==2.21.0 -sanic==19.3.1 -sanic-jinja2==0.7.2 +multidict==5.1.0 +peewee==3.14.4 +pkg-resources==0.0.0 +requests==2.25.1 +sanic==21.3.4 +sanic-jinja2==0.10.0 +sanic-routing==0.6.2 +typing-extensions==3.10.0.0 ujson==1.35 urllib3==1.24.2 uvloop==0.11.3 -websockets==6.0 +websockets==8.1 yarl==1.3.0 diff --git a/requirements.txt b/requirements.txt index 30f1f0b..ff70ef1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ aiohttp aiofiles peewee sanic-jinja2 -websockets<6.0 +websockets # cli argh diff --git a/run.py b/run.py index 41fd959..1d59ba4 100644 --- a/run.py +++ b/run.py @@ -20,7 +20,7 @@ from functools import wraps from concurrent.futures._base import CancelledError from asyncio import Task -import ujson +import json import aiohttp import aiofiles @@ -30,8 +30,8 @@ from websockets import WebSocketCommonProtocol from sanic import Sanic, response from sanic.exceptions import NotFound, abort from sanic.log import LOGGING_CONFIG_DEFAULTS -from sanic.response import json +from jinja2 import FileSystemLoader from sanic_jinja2 import SanicJinja2 from peewee import fn @@ -80,10 +80,11 @@ LOGGING_CONFIG_DEFAULTS["formatters"] = { task_logger = logging.getLogger("task") api_logger = logging.getLogger("api") -app = Sanic() +app = Sanic(__name__) app.static('/static', './static/') -jinja = SanicJinja2(app) +loader = FileSystemLoader(os.path.abspath(os.path.dirname(__file__)) + '/templates', encoding='utf8') +jinja = SanicJinja2(app, loader=loader) # to avoid conflict with vue.js jinja.env.block_start_string = '<%' @@ -104,6 +105,11 @@ subscriptions = defaultdict(list) jobs_in_memory_state = {} +def datetime_to_epoch_json_converter(o): + if isinstance(o, datetime): + return o.strftime('%s') + + @asyncio.coroutine def wait_closed(self): """ @@ -464,16 +470,12 @@ async def broadcast(message, channels): for channel in channels: ws_list = subscriptions[channel] - dead_ws = [] for ws in ws_list: try: - await ws.send(ujson.dumps(message)) + await ws.send(json.dumps(message, default=datetime_to_epoch_json_converter)) except ConnectionClosed: - dead_ws.append(ws) - - for to_remove in dead_ws: - ws_list.remove(to_remove) + ws_list.remove(ws) def subscribe(ws, channel): @@ -518,6 +520,7 @@ def chunks(l, n): yield chunk + @app.websocket('/index-ws') @clean_websocket async def ws_index(request, websocket): @@ -560,21 +563,21 @@ async def ws_index(request, websocket): first_chunck = next(data) - await websocket.send(ujson.dumps({ + await websocket.send(json.dumps({ "action": "init_jobs", "data": first_chunck, # send first chunk - })) + }, default=datetime_to_epoch_json_converter)) for chunk in data: - await websocket.send(ujson.dumps({ + await websocket.send(json.dumps({ "action": "init_jobs_stream", "data": chunk, - })) + }, default=datetime_to_epoch_json_converter)) await websocket.wait_closed() -@app.websocket('/job--ws') +@app.websocket('/job-ws/') @clean_websocket async def ws_job(request, websocket, job_id): job = Job.select().where(Job.id == job_id) @@ -586,10 +589,10 @@ async def ws_job(request, websocket, job_id): subscribe(websocket, f"job-{job.id}") - await websocket.send(ujson.dumps({ + await websocket.send(json.dumps({ "action": "init_job", "data": model_to_dict(job), - })) + }, default=datetime_to_epoch_json_converter)) await websocket.wait_closed() @@ -686,15 +689,15 @@ async def ws_apps(request, websocket): repos = sorted(repos, key=lambda x: x["name"]) - await websocket.send(ujson.dumps({ + await websocket.send(json.dumps({ "action": "init_apps", "data": repos, - })) + }, default=datetime_to_epoch_json_converter)) await websocket.wait_closed() -@app.websocket('/app--ws') +@app.websocket('/app-ws/') @clean_websocket async def ws_app(request, websocket, app_name): # XXX I don't check if the app exists because this websocket is supposed to @@ -703,11 +706,11 @@ async def ws_app(request, websocket, app_name): subscribe(websocket, f"app-jobs-{app.url}") - await websocket.send(ujson.dumps({ + await websocket.send(json.dumps({ "action": "init_jobs", "data": Job.select().where(Job.url_or_path == app.url).order_by(-Job.id), - })) + }, default=datetime_to_epoch_json_converter)) await websocket.wait_closed() @@ -807,9 +810,7 @@ async def api_delete_job(request, job_id): return response.text("ok") -@app.route("/api/job//stop", methods=['POST']) -async def api_stop_job(request, job_id): - # TODO auth or some kind +async def stop_job(job_id): job = Job.select().where(Job.id == job_id) if job.count() == 0: @@ -862,10 +863,17 @@ async def api_stop_job(request, job_id): f"{job.state}") +@app.route("/api/job//stop", methods=['POST']) +async def api_stop_job(request, job_id): + # TODO auth or some kind + await stop_job(job_id) + + @app.route("/api/job//restart", methods=['POST']) async def api_restart_job(request, job_id): api_logger.info(f"Request to restart job {job_id}") - await api_stop_job(request, job_id) + # Calling a route (eg api_stop_job) doesn't work anymore + await stop_job(job_id) # no need to check if job existss, api_stop_job will do it for us job = Job.select().where(Job.id == job_id)[0] @@ -930,7 +938,7 @@ async def html_job(request, job_id): } -@app.route('/apps/') +@app.route('/apps/', strict_slashes=True) # To avoid reaching the route "/apps//" with an empty string @jinja.template('apps.html') async def html_apps(request): return {'relative_path_to_root': '../', 'path': request.path} @@ -965,7 +973,7 @@ async def monitor(request): tasks = Task.all_tasks() - return json({ + return response.json({ "top_20_trace": [str(x) for x in top_stats[:20]], "tasks": { "number": len(tasks), @@ -1075,7 +1083,7 @@ async def github(request): token = open("./github_bot_token").read().strip() async with aiohttp.ClientSession(headers={"Authorization": f"token {token}"}) as session: - async with session.post(comments_url, data=ujson.dumps({"body": body})) as resp: + async with session.post(comments_url, data=json.dumps({"body": body}, default=datetime_to_epoch_json_converter)) as resp: api_logger.info("Added comment %s" % resp.json()["html_url"]) catchphrases = ["Alrighty!", "Fingers crossed!", "May the CI gods be with you!", ":carousel_horse:", ":rocket:", ":sunflower:", "Meow :cat2:", ":v:", ":stuck_out_tongue_winking_eye:" ] @@ -1130,11 +1138,11 @@ def main(config="./config.py"): "MONTHLY_JOBS": False, } - app.config.from_object(default_config) - app.config.from_pyfile(config) + app.config.update_config(default_config) + app.config.update_config(config) if not os.path.exists(app.config.PATH_TO_ANALYZER): - print(f"Error: analyzer script doesn't exist at '{PATH_TO_ANALYZER}'. Please fix the configuration in {config}") + print(f"Error: analyzer script doesn't exist at '{app.config.PATH_TO_ANALYZER}'. Please fix the configuration in {config}") sys.exit(1) reset_pending_jobs() diff --git a/static/js/app.js b/static/js/app.js index c22d5ed..d46e1df 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -339,7 +339,7 @@ var AnsiUp = (function () { }; return AnsiUp; }()); -//# sourceMappingURL=ansi_up.js.map +// sourceMappingURL=ansi_up.js.map Object.defineProperty(exports, "__esModule", { value: true }); exports.default = AnsiUp; })); diff --git a/templates/app.html b/templates/app.html index 039d9bd..1073b2d 100644 --- a/templates/app.html +++ b/templates/app.html @@ -44,7 +44,7 @@ } }) - ws = new ReconnectingWebSocket(websocketPrefix() + '://' + document.domain + ':' + location.port + websocketRelativePath('<{ path }>') + '/app-<{ app.name }>-ws'); + ws = new ReconnectingWebSocket(websocketPrefix() + '://' + document.domain + ':' + location.port + websocketRelativePath('<{ path }>') + '/app-ws/<{ app.name }>'); ws.onmessage = function (event) { var message = JSON.parse(event.data); diff --git a/templates/job.html b/templates/job.html index d19f296..e50a5bb 100644 --- a/templates/job.html +++ b/templates/job.html @@ -67,7 +67,7 @@ } }) - ws = new ReconnectingWebSocket(websocketPrefix() + '://' + document.domain + ':' + location.port + websocketRelativePath('<{ path }>') + '/job-<{ job.id }>-ws'); + ws = new ReconnectingWebSocket(websocketPrefix() + '://' + document.domain + ':' + location.port + websocketRelativePath('<{ path }>') + '/job-ws/<{ job.id }>'); ws.onmessage = function (event) { var message = JSON.parse(event.data); From f60055ae8efe60adc69dbe3fe241ccadbe684099 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Wed, 16 Jun 2021 19:09:35 +0200 Subject: [PATCH 02/11] Update Jinja2 and urllib3 --- requirements-frozen.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-frozen.txt b/requirements-frozen.txt index d5b8d3a..36125e9 100644 --- a/requirements-frozen.txt +++ b/requirements-frozen.txt @@ -8,8 +8,8 @@ chardet==3.0.4 httptools==0.0.11 idna==2.8 idna-ssl==1.1.0 -Jinja2==2.10.1 -MarkupSafe==1.1.0 +Jinja2==3.0.1 +MarkupSafe==2.0.1 multidict==5.1.0 peewee==3.14.4 pkg-resources==0.0.0 @@ -19,7 +19,7 @@ sanic-jinja2==0.10.0 sanic-routing==0.6.2 typing-extensions==3.10.0.0 ujson==1.35 -urllib3==1.24.2 +urllib3==1.26.5 uvloop==0.11.3 websockets==8.1 yarl==1.3.0 From 84df910e200a1e3036802564100f1ed69d49afda Mon Sep 17 00:00:00 2001 From: Kay0u Date: Wed, 16 Jun 2021 19:24:19 +0200 Subject: [PATCH 03/11] fix an issue in app view --- requirements-frozen.txt | 2 +- run.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-frozen.txt b/requirements-frozen.txt index 36125e9..7eca41b 100644 --- a/requirements-frozen.txt +++ b/requirements-frozen.txt @@ -18,7 +18,7 @@ sanic==21.3.4 sanic-jinja2==0.10.0 sanic-routing==0.6.2 typing-extensions==3.10.0.0 -ujson==1.35 +ujson==4.0.2 urllib3==1.26.5 uvloop==0.11.3 websockets==8.1 diff --git a/run.py b/run.py index 1d59ba4..937ca69 100644 --- a/run.py +++ b/run.py @@ -706,10 +706,10 @@ async def ws_app(request, websocket, app_name): subscribe(websocket, f"app-jobs-{app.url}") + job = list(Job.select().where(Job.url_or_path == app.url).order_by(-Job.id).dicts()) await websocket.send(json.dumps({ "action": "init_jobs", - "data": Job.select().where(Job.url_or_path == - app.url).order_by(-Job.id), + "data": job, }, default=datetime_to_epoch_json_converter)) await websocket.wait_closed() From 657810edc35217ec08ef4dcc352cf3a9a35c895d Mon Sep 17 00:00:00 2001 From: Kay0u Date: Thu, 17 Jun 2021 16:13:10 +0200 Subject: [PATCH 04/11] revert the way we remove websockets in a list --- run.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/run.py b/run.py index 937ca69..08aa4ad 100644 --- a/run.py +++ b/run.py @@ -470,13 +470,16 @@ async def broadcast(message, channels): for channel in channels: ws_list = subscriptions[channel] + dead_ws = [] for ws in ws_list: try: await ws.send(json.dumps(message, default=datetime_to_epoch_json_converter)) except ConnectionClosed: - ws_list.remove(ws) + dead_ws.append(ws) + for to_remove in dead_ws: + ws_list.remove(to_remove) def subscribe(ws, channel): subscriptions[channel].append(ws) From 19f5bcd0004008ebb53cb4099685a70f239c32a2 Mon Sep 17 00:00:00 2001 From: yalh76 Date: Wed, 30 Jun 2021 01:22:10 +0200 Subject: [PATCH 05/11] Update requirements for bullseye --- requirements-frozen.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-frozen.txt b/requirements-frozen.txt index c5b4219..3f7fc62 100644 --- a/requirements-frozen.txt +++ b/requirements-frozen.txt @@ -20,6 +20,6 @@ sanic-routing==0.6.2 typing-extensions==3.10.0.0 ujson==4.0.2 urllib3==1.26.5 -uvloop==0.11.3 +uvloop==0.14.0 websockets==8.1 yarl==1.3.0 From 5572f746ef9ef735a854e3e8e96865c721e962b0 Mon Sep 17 00:00:00 2001 From: yalh76 Date: Wed, 30 Jun 2021 01:48:24 +0200 Subject: [PATCH 06/11] Fix AttributeError: type object '_asyncio.Task' has no attribute 'all_tasks' --- run.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/run.py b/run.py index 99f8a20..20cea41 100644 --- a/run.py +++ b/run.py @@ -40,6 +40,11 @@ from playhouse.shortcuts import model_to_dict from models import Repo, Job, db, Worker from schedule import always_relaunch, once_per_day +try: + asyncio_all_tasks = asyncio.all_tasks +except AttributeError as e: + asyncio_all_tasks = asyncio.Task.all_tasks + LOGGING_CONFIG_DEFAULTS["loggers"] = { "task": { "level": "INFO", @@ -971,7 +976,7 @@ async def html_index(request): @always_relaunch(sleep=2) async def number_of_tasks(): - print("Number of tasks: %s" % len(Task.all_tasks())) + print("Number of tasks: %s" % len(asyncio_all_tasks())) @app.route('/monitor') @@ -979,7 +984,7 @@ async def monitor(request): snapshot = tracemalloc.take_snapshot() top_stats = snapshot.statistics('lineno') - tasks = Task.all_tasks() + tasks = asyncio_all_tasks() return response.json({ "top_20_trace": [str(x) for x in top_stats[:20]], From 6508dcf51d094d3c748269d6a3fb1a1acc26537a Mon Sep 17 00:00:00 2001 From: yalh76 Date: Wed, 30 Jun 2021 02:13:08 +0200 Subject: [PATCH 07/11] Fix DeprecationWarning: "@coroutine" python[11090]: /var/www/yunorunner/./run.py:119: DeprecationWarning: "@coroutine" decorator is deprecated since Python 3.8, use "async def" instead python[11090]: def wait_closed(self): --- run.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/run.py b/run.py index 20cea41..77711c7 100644 --- a/run.py +++ b/run.py @@ -115,8 +115,7 @@ def datetime_to_epoch_json_converter(o): return o.strftime('%s') -@asyncio.coroutine -def wait_closed(self): +async def wait_closed(self): """ Wait until the connection is closed. @@ -126,7 +125,7 @@ def wait_closed(self): of its cause, in tasks that interact with the WebSocket connection. """ - yield from asyncio.shield(self.connection_lost_waiter) + await asyncio.shield(self.connection_lost_waiter) # this is a backport of websockets 7.0 which sanic doesn't support yet From dabfc174f4194048fa3b5196719a6da699aa8998 Mon Sep 17 00:00:00 2001 From: yalh76 Date: Wed, 30 Jun 2021 03:17:50 +0200 Subject: [PATCH 08/11] Update requirements.txt --- requirements.txt | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/requirements.txt b/requirements.txt index ff70ef1..39ad08a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,8 @@ -sanic -aiohttp -aiofiles -peewee -sanic-jinja2 -websockets - -# cli -argh -requests +Jinja2==2.11.3 +aiofiles==0.7.0 +aiohttp==3.7.4.post0 +argh==0.26.2 +peewee==3.14.4 +sanic==21.6.0 +sanic_jinja2==0.10.0 +websockets==9.1 From fe8789e4a6f7e552c4f78ec7871b5d2f7f5a9f00 Mon Sep 17 00:00:00 2001 From: yalh76 Date: Wed, 30 Jun 2021 03:58:48 +0200 Subject: [PATCH 09/11] Update requirements-frozen.txt --- requirements-frozen.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-frozen.txt b/requirements-frozen.txt index 3f7fc62..0271ead 100644 --- a/requirements-frozen.txt +++ b/requirements-frozen.txt @@ -21,5 +21,5 @@ typing-extensions==3.10.0.0 ujson==4.0.2 urllib3==1.26.5 uvloop==0.14.0 -websockets==8.1 +websockets==9.1 yarl==1.3.0 From 9cde6e979a59424396659fa934323a677ba8c960 Mon Sep 17 00:00:00 2001 From: yalh76 Date: Wed, 30 Jun 2021 04:19:53 +0200 Subject: [PATCH 10/11] Update requirements-frozen.txt --- requirements-frozen.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-frozen.txt b/requirements-frozen.txt index 0271ead..3f7fc62 100644 --- a/requirements-frozen.txt +++ b/requirements-frozen.txt @@ -21,5 +21,5 @@ typing-extensions==3.10.0.0 ujson==4.0.2 urllib3==1.26.5 uvloop==0.14.0 -websockets==9.1 +websockets==8.1 yarl==1.3.0 From 03ee99a8f72203d8f14986bc88cca45e36ac4895 Mon Sep 17 00:00:00 2001 From: yalh76 Date: Wed, 30 Jun 2021 05:10:10 +0200 Subject: [PATCH 11/11] Update requirements.txt --- requirements.txt | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/requirements.txt b/requirements.txt index 39ad08a..d424716 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,10 @@ -Jinja2==2.11.3 -aiofiles==0.7.0 -aiohttp==3.7.4.post0 -argh==0.26.2 -peewee==3.14.4 -sanic==21.6.0 -sanic_jinja2==0.10.0 -websockets==9.1 +sanic +aiohttp +aiofiles +peewee +sanic-jinja2 +websockets + +# cli +argh +requests \ No newline at end of file