[enh] don't fail on job error and display job as error

models.py

@@ -29,6 +29,7 @@ class Job(peewee.Model):
         ('runnning', 'Running'),
         ('done', 'Done'),
         ('failure', 'Failure'),
+        ('error', 'Error'),
         ('canceled', 'Canceled'),
     ), default="scheduled")
 

run.py

@@ -6,6 +6,7 @@ import argh
 import random
 import logging
 import asyncio
+import traceback
 import itertools
 
 from datetime import datetime, date
@@ -314,6 +315,7 @@ async def run_job(worker, job):
     cwd = os.path.split(path_to_analyseCI)[0]
     arguments = f' {job.url_or_path} "{job.name}"'
     task_logger.info(f"Launch command: /bin/bash " + path_to_analyseCI + arguments)
+    try:
         command = await asyncio.create_subprocess_shell("/bin/bash " + path_to_analyseCI + arguments,
                                                         cwd=cwd,
                                                         # default limit is not enough in some situations
@@ -336,6 +338,17 @@ async def run_job(worker, job):
                 "data": model_to_dict(job),
             }, ["jobs", f"job-{job.id}"])
 
+    except Exception as e:
+        traceback.print_exc()
+        task_logger.exception(f"ERROR in job '{job.name} #{job.id}'")
+
+        job.end_time = datetime.now()
+        job.state = "error"
+        job.save()
+
+        # XXX add mechanism to reschedule error jobs
+
+    else:
         task_logger.info(f"Finished job '{job.name}'")
 
         await command.wait()
@@ -384,7 +397,7 @@ async def ws_index(request, websocket):
 
     JobAlias = Job.alias()
     subquery = JobAlias.select()\
-                       .where(JobAlias.state << ("done", "failure", "canceled"))\
+                       .where(JobAlias.state << ("done", "failure", "canceled", "error"))\
                        .group_by(JobAlias.url_or_path)\
                        .select(fn.Max(JobAlias.id).alias("max_id"))
 
@@ -558,7 +571,7 @@ async def api_stop_job(request, job_id):
 
         return response.text("ok")
 
-    if job.state in ("done", "canceled", "failure"):
+    if job.state in ("done", "canceled", "failure", "error"):
         api_logger.info(f"Request to cancel job '{job.name}' [job.id] but job is already in '{job.state}' state, do nothing")
         # nothing to do, task is already done
         return response.text("ok")