276 lines
11 KiB
Python
276 lines
11 KiB
Python
#
|
|
# SPDX-License-Identifier: GPL-2.0-only
|
|
#
|
|
|
|
from django.core.management.base import BaseCommand
|
|
from django.db import transaction
|
|
from django.db.models import Q
|
|
|
|
from bldcontrol.bbcontroller import getBuildEnvironmentController
|
|
from bldcontrol.models import BuildRequest, BuildEnvironment
|
|
from bldcontrol.models import BRError, BRVariable
|
|
|
|
from orm.models import Build, LogMessage, Target
|
|
|
|
import logging
|
|
import traceback
|
|
import signal
|
|
import os
|
|
|
|
logger = logging.getLogger("toaster")
|
|
|
|
|
|
class Command(BaseCommand):
|
|
args = ""
|
|
help = "Schedules and executes build requests as possible. "\
|
|
"Does not return (interrupt with Ctrl-C)"
|
|
|
|
@transaction.atomic
|
|
def _selectBuildEnvironment(self):
|
|
bec = getBuildEnvironmentController(lock=BuildEnvironment.LOCK_FREE)
|
|
bec.be.lock = BuildEnvironment.LOCK_LOCK
|
|
bec.be.save()
|
|
return bec
|
|
|
|
@transaction.atomic
|
|
def _selectBuildRequest(self):
|
|
br = BuildRequest.objects.filter(state=BuildRequest.REQ_QUEUED).first()
|
|
return br
|
|
|
|
def schedule(self):
|
|
try:
|
|
# select the build environment and the request to build
|
|
br = self._selectBuildRequest()
|
|
if br:
|
|
br.state = BuildRequest.REQ_INPROGRESS
|
|
br.save()
|
|
else:
|
|
return
|
|
|
|
try:
|
|
bec = self._selectBuildEnvironment()
|
|
except IndexError as e:
|
|
# we could not find a BEC; postpone the BR
|
|
br.state = BuildRequest.REQ_QUEUED
|
|
br.save()
|
|
logger.debug("runbuilds: No build env (%s)" % e)
|
|
return
|
|
|
|
logger.info("runbuilds: starting build %s, environment %s" %
|
|
(br, bec.be))
|
|
|
|
# let the build request know where it is being executed
|
|
br.environment = bec.be
|
|
br.save()
|
|
|
|
# this triggers an async build
|
|
bec.triggerBuild(br.brbitbake, br.brlayer_set.all(),
|
|
br.brvariable_set.all(), br.brtarget_set.all(),
|
|
"%d:%d" % (br.pk, bec.be.pk))
|
|
|
|
except Exception as e:
|
|
logger.error("runbuilds: Error launching build %s" % e)
|
|
traceback.print_exc()
|
|
if "[Errno 111] Connection refused" in str(e):
|
|
# Connection refused, read toaster_server.out
|
|
errmsg = bec.readServerLogFile()
|
|
else:
|
|
errmsg = str(e)
|
|
|
|
BRError.objects.create(req=br, errtype=str(type(e)), errmsg=errmsg,
|
|
traceback=traceback.format_exc())
|
|
br.state = BuildRequest.REQ_FAILED
|
|
br.save()
|
|
bec.be.lock = BuildEnvironment.LOCK_FREE
|
|
bec.be.save()
|
|
# Cancel the pending build and report the exception to the UI
|
|
log_object = LogMessage.objects.create(
|
|
build = br.build,
|
|
level = LogMessage.EXCEPTION,
|
|
message = errmsg)
|
|
log_object.save()
|
|
br.build.outcome = Build.FAILED
|
|
br.build.save()
|
|
|
|
def archive(self):
|
|
for br in BuildRequest.objects.filter(state=BuildRequest.REQ_ARCHIVE):
|
|
if br.build is None:
|
|
br.state = BuildRequest.REQ_FAILED
|
|
else:
|
|
br.state = BuildRequest.REQ_COMPLETED
|
|
br.save()
|
|
|
|
def cleanup(self):
|
|
from django.utils import timezone
|
|
from datetime import timedelta
|
|
# environments locked for more than 30 seconds
|
|
# they should be unlocked
|
|
BuildEnvironment.objects.filter(
|
|
Q(buildrequest__state__in=[BuildRequest.REQ_FAILED,
|
|
BuildRequest.REQ_COMPLETED,
|
|
BuildRequest.REQ_CANCELLING]) &
|
|
Q(lock=BuildEnvironment.LOCK_LOCK) &
|
|
Q(updated__lt=timezone.now() - timedelta(seconds=30))
|
|
).update(lock=BuildEnvironment.LOCK_FREE)
|
|
|
|
# update all Builds that were in progress and failed to start
|
|
for br in BuildRequest.objects.filter(
|
|
state=BuildRequest.REQ_FAILED,
|
|
build__outcome=Build.IN_PROGRESS):
|
|
# transpose the launch errors in ToasterExceptions
|
|
br.build.outcome = Build.FAILED
|
|
for brerror in br.brerror_set.all():
|
|
logger.debug("Saving error %s" % brerror)
|
|
LogMessage.objects.create(build=br.build,
|
|
level=LogMessage.EXCEPTION,
|
|
message=brerror.errmsg)
|
|
br.build.save()
|
|
|
|
# we don't have a true build object here; hence, toasterui
|
|
# didn't have a change to release the BE lock
|
|
br.environment.lock = BuildEnvironment.LOCK_FREE
|
|
br.environment.save()
|
|
|
|
# update all BuildRequests without a build created
|
|
for br in BuildRequest.objects.filter(build=None):
|
|
br.build = Build.objects.create(project=br.project,
|
|
completed_on=br.updated,
|
|
started_on=br.created)
|
|
br.build.outcome = Build.FAILED
|
|
try:
|
|
br.build.machine = br.brvariable_set.get(name='MACHINE').value
|
|
except BRVariable.DoesNotExist:
|
|
pass
|
|
br.save()
|
|
# transpose target information
|
|
for brtarget in br.brtarget_set.all():
|
|
Target.objects.create(build=br.build,
|
|
target=brtarget.target,
|
|
task=brtarget.task)
|
|
# transpose the launch errors in ToasterExceptions
|
|
for brerror in br.brerror_set.all():
|
|
LogMessage.objects.create(build=br.build,
|
|
level=LogMessage.EXCEPTION,
|
|
message=brerror.errmsg)
|
|
|
|
br.build.save()
|
|
|
|
# Make sure the LOCK is removed for builds which have been fully
|
|
# cancelled
|
|
for br in BuildRequest.objects.filter(
|
|
Q(build__outcome=Build.CANCELLED) &
|
|
Q(state=BuildRequest.REQ_CANCELLING) &
|
|
~Q(environment=None)):
|
|
br.environment.lock = BuildEnvironment.LOCK_FREE
|
|
br.environment.save()
|
|
|
|
def runbuild(self):
|
|
try:
|
|
self.cleanup()
|
|
except Exception as e:
|
|
logger.warning("runbuilds: cleanup exception %s" % str(e))
|
|
|
|
try:
|
|
self.archive()
|
|
except Exception as e:
|
|
logger.warning("runbuilds: archive exception %s" % str(e))
|
|
|
|
try:
|
|
self.schedule()
|
|
except Exception as e:
|
|
logger.warning("runbuilds: schedule exception %s" % str(e))
|
|
|
|
# Test to see if a build pre-maturely died due to a bitbake crash
|
|
def check_dead_builds(self):
|
|
do_cleanup = False
|
|
try:
|
|
for br in BuildRequest.objects.filter(state=BuildRequest.REQ_INPROGRESS):
|
|
# Get the build directory
|
|
if br.project.builddir:
|
|
builddir = br.project.builddir
|
|
else:
|
|
builddir = '%s-toaster-%d' % (br.environment.builddir,br.project.id)
|
|
# Check log to see if there is a recent traceback
|
|
toaster_ui_log = os.path.join(builddir, 'toaster_ui.log')
|
|
test_file = os.path.join(builddir, '._toaster_check.txt')
|
|
os.system("tail -n 50 %s > %s" % (os.path.join(builddir, 'toaster_ui.log'),test_file))
|
|
traceback_text = ''
|
|
is_traceback = False
|
|
with open(test_file,'r') as test_file_fd:
|
|
test_file_tail = test_file_fd.readlines()
|
|
for line in test_file_tail:
|
|
if line.startswith('Traceback (most recent call last):'):
|
|
traceback_text = line
|
|
is_traceback = True
|
|
elif line.startswith('NOTE: ToasterUI waiting for events'):
|
|
# Ignore any traceback before new build start
|
|
traceback_text = ''
|
|
is_traceback = False
|
|
elif line.startswith('Note: Toaster traceback auto-stop'):
|
|
# Ignore any traceback before this previous traceback catch
|
|
traceback_text = ''
|
|
is_traceback = False
|
|
elif is_traceback:
|
|
traceback_text += line
|
|
# Test the results
|
|
is_stop = False
|
|
if is_traceback:
|
|
# Found a traceback
|
|
errtype = 'Bitbake crash'
|
|
errmsg = 'Bitbake crash\n' + traceback_text
|
|
state = BuildRequest.REQ_FAILED
|
|
# Clean up bitbake files
|
|
bitbake_lock = os.path.join(builddir, 'bitbake.lock')
|
|
if os.path.isfile(bitbake_lock):
|
|
os.remove(bitbake_lock)
|
|
bitbake_sock = os.path.join(builddir, 'bitbake.sock')
|
|
if os.path.isfile(bitbake_sock):
|
|
os.remove(bitbake_sock)
|
|
if os.path.isfile(test_file):
|
|
os.remove(test_file)
|
|
# Add note to ignore this traceback on next check
|
|
os.system('echo "Note: Toaster traceback auto-stop" >> %s' % toaster_ui_log)
|
|
is_stop = True
|
|
# Add more tests here
|
|
#elif ...
|
|
# Stop the build request?
|
|
if is_stop:
|
|
brerror = BRError(
|
|
req = br,
|
|
errtype = errtype,
|
|
errmsg = errmsg,
|
|
traceback = traceback_text,
|
|
)
|
|
brerror.save()
|
|
br.state = state
|
|
br.save()
|
|
do_cleanup = True
|
|
# Do cleanup
|
|
if do_cleanup:
|
|
self.cleanup()
|
|
except Exception as e:
|
|
logger.error("runbuilds: Error in check_dead_builds %s" % e)
|
|
|
|
def handle(self, **options):
|
|
pidfile_path = os.path.join(os.environ.get("BUILDDIR", "."),
|
|
".runbuilds.pid")
|
|
|
|
with open(pidfile_path, 'w') as pidfile:
|
|
pidfile.write("%s" % os.getpid())
|
|
|
|
# Clean up any stale/failed builds from previous Toaster run
|
|
self.runbuild()
|
|
|
|
signal.signal(signal.SIGUSR1, lambda sig, frame: None)
|
|
|
|
while True:
|
|
sigset = signal.sigtimedwait([signal.SIGUSR1], 5)
|
|
if sigset:
|
|
for sig in sigset:
|
|
# Consume each captured pending event
|
|
self.runbuild()
|
|
else:
|
|
# Check for build exceptions
|
|
self.check_dead_builds()
|
|
|