Package coprs :: Package logic :: Module builds_logic
[hide private]
[frames] | no frames]

Source Code for Module coprs.logic.builds_logic

   1  import tempfile 
   2  import shutil 
   3  import json 
   4  import os 
   5  import pprint 
   6  import time 
   7  import requests 
   8   
   9  from sqlalchemy.sql import text 
  10  from sqlalchemy import or_ 
  11  from sqlalchemy import and_ 
  12  from sqlalchemy import func, desc 
  13  from sqlalchemy.sql import false,true 
  14  from werkzeug.utils import secure_filename 
  15  from sqlalchemy import bindparam, Integer, String 
  16  from sqlalchemy.exc import IntegrityError 
  17   
  18  from copr_common.enums import FailTypeEnum, StatusEnum 
  19  from coprs import app 
  20  from coprs import db 
  21  from coprs import models 
  22  from coprs import helpers 
  23  from coprs.constants import DEFAULT_BUILD_TIMEOUT, MAX_BUILD_TIMEOUT 
  24  from coprs.exceptions import MalformedArgumentException, ActionInProgressException, InsufficientRightsException, \ 
  25                               UnrepeatableBuildException, RequestCannotBeExecuted, DuplicateException 
  26   
  27  from coprs.logic import coprs_logic 
  28  from coprs.logic import users_logic 
  29  from coprs.logic.actions_logic import ActionsLogic 
  30  from coprs.models import BuildChroot 
  31  from .coprs_logic import MockChrootsLogic 
  32  from coprs.logic.packages_logic import PackagesLogic 
  33   
  34  log = app.logger 
35 36 37 -class BuildsLogic(object):
38 @classmethod
39 - def get(cls, build_id):
40 return models.Build.query.filter(models.Build.id == build_id)
41 42 @classmethod
43 - def get_build_tasks(cls, status, background=None):
44 """ Returns tasks with given status. If background is specified then 45 returns normal jobs (false) or background jobs (true) 46 """ 47 result = models.BuildChroot.query.join(models.Build)\ 48 .filter(models.BuildChroot.status == status)\ 49 .order_by(models.Build.id.asc()) 50 if background is not None: 51 result = result.filter(models.Build.is_background == (true() if background else false())) 52 return result
53 54 @classmethod
55 - def get_srpm_build_tasks(cls, status, background=None):
56 """ Returns srpm build tasks with given status. If background is 57 specified then returns normal jobs (false) or background jobs (true) 58 """ 59 result = models.Build.query\ 60 .filter(models.Build.source_status == status)\ 61 .order_by(models.Build.id.asc()) 62 if background is not None: 63 result = result.filter(models.Build.is_background == (true() if background else false())) 64 return result
65 66 @classmethod
67 - def get_recent_tasks(cls, user=None, limit=None):
68 if not limit: 69 limit = 100 70 71 query = models.Build.query 72 if user is not None: 73 query = query.filter(models.Build.user_id == user.id) 74 75 query = query.join( 76 models.BuildChroot.query 77 .filter(models.BuildChroot.ended_on.isnot(None)) 78 .order_by(models.BuildChroot.ended_on.desc()) 79 .subquery() 80 ).order_by(models.Build.id.desc()) 81 82 # Workaround - otherwise it could take less records than `limit` even though there are more of them. 83 query = query.limit(limit if limit > 100 else 100) 84 return list(query.all()[:4])
85 86 @classmethod
87 - def get_running_tasks_by_time(cls, start, end):
88 result = models.BuildChroot.query\ 89 .filter(models.BuildChroot.ended_on > start)\ 90 .filter(models.BuildChroot.started_on < end)\ 91 .order_by(models.BuildChroot.started_on.asc()) 92 93 return result
94 95 @classmethod
96 - def get_chroot_histogram(cls, start, end):
97 chroots = [] 98 chroot_query = BuildChroot.query\ 99 .filter(models.BuildChroot.started_on < end)\ 100 .filter(models.BuildChroot.ended_on > start)\ 101 .with_entities(BuildChroot.mock_chroot_id, 102 func.count(BuildChroot.mock_chroot_id))\ 103 .group_by(BuildChroot.mock_chroot_id)\ 104 .order_by(BuildChroot.mock_chroot_id) 105 106 for chroot in chroot_query: 107 chroots.append([chroot[0], chroot[1]]) 108 109 mock_chroots = coprs_logic.MockChrootsLogic.get_multiple() 110 for mock_chroot in mock_chroots: 111 for l in chroots: 112 if l[0] == mock_chroot.id: 113 l[0] = mock_chroot.name 114 115 return chroots
116 117 @classmethod
118 - def get_pending_jobs_bucket(cls, start, end):
119 query = text(""" 120 SELECT COUNT(*) as result 121 FROM build_chroot JOIN build on build.id = build_chroot.build_id 122 WHERE 123 build.submitted_on < :end 124 AND ( 125 build_chroot.started_on > :start 126 OR (build_chroot.started_on is NULL AND build_chroot.status = :status) 127 -- for currently pending builds we need to filter on status=pending because there might be 128 -- failed builds that have started_on=NULL 129 ) 130 AND NOT build.canceled 131 """) 132 133 res = db.engine.execute(query, start=start, end=end, status=StatusEnum("pending")) 134 return res.first().result
135 136 @classmethod
137 - def get_running_jobs_bucket(cls, start, end):
138 query = text(""" 139 SELECT COUNT(*) as result 140 FROM build_chroot 141 WHERE 142 started_on < :end 143 AND (ended_on > :start OR (ended_on is NULL AND status = :status)) 144 -- for currently running builds we need to filter on status=running because there might be failed 145 -- builds that have ended_on=NULL 146 """) 147 148 res = db.engine.execute(query, start=start, end=end, status=StatusEnum("running")) 149 return res.first().result
150 151 @classmethod
152 - def get_cached_graph_data(cls, params):
153 data = { 154 "pending": [], 155 "running": [], 156 } 157 result = models.BuildsStatistics.query\ 158 .filter(models.BuildsStatistics.stat_type == params["type"])\ 159 .filter(models.BuildsStatistics.time >= params["start"])\ 160 .filter(models.BuildsStatistics.time <= params["end"])\ 161 .order_by(models.BuildsStatistics.time) 162 163 for row in result: 164 data["pending"].append(row.pending) 165 data["running"].append(row.running) 166 167 return data
168 169 @classmethod
170 - def get_task_graph_data(cls, type):
171 data = [["pending"], ["running"], ["avg running"], ["time"]] 172 params = cls.get_graph_parameters(type) 173 cached_data = cls.get_cached_graph_data(params) 174 data[0].extend(cached_data["pending"]) 175 data[1].extend(cached_data["running"]) 176 177 for i in range(len(data[0]) - 1, params["steps"]): 178 step_start = params["start"] + i * params["step"] 179 step_end = step_start + params["step"] 180 pending = cls.get_pending_jobs_bucket(step_start, step_end) 181 running = cls.get_running_jobs_bucket(step_start, step_end) 182 data[0].append(pending) 183 data[1].append(running) 184 cls.cache_graph_data(type, time=step_start, pending=pending, running=running) 185 186 running_total = 0 187 for i in range(1, params["steps"] + 1): 188 running_total += data[1][i] 189 190 data[2].extend([running_total * 1.0 / params["steps"]] * (len(data[0]) - 1)) 191 192 for i in range(params["start"], params["end"], params["step"]): 193 data[3].append(time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(i))) 194 195 return data
196 197 @classmethod
198 - def get_small_graph_data(cls, type):
199 data = [[""]] 200 params = cls.get_graph_parameters(type) 201 cached_data = cls.get_cached_graph_data(params) 202 data[0].extend(cached_data["running"]) 203 204 for i in range(len(data[0]) - 1, params["steps"]): 205 step_start = params["start"] + i * params["step"] 206 step_end = step_start + params["step"] 207 running = cls.get_running_jobs_bucket(step_start, step_end) 208 data[0].append(running) 209 cls.cache_graph_data(type, time=step_start, running=running) 210 211 return data
212 213 @classmethod
214 - def cache_graph_data(cls, type, time, pending=0, running=0):
215 result = models.BuildsStatistics.query\ 216 .filter(models.BuildsStatistics.stat_type == type)\ 217 .filter(models.BuildsStatistics.time == time).first() 218 if result: 219 return 220 221 try: 222 cached_data = models.BuildsStatistics( 223 time = time, 224 stat_type = type, 225 running = running, 226 pending = pending 227 ) 228 db.session.add(cached_data) 229 db.session.commit() 230 except IntegrityError: # other process already calculated the graph data and cached it 231 db.session.rollback()
232 233 @classmethod
234 - def get_graph_parameters(cls, type):
235 if type is "10min": 236 # 24 hours with 10 minute intervals 237 step = 600 238 steps = 144 239 elif type is "30min": 240 # 24 hours with 30 minute intervals 241 step = 1800 242 steps = 48 243 elif type is "24h": 244 # 90 days with 24 hour intervals 245 step = 86400 246 steps = 90 247 248 end = int(time.time()) 249 end = end - (end % step) # align graph interval to a multiple of step 250 start = end - (steps * step) 251 252 return { 253 "type": type, 254 "step": step, 255 "steps": steps, 256 "start": start, 257 "end": end, 258 }
259 260 @classmethod
261 - def get_build_importing_queue(cls, background=None):
262 """ 263 Returns Builds which are waiting to be uploaded to dist git 264 """ 265 query = (models.Build.query 266 .filter(models.Build.canceled == false()) 267 .filter(models.Build.source_status == StatusEnum("importing")) 268 .order_by(models.Build.id.asc())) 269 if background is not None: 270 query = query.filter(models.Build.is_background == (true() if background else false())) 271 return query
272 273 @classmethod
274 - def get_pending_srpm_build_tasks(cls, background=None):
275 query = (models.Build.query 276 .filter(models.Build.canceled == false()) 277 .filter(models.Build.source_status == StatusEnum("pending")) 278 .order_by(models.Build.is_background.asc(), models.Build.id.asc())) 279 if background is not None: 280 query = query.filter(models.Build.is_background == (true() if background else false())) 281 return query
282 283 @classmethod
284 - def get_pending_build_tasks(cls, background=None):
285 query = (models.BuildChroot.query.join(models.Build) 286 .filter(models.Build.canceled == false()) 287 .filter(or_( 288 models.BuildChroot.status == StatusEnum("pending"), 289 and_( 290 models.BuildChroot.status == StatusEnum("running"), 291 models.BuildChroot.started_on < int(time.time() - 1.1 * MAX_BUILD_TIMEOUT), 292 models.BuildChroot.ended_on.is_(None) 293 ) 294 )) 295 .order_by(models.Build.is_background.asc(), models.Build.id.asc())) 296 if background is not None: 297 query = query.filter(models.Build.is_background == (true() if background else false())) 298 return query
299 300 @classmethod
301 - def get_build_task(cls, task_id):
302 try: 303 build_id, chroot_name = task_id.split("-", 1) 304 except ValueError: 305 raise MalformedArgumentException("Invalid task_id {}".format(task_id)) 306 307 build_chroot = BuildChrootsLogic.get_by_build_id_and_name(build_id, chroot_name) 308 return build_chroot.join(models.Build).first()
309 310 @classmethod
311 - def get_srpm_build_task(cls, build_id):
312 return BuildsLogic.get_by_id(build_id).first()
313 314 @classmethod
315 - def get_multiple(cls):
316 return models.Build.query.order_by(models.Build.id.desc())
317 318 @classmethod
319 - def get_multiple_by_copr(cls, copr):
320 """ Get collection of builds in copr sorted by build_id descending 321 """ 322 return cls.get_multiple().filter(models.Build.copr == copr)
323 324 @classmethod
325 - def get_multiple_by_user(cls, user):
326 """ Get collection of builds in copr sorted by build_id descending 327 form the copr belonging to `user` 328 """ 329 return cls.get_multiple().join(models.Build.copr).filter( 330 models.Copr.user == user)
331 332 @classmethod
333 - def init_db(cls):
334 if db.engine.url.drivername == "sqlite": 335 return 336 337 status_to_order = """ 338 CREATE OR REPLACE FUNCTION status_to_order (x integer) 339 RETURNS integer AS $$ BEGIN 340 RETURN CASE WHEN x = 3 THEN 1 341 WHEN x = 6 THEN 2 342 WHEN x = 7 THEN 3 343 WHEN x = 4 THEN 4 344 WHEN x = 0 THEN 5 345 WHEN x = 1 THEN 6 346 WHEN x = 5 THEN 7 347 WHEN x = 2 THEN 8 348 WHEN x = 8 THEN 9 349 WHEN x = 9 THEN 10 350 ELSE x 351 END; END; 352 $$ LANGUAGE plpgsql; 353 """ 354 355 order_to_status = """ 356 CREATE OR REPLACE FUNCTION order_to_status (x integer) 357 RETURNS integer AS $$ BEGIN 358 RETURN CASE WHEN x = 1 THEN 3 359 WHEN x = 2 THEN 6 360 WHEN x = 3 THEN 7 361 WHEN x = 4 THEN 4 362 WHEN x = 5 THEN 0 363 WHEN x = 6 THEN 1 364 WHEN x = 7 THEN 5 365 WHEN x = 8 THEN 2 366 WHEN x = 9 THEN 8 367 WHEN x = 10 THEN 9 368 ELSE x 369 END; END; 370 $$ LANGUAGE plpgsql; 371 """ 372 373 db.engine.connect() 374 db.engine.execute(status_to_order) 375 db.engine.execute(order_to_status)
376 377 @classmethod
378 - def get_copr_builds_list(cls, copr, dirname=''):
379 query_select = """ 380 SELECT build.id, build.source_status, MAX(package.name) AS pkg_name, build.pkg_version, build.submitted_on, 381 MIN(statuses.started_on) AS started_on, MAX(statuses.ended_on) AS ended_on, order_to_status(MIN(statuses.st)) AS status, 382 build.canceled, MIN("group".name) AS group_name, MIN(copr.name) as copr_name, MIN("user".username) as user_name, build.copr_id 383 FROM build 384 LEFT OUTER JOIN package 385 ON build.package_id = package.id 386 LEFT OUTER JOIN (SELECT build_chroot.build_id, started_on, ended_on, status_to_order(status) AS st FROM build_chroot) AS statuses 387 ON statuses.build_id=build.id 388 LEFT OUTER JOIN copr 389 ON copr.id = build.copr_id 390 LEFT OUTER JOIN copr_dir 391 ON build.copr_dir_id = copr_dir.id 392 LEFT OUTER JOIN "user" 393 ON copr.user_id = "user".id 394 LEFT OUTER JOIN "group" 395 ON copr.group_id = "group".id 396 WHERE build.copr_id = :copr_id 397 AND (:dirname = '' OR :dirname = copr_dir.name) 398 GROUP BY 399 build.id 400 ORDER BY 401 build.id DESC; 402 """ 403 404 if db.engine.url.drivername == "sqlite": 405 def sqlite_status_to_order(x): 406 if x == 3: 407 return 1 408 elif x == 6: 409 return 2 410 elif x == 7: 411 return 3 412 elif x == 4: 413 return 4 414 elif x == 0: 415 return 5 416 elif x == 1: 417 return 6 418 elif x == 5: 419 return 7 420 elif x == 2: 421 return 8 422 elif x == 8: 423 return 9 424 elif x == 9: 425 return 10 426 return 1000
427 428 def sqlite_order_to_status(x): 429 if x == 1: 430 return 3 431 elif x == 2: 432 return 6 433 elif x == 3: 434 return 7 435 elif x == 4: 436 return 4 437 elif x == 5: 438 return 0 439 elif x == 6: 440 return 1 441 elif x == 7: 442 return 5 443 elif x == 8: 444 return 2 445 elif x == 9: 446 return 8 447 elif x == 10: 448 return 9 449 return 1000
450 451 conn = db.engine.connect() 452 conn.connection.create_function("status_to_order", 1, sqlite_status_to_order) 453 conn.connection.create_function("order_to_status", 1, sqlite_order_to_status) 454 statement = text(query_select) 455 statement.bindparams(bindparam("copr_id", Integer)) 456 statement.bindparams(bindparam("dirname", String)) 457 result = conn.execute(statement, {"copr_id": copr.id, "dirname": dirname}) 458 else: 459 statement = text(query_select) 460 statement.bindparams(bindparam("copr_id", Integer)) 461 statement.bindparams(bindparam("dirname", String)) 462 result = db.engine.execute(statement, {"copr_id": copr.id, "dirname": dirname}) 463 464 return result 465 466 @classmethod
467 - def join_group(cls, query):
468 return query.join(models.Copr).outerjoin(models.Group)
469 470 @classmethod
471 - def get_multiple_by_name(cls, username, coprname):
472 query = cls.get_multiple() 473 return (query.join(models.Build.copr) 474 .options(db.contains_eager(models.Build.copr)) 475 .join(models.Copr.user) 476 .filter(models.Copr.name == coprname) 477 .filter(models.User.username == username))
478 479 @classmethod
480 - def get_by_ids(cls, ids):
481 return models.Build.query.filter(models.Build.id.in_(ids))
482 483 @classmethod
484 - def get_by_id(cls, build_id):
485 return models.Build.query.filter(models.Build.id == build_id)
486 487 @classmethod
488 - def create_new_from_other_build(cls, user, copr, source_build, 489 chroot_names=None, **build_options):
490 skip_import = False 491 git_hashes = {} 492 493 if source_build.source_type == helpers.BuildSourceEnum('upload'): 494 if source_build.repeatable: 495 skip_import = True 496 for chroot in source_build.build_chroots: 497 git_hashes[chroot.name] = chroot.git_hash 498 else: 499 raise UnrepeatableBuildException("Build sources were not fully imported into CoprDistGit.") 500 501 build = cls.create_new(user, copr, source_build.source_type, source_build.source_json, chroot_names, 502 pkgs=source_build.pkgs, git_hashes=git_hashes, skip_import=skip_import, 503 srpm_url=source_build.srpm_url, copr_dirname=source_build.copr_dir.name, **build_options) 504 build.package_id = source_build.package_id 505 build.pkg_version = source_build.pkg_version 506 return build
507 508 @classmethod
509 - def create_new_from_url(cls, user, copr, url, chroot_names=None, 510 copr_dirname=None, **build_options):
511 """ 512 :type user: models.User 513 :type copr: models.Copr 514 515 :type chroot_names: List[str] 516 517 :rtype: models.Build 518 """ 519 source_type = helpers.BuildSourceEnum("link") 520 source_json = json.dumps({"url": url}) 521 srpm_url = None if url.endswith('.spec') else url 522 return cls.create_new(user, copr, source_type, source_json, chroot_names, 523 pkgs=url, srpm_url=srpm_url, copr_dirname=copr_dirname, **build_options)
524 525 @classmethod
526 - def create_new_from_scm(cls, user, copr, scm_type, clone_url, 527 committish='', subdirectory='', spec='', srpm_build_method='rpkg', 528 chroot_names=None, copr_dirname=None, **build_options):
529 """ 530 :type user: models.User 531 :type copr: models.Copr 532 533 :type chroot_names: List[str] 534 535 :rtype: models.Build 536 """ 537 source_type = helpers.BuildSourceEnum("scm") 538 source_json = json.dumps({"type": scm_type, 539 "clone_url": clone_url, 540 "committish": committish, 541 "subdirectory": subdirectory, 542 "spec": spec, 543 "srpm_build_method": srpm_build_method}) 544 return cls.create_new(user, copr, source_type, source_json, chroot_names, copr_dirname=copr_dirname, **build_options)
545 546 @classmethod
547 - def create_new_from_pypi(cls, user, copr, pypi_package_name, pypi_package_version, spec_template, 548 python_versions, chroot_names=None, copr_dirname=None, **build_options):
549 """ 550 :type user: models.User 551 :type copr: models.Copr 552 :type package_name: str 553 :type version: str 554 :type python_versions: List[str] 555 556 :type chroot_names: List[str] 557 558 :rtype: models.Build 559 """ 560 source_type = helpers.BuildSourceEnum("pypi") 561 source_json = json.dumps({"pypi_package_name": pypi_package_name, 562 "pypi_package_version": pypi_package_version, 563 "spec_template": spec_template, 564 "python_versions": python_versions}) 565 return cls.create_new(user, copr, source_type, source_json, chroot_names, copr_dirname=copr_dirname, **build_options)
566 567 @classmethod
568 - def create_new_from_rubygems(cls, user, copr, gem_name, chroot_names=None, 569 copr_dirname=None, **build_options):
570 """ 571 :type user: models.User 572 :type copr: models.Copr 573 :type gem_name: str 574 :type chroot_names: List[str] 575 :rtype: models.Build 576 """ 577 source_type = helpers.BuildSourceEnum("rubygems") 578 source_json = json.dumps({"gem_name": gem_name}) 579 return cls.create_new(user, copr, source_type, source_json, chroot_names, copr_dirname=copr_dirname, **build_options)
580 581 @classmethod
582 - def create_new_from_custom(cls, user, copr, script, script_chroot=None, script_builddeps=None, 583 script_resultdir=None, chroot_names=None, copr_dirname=None, **kwargs):
584 """ 585 :type user: models.User 586 :type copr: models.Copr 587 :type script: str 588 :type script_chroot: str 589 :type script_builddeps: str 590 :type script_resultdir: str 591 :type chroot_names: List[str] 592 :rtype: models.Build 593 """ 594 source_type = helpers.BuildSourceEnum("custom") 595 source_dict = { 596 'script': script, 597 'chroot': script_chroot, 598 'builddeps': script_builddeps, 599 'resultdir': script_resultdir, 600 } 601 602 return cls.create_new(user, copr, source_type, json.dumps(source_dict), 603 chroot_names, copr_dirname=copr_dirname, **kwargs)
604 605 @classmethod
606 - def create_new_from_upload(cls, user, copr, f_uploader, orig_filename, 607 chroot_names=None, copr_dirname=None, **build_options):
608 """ 609 :type user: models.User 610 :type copr: models.Copr 611 :param f_uploader(file_path): function which stores data at the given `file_path` 612 :return: 613 """ 614 tmp = tempfile.mkdtemp(dir=app.config["STORAGE_DIR"]) 615 tmp_name = os.path.basename(tmp) 616 filename = secure_filename(orig_filename) 617 file_path = os.path.join(tmp, filename) 618 f_uploader(file_path) 619 620 # make the pkg public 621 pkg_url = "{baseurl}/tmp/{tmp_dir}/{filename}".format( 622 baseurl=app.config["PUBLIC_COPR_BASE_URL"], 623 tmp_dir=tmp_name, 624 filename=filename) 625 626 # create json describing the build source 627 source_type = helpers.BuildSourceEnum("upload") 628 source_json = json.dumps({"url": pkg_url, "pkg": filename, "tmp": tmp_name}) 629 srpm_url = None if pkg_url.endswith('.spec') else pkg_url 630 631 try: 632 build = cls.create_new(user, copr, source_type, source_json, 633 chroot_names, pkgs=pkg_url, srpm_url=srpm_url, 634 copr_dirname=copr_dirname, **build_options) 635 except Exception: 636 shutil.rmtree(tmp) # todo: maybe we should delete in some cleanup procedure? 637 raise 638 639 return build
640 641 @classmethod
642 - def create_new(cls, user, copr, source_type, source_json, chroot_names=None, pkgs="", 643 git_hashes=None, skip_import=False, background=False, batch=None, 644 srpm_url=None, copr_dirname=None, **build_options):
645 """ 646 :type user: models.User 647 :type copr: models.Copr 648 :type chroot_names: List[str] 649 :type source_type: int value from helpers.BuildSourceEnum 650 :type source_json: str in json format 651 :type pkgs: str 652 :type git_hashes: dict 653 :type skip_import: bool 654 :type background: bool 655 :type batch: models.Batch 656 :rtype: models.Build 657 """ 658 chroots = None 659 if chroot_names: 660 chroots = [] 661 for chroot in copr.active_chroots: 662 if chroot.name in chroot_names: 663 chroots.append(chroot) 664 665 build = cls.add( 666 user=user, 667 pkgs=pkgs, 668 copr=copr, 669 chroots=chroots, 670 source_type=source_type, 671 source_json=source_json, 672 enable_net=build_options.get("enable_net", copr.build_enable_net), 673 background=background, 674 git_hashes=git_hashes, 675 skip_import=skip_import, 676 batch=batch, 677 srpm_url=srpm_url, 678 copr_dirname=copr_dirname, 679 ) 680 681 if user.proven: 682 if "timeout" in build_options: 683 build.timeout = build_options["timeout"] 684 685 return build
686 687 @classmethod
688 - def add(cls, user, pkgs, copr, source_type=None, source_json=None, 689 repos=None, chroots=None, timeout=None, enable_net=True, 690 git_hashes=None, skip_import=False, background=False, batch=None, 691 srpm_url=None, copr_dirname=None):
692 693 if chroots is None: 694 chroots = [] 695 696 coprs_logic.CoprsLogic.raise_if_unfinished_blocking_action( 697 copr, "Can't build while there is an operation in progress: {action}") 698 users_logic.UsersLogic.raise_if_cant_build_in_copr( 699 user, copr, 700 "You don't have permissions to build in this copr.") 701 702 if not repos: 703 repos = copr.repos 704 705 # todo: eliminate pkgs and this check 706 if pkgs and (" " in pkgs or "\n" in pkgs or "\t" in pkgs or pkgs.strip() != pkgs): 707 raise MalformedArgumentException("Trying to create a build using src_pkg " 708 "with bad characters. Forgot to split?") 709 710 # just temporary to keep compatibility 711 if not source_type or not source_json: 712 source_type = helpers.BuildSourceEnum("link") 713 source_json = json.dumps({"url":pkgs}) 714 715 if skip_import and srpm_url: 716 chroot_status = StatusEnum("pending") 717 source_status = StatusEnum("succeeded") 718 else: 719 chroot_status = StatusEnum("waiting") 720 source_status = StatusEnum("pending") 721 722 copr_dir = None 723 if copr_dirname: 724 if not copr_dirname.startswith(copr.name+':') and copr_dirname != copr.name: 725 raise MalformedArgumentException("Copr dirname not starting with copr name.") 726 copr_dir = coprs_logic.CoprDirsLogic.get_or_create(copr, copr_dirname) 727 728 build = models.Build( 729 user=user, 730 pkgs=pkgs, 731 copr=copr, 732 repos=repos, 733 source_type=source_type, 734 source_json=source_json, 735 source_status=source_status, 736 submitted_on=int(time.time()), 737 enable_net=bool(enable_net), 738 is_background=bool(background), 739 batch=batch, 740 srpm_url=srpm_url, 741 copr_dir=copr_dir, 742 ) 743 744 if timeout: 745 build.timeout = timeout or DEFAULT_BUILD_TIMEOUT 746 747 db.session.add(build) 748 749 for chroot in chroots: 750 # Chroots were explicitly set per-build. 751 git_hash = None 752 if git_hashes: 753 git_hash = git_hashes.get(chroot.name) 754 buildchroot = models.BuildChroot( 755 build=build, 756 status=chroot_status, 757 mock_chroot=chroot, 758 git_hash=git_hash, 759 ) 760 db.session.add(buildchroot) 761 762 return build
763 764 @classmethod
765 - def rebuild_package(cls, package, source_dict_update={}, copr_dir=None, update_callback=None, 766 scm_object_type=None, scm_object_id=None, scm_object_url=None):
767 768 source_dict = package.source_json_dict 769 source_dict.update(source_dict_update) 770 source_json = json.dumps(source_dict) 771 772 if not copr_dir: 773 copr_dir = package.copr.main_dir 774 775 build = models.Build( 776 user=None, 777 pkgs=None, 778 package=package, 779 copr=package.copr, 780 repos=package.copr.repos, 781 source_status=StatusEnum("pending"), 782 source_type=package.source_type, 783 source_json=source_json, 784 submitted_on=int(time.time()), 785 enable_net=package.copr.build_enable_net, 786 timeout=DEFAULT_BUILD_TIMEOUT, 787 copr_dir=copr_dir, 788 update_callback=update_callback, 789 scm_object_type=scm_object_type, 790 scm_object_id=scm_object_id, 791 scm_object_url=scm_object_url, 792 ) 793 db.session.add(build) 794 795 status = StatusEnum("waiting") 796 for chroot in package.chroots: 797 buildchroot = models.BuildChroot( 798 build=build, 799 status=status, 800 mock_chroot=chroot, 801 git_hash=None 802 ) 803 db.session.add(buildchroot) 804 805 cls.process_update_callback(build) 806 return build
807 808 809 terminal_states = {StatusEnum("failed"), StatusEnum("succeeded"), StatusEnum("canceled")} 810 811 @classmethod
812 - def get_buildchroots_by_build_id_and_branch(cls, build_id, branch):
813 """ 814 Returns a list of BuildChroots identified by build_id and dist-git 815 branch name. 816 """ 817 return ( 818 models.BuildChroot.query 819 .join(models.MockChroot) 820 .filter(models.BuildChroot.build_id==build_id) 821 .filter(models.MockChroot.distgit_branch_name==branch) 822 ).all()
823 824 825 @classmethod
826 - def delete_local_source(cls, build):
827 """ 828 Deletes the locally stored data for build purposes. This is typically 829 uploaded srpm file, uploaded spec file or webhook POST content. 830 """ 831 # is it hosted on the copr frontend? 832 data = json.loads(build.source_json) 833 if 'tmp' in data: 834 tmp = data["tmp"] 835 storage_path = app.config["STORAGE_DIR"] 836 try: 837 shutil.rmtree(os.path.join(storage_path, tmp)) 838 except: 839 pass
840 841 842 @classmethod
843 - def update_state_from_dict(cls, build, upd_dict):
844 """ 845 :param build: 846 :param upd_dict: 847 example: 848 { 849 "builds":[ 850 { 851 "id": 1, 852 "copr_id": 2, 853 "started_on": 1390866440 854 }, 855 { 856 "id": 2, 857 "copr_id": 1, 858 "status": 0, 859 "chroot": "fedora-18-x86_64", 860 "result_dir": "baz", 861 "ended_on": 1390866440 862 }] 863 } 864 """ 865 log.info("Updating build {} by: {}".format(build.id, upd_dict)) 866 867 # create the package if it doesn't exist 868 pkg_name = upd_dict.get('pkg_name', None) 869 if pkg_name: 870 if not PackagesLogic.get(build.copr_dir.id, pkg_name).first(): 871 try: 872 package = PackagesLogic.add( 873 build.copr.user, build.copr_dir, 874 pkg_name, build.source_type, build.source_json) 875 db.session.add(package) 876 db.session.commit() 877 except (IntegrityError, DuplicateException) as e: 878 app.logger.exception(e) 879 db.session.rollback() 880 return 881 build.package = PackagesLogic.get(build.copr_dir.id, pkg_name).first() 882 883 for attr in ["built_packages", "srpm_url", "pkg_version"]: 884 value = upd_dict.get(attr, None) 885 if value: 886 setattr(build, attr, value) 887 888 # update source build status 889 if str(upd_dict.get("task_id")) == str(build.task_id): 890 build.result_dir = upd_dict.get("result_dir", "") 891 892 new_status = upd_dict.get("status") 893 if new_status == StatusEnum("succeeded"): 894 new_status = StatusEnum("importing") 895 chroot_status=StatusEnum("waiting") 896 if not build.build_chroots: 897 # create the BuildChroots from Package setting, if not 898 # already set explicitly for concrete build 899 for chroot in build.package.chroots: 900 buildchroot = models.BuildChroot( 901 build=build, 902 status=chroot_status, 903 mock_chroot=chroot, 904 git_hash=None, 905 ) 906 db.session.add(buildchroot) 907 else: 908 for buildchroot in build.build_chroots: 909 buildchroot.status = chroot_status 910 db.session.add(buildchroot) 911 912 build.source_status = new_status 913 if new_status == StatusEnum("failed") or \ 914 new_status == StatusEnum("skipped"): 915 for ch in build.build_chroots: 916 ch.status = new_status 917 ch.ended_on = upd_dict.get("ended_on") or time.time() 918 db.session.add(ch) 919 920 if new_status == StatusEnum("failed"): 921 build.fail_type = FailTypeEnum("srpm_build_error") 922 923 cls.process_update_callback(build) 924 db.session.add(build) 925 return 926 927 if "chroot" in upd_dict: 928 # update respective chroot status 929 for build_chroot in build.build_chroots: 930 if build_chroot.name == upd_dict["chroot"]: 931 build_chroot.result_dir = upd_dict.get("result_dir", "") 932 933 if "status" in upd_dict and build_chroot.status not in BuildsLogic.terminal_states: 934 build_chroot.status = upd_dict["status"] 935 936 if upd_dict.get("status") in BuildsLogic.terminal_states: 937 build_chroot.ended_on = upd_dict.get("ended_on") or time.time() 938 939 if upd_dict.get("status") == StatusEnum("starting"): 940 build_chroot.started_on = upd_dict.get("started_on") or time.time() 941 942 db.session.add(build_chroot) 943 944 # If the last package of a module was successfully built, 945 # then send an action to create module repodata on backend 946 if (build.module 947 and upd_dict.get("status") == StatusEnum("succeeded") 948 and all(b.status == StatusEnum("succeeded") for b in build.module.builds)): 949 ActionsLogic.send_build_module(build.copr, build.module) 950 951 cls.process_update_callback(build) 952 db.session.add(build)
953 954 @classmethod
955 - def process_update_callback(cls, build):
956 parsed_git_url = helpers.get_parsed_git_url(build.copr.scm_repo_url) 957 if not parsed_git_url: 958 return 959 960 if build.update_callback == 'pagure_flag_pull_request': 961 api_url = 'https://{0}/api/0/{1}/pull-request/{2}/flag'.format( 962 parsed_git_url.netloc, parsed_git_url.path, build.scm_object_id) 963 return cls.pagure_flag(build, api_url) 964 965 elif build.update_callback == 'pagure_flag_commit': 966 api_url = 'https://{0}/api/0/{1}/c/{2}/flag'.format( 967 parsed_git_url.netloc, parsed_git_url.path, build.scm_object_id) 968 return cls.pagure_flag(build, api_url)
969 970 @classmethod
971 - def pagure_flag(cls, build, api_url):
972 headers = { 973 'Authorization': 'token {}'.format(build.copr.scm_api_auth.get('api_key')) 974 } 975 976 if build.srpm_url: 977 progress = 50 978 else: 979 progress = 10 980 981 state_table = { 982 'failed': ('failure', 0), 983 'succeeded': ('success', 100), 984 'canceled': ('canceled', 0), 985 'running': ('pending', progress), 986 'pending': ('pending', progress), 987 'skipped': ('error', 0), 988 'starting': ('pending', progress), 989 'importing': ('pending', progress), 990 'forked': ('error', 0), 991 'waiting': ('pending', progress), 992 'unknown': ('error', 0), 993 } 994 995 build_url = os.path.join( 996 app.config['PUBLIC_COPR_BASE_URL'], 997 'coprs', build.copr.full_name.replace('@', 'g/'), 998 'build', str(build.id) 999 ) 1000 1001 data = { 1002 'username': 'Copr build', 1003 'comment': '#{}'.format(build.id), 1004 'url': build_url, 1005 'status': state_table[build.state][0], 1006 'percent': state_table[build.state][1], 1007 'uid': str(build.id), 1008 } 1009 1010 log.info('Sending data to Pagure API: %s', pprint.pformat(data)) 1011 response = requests.post(api_url, data=data, headers=headers) 1012 log.info('Pagure API response: %s', response.text)
1013 1014 @classmethod
1015 - def cancel_build(cls, user, build):
1016 if not user.can_build_in(build.copr): 1017 raise InsufficientRightsException( 1018 "You are not allowed to cancel this build.") 1019 if not build.cancelable: 1020 if build.status == StatusEnum("starting"): 1021 # this is not intuitive, that's why we provide more specific message 1022 err_msg = "Cannot cancel build {} in state 'starting'".format(build.id) 1023 else: 1024 err_msg = "Cannot cancel build {}".format(build.id) 1025 raise RequestCannotBeExecuted(err_msg) 1026 1027 if build.status == StatusEnum("running"): # otherwise the build is just in frontend 1028 ActionsLogic.send_cancel_build(build) 1029 1030 build.canceled = True 1031 cls.process_update_callback(build) 1032 1033 for chroot in build.build_chroots: 1034 chroot.status = 2 # canceled 1035 if chroot.ended_on is not None: 1036 chroot.ended_on = time.time()
1037 1038 @classmethod
1039 - def delete_build(cls, user, build, send_delete_action=True):
1040 """ 1041 :type user: models.User 1042 :type build: models.Build 1043 """ 1044 if not user.can_edit(build.copr) or build.persistent: 1045 raise InsufficientRightsException( 1046 "You are not allowed to delete build `{}`.".format(build.id)) 1047 1048 if not build.finished: 1049 raise ActionInProgressException( 1050 "You can not delete build `{}` which is not finished.".format(build.id), 1051 "Unfinished build") 1052 1053 if send_delete_action: 1054 ActionsLogic.send_delete_build(build) 1055 1056 for build_chroot in build.build_chroots: 1057 db.session.delete(build_chroot) 1058 1059 db.session.delete(build)
1060 1061 @classmethod
1062 - def mark_as_failed(cls, build_id):
1063 """ 1064 Marks build as failed on all its non-finished chroots 1065 """ 1066 build = cls.get(build_id).one() 1067 chroots = filter(lambda x: x.status != StatusEnum("succeeded"), build.build_chroots) 1068 for chroot in chroots: 1069 chroot.status = StatusEnum("failed") 1070 cls.process_update_callback(build) 1071 return build
1072 1073 @classmethod
1074 - def last_modified(cls, copr):
1075 """ Get build datetime (as epoch) of last successful build 1076 1077 :arg copr: object of copr 1078 """ 1079 builds = cls.get_multiple_by_copr(copr) 1080 1081 last_build = ( 1082 builds.join(models.BuildChroot) 1083 .filter((models.BuildChroot.status == StatusEnum("succeeded")) 1084 | (models.BuildChroot.status == StatusEnum("skipped"))) 1085 .filter(models.BuildChroot.ended_on.isnot(None)) 1086 .order_by(models.BuildChroot.ended_on.desc()) 1087 ).first() 1088 if last_build: 1089 return last_build.ended_on 1090 else: 1091 return None
1092 1093 @classmethod
1094 - def filter_is_finished(cls, query, is_finished):
1095 # todo: check that ended_on is set correctly for all cases 1096 # e.g.: failed dist-git import, cancellation 1097 if is_finished: 1098 return query.join(models.BuildChroot).filter(models.BuildChroot.ended_on.isnot(None)) 1099 else: 1100 return query.join(models.BuildChroot).filter(models.BuildChroot.ended_on.is_(None))
1101 1102 @classmethod
1103 - def filter_by_group_name(cls, query, group_name):
1104 return query.filter(models.Group.name == group_name)
1105 1106 @classmethod
1107 - def filter_by_package_name(cls, query, package_name):
1108 return query.join(models.Package).filter(models.Package.name == package_name)
1109 1110 @classmethod
1111 - def clean_old_builds(cls):
1112 dirs = ( 1113 db.session.query( 1114 models.CoprDir.id, 1115 models.Package.id, 1116 models.Package.max_builds) 1117 .join(models.Build, models.Build.copr_dir_id==models.CoprDir.id) 1118 .join(models.Package) 1119 .filter(models.Package.max_builds > 0) 1120 .group_by( 1121 models.CoprDir.id, 1122 models.Package.max_builds, 1123 models.Package.id) 1124 .having(func.count(models.Build.id) > models.Package.max_builds) 1125 ) 1126 1127 for dir_id, package_id, limit in dirs.all(): 1128 delete_builds = ( 1129 models.Build.query.filter( 1130 models.Build.copr_dir_id==dir_id, 1131 models.Build.package_id==package_id) 1132 .order_by(desc(models.Build.id)) 1133 .offset(limit) 1134 .all() 1135 ) 1136 1137 for build in delete_builds: 1138 try: 1139 cls.delete_build(build.copr.user, build) 1140 except ActionInProgressException: 1141 # postpone this one to next day run 1142 log.error("Build(id={}) delete failed, unfinished action.".format(build.id))
1143
1144 -class BuildChrootsLogic(object):
1145 @classmethod
1146 - def get_by_build_id_and_name(cls, build_id, name):
1147 mc = MockChrootsLogic.get_from_name(name).one() 1148 1149 return ( 1150 BuildChroot.query 1151 .filter(BuildChroot.build_id == build_id) 1152 .filter(BuildChroot.mock_chroot_id == mc.id) 1153 )
1154 1155 @classmethod
1156 - def get_multiply(cls):
1157 query = ( 1158 models.BuildChroot.query 1159 .join(models.BuildChroot.build) 1160 .join(models.BuildChroot.mock_chroot) 1161 .join(models.Build.copr) 1162 .join(models.Copr.user) 1163 .outerjoin(models.Group) 1164 ) 1165 return query
1166 1167 @classmethod
1168 - def filter_by_build_id(cls, query, build_id):
1169 return query.filter(models.Build.id == build_id)
1170 1171 @classmethod
1172 - def filter_by_project_id(cls, query, project_id):
1173 return query.filter(models.Copr.id == project_id)
1174 1175 @classmethod
1176 - def filter_by_project_user_name(cls, query, username):
1177 return query.filter(models.User.username == username)
1178 1179 @classmethod
1180 - def filter_by_state(cls, query, state):
1181 return query.filter(models.BuildChroot.status == StatusEnum(state))
1182 1183 @classmethod
1184 - def filter_by_group_name(cls, query, group_name):
1185 return query.filter(models.Group.name == group_name)
1186
1187 1188 -class BuildsMonitorLogic(object):
1189 @classmethod
1190 - def get_monitor_data(cls, copr):
1191 query = """ 1192 SELECT 1193 package.id as package_id, 1194 package.name AS package_name, 1195 build.id AS build_id, 1196 build_chroot.status AS build_chroot_status, 1197 build.pkg_version AS build_pkg_version, 1198 mock_chroot.id AS mock_chroot_id, 1199 mock_chroot.os_release AS mock_chroot_os_release, 1200 mock_chroot.os_version AS mock_chroot_os_version, 1201 mock_chroot.arch AS mock_chroot_arch 1202 FROM package 1203 JOIN (SELECT 1204 MAX(build.id) AS max_build_id_for_chroot, 1205 build.package_id AS package_id, 1206 build_chroot.mock_chroot_id AS mock_chroot_id 1207 FROM build 1208 JOIN build_chroot 1209 ON build.id = build_chroot.build_id 1210 WHERE build.copr_id = {copr_id} 1211 AND build_chroot.status != 2 1212 GROUP BY build.package_id, 1213 build_chroot.mock_chroot_id) AS max_build_ids_for_a_chroot 1214 ON package.id = max_build_ids_for_a_chroot.package_id 1215 JOIN build 1216 ON build.id = max_build_ids_for_a_chroot.max_build_id_for_chroot 1217 JOIN build_chroot 1218 ON build_chroot.mock_chroot_id = max_build_ids_for_a_chroot.mock_chroot_id 1219 AND build_chroot.build_id = max_build_ids_for_a_chroot.max_build_id_for_chroot 1220 JOIN mock_chroot 1221 ON mock_chroot.id = max_build_ids_for_a_chroot.mock_chroot_id 1222 ORDER BY package.name ASC, package.id ASC, mock_chroot.os_release ASC, mock_chroot.os_version ASC, mock_chroot.arch ASC 1223 """.format(copr_id=copr.id) 1224 rows = db.session.execute(query) 1225 return rows
1226