Module manage
[hide private]
[frames] | no frames]

Source Code for Module manage

  1  #!/usr/bin/python 
  2   
  3  import argparse 
  4  import os 
  5  import subprocess 
  6  import datetime 
  7  import sqlalchemy 
  8  import time 
  9   
 10  import flask 
 11  from flask_script import Manager, Command, Option, Group 
 12  from flask_whooshee import Whooshee 
 13   
 14  from coprs import app 
 15  from coprs import db 
 16  from coprs import exceptions 
 17  from coprs import models 
 18  from coprs.logic import coprs_logic, packages_logic, actions_logic, builds_logic 
 19  from coprs.views.misc import create_user_wrapper 
 20  from coprs.whoosheers import CoprWhoosheer 
 21  from run import generate_repo_packages 
 22  from sqlalchemy import or_ 
 23  from coprs.helpers import chroot_to_branch 
 24   
 25   
26 -class TestCommand(Command):
27
28 - def run(self, test_args):
29 os.environ["COPRS_ENVIRON_UNITTEST"] = "1" 30 if not (("COPR_CONFIG" in os.environ) and os.environ["COPR_CONFIG"]): 31 os.environ["COPR_CONFIG"] = "/etc/copr/copr_unit_test.conf" 32 os.environ["PYTHONPATH"] = "." 33 return subprocess.call(["/usr/bin/python", "-m", "pytest"] + (test_args or []))
34 35 option_list = ( 36 Option("-a", 37 dest="test_args", 38 nargs=argparse.REMAINDER), 39 )
40 41
42 -class CreateSqliteFileCommand(Command):
43 44 """ 45 Create the sqlite DB file (not the tables). 46 Used for alembic, "create_db" does this automatically. 47 """ 48
49 - def run(self):
50 if flask.current_app.config["SQLALCHEMY_DATABASE_URI"].startswith("sqlite"): 51 # strip sqlite:/// 52 datadir_name = os.path.dirname( 53 flask.current_app.config["SQLALCHEMY_DATABASE_URI"][10:]) 54 if not os.path.exists(datadir_name): 55 os.makedirs(datadir_name)
56 57
58 -class CreateDBCommand(Command):
59 60 """ 61 Create the DB schema 62 """ 63
64 - def run(self, alembic_ini=None):
65 CreateSqliteFileCommand().run() 66 db.create_all() 67 68 # load the Alembic configuration and generate the 69 # version table, "stamping" it with the most recent rev: 70 from alembic.config import Config 71 from alembic import command 72 alembic_cfg = Config(alembic_ini) 73 command.stamp(alembic_cfg, "head") 74 75 # Functions are not covered by models.py, and no migrations are run 76 # by command.stamp() above. Create functions explicitly: 77 builds_logic.BuildsLogic.init_db()
78 79 option_list = ( 80 Option("--alembic", 81 "-f", 82 dest="alembic_ini", 83 help="Path to the alembic configuration file (alembic.ini)", 84 required=True), 85 )
86 87
88 -class DropDBCommand(Command):
89 90 """ 91 Delete DB 92 """ 93
94 - def run(self):
95 db.drop_all()
96 97
98 -class ChrootCommand(Command):
99
100 - def print_invalid_format(self, chroot_name):
101 print( 102 "{0} - invalid chroot format, must be '{release}-{version}-{arch}'." 103 .format(chroot_name))
104
105 - def print_already_exists(self, chroot_name):
106 print("{0} - already exists.".format(chroot_name))
107
108 - def print_doesnt_exist(self, chroot_name):
109 print("{0} - chroot doesn\"t exist.".format(chroot_name))
110 111 option_list = ( 112 Option("chroot_names", 113 help="Chroot name, e.g. fedora-18-x86_64.", 114 nargs="+"), 115 )
116 117
118 -class CreateChrootCommand(ChrootCommand):
119 120 "Creates a mock chroot in DB" 121
122 - def __init__(self):
123 self.option_list += Option( 124 "--dist-git-branch", 125 "-b", 126 dest="branch", 127 help="Branch name for this set of new chroots"),
128
129 - def run(self, chroot_names, branch=None):
130 for chroot_name in chroot_names: 131 if not branch: 132 branch = chroot_to_branch(chroot_name) 133 branch_object = coprs_logic.BranchesLogic.get_or_create(branch) 134 try: 135 chroot = coprs_logic.MockChrootsLogic.add(chroot_name) 136 chroot.distgit_branch = branch_object 137 db.session.commit() 138 except exceptions.MalformedArgumentException: 139 self.print_invalid_format(chroot_name) 140 except exceptions.DuplicateException: 141 self.print_already_exists(chroot_name)
142 143
144 -class RawhideToReleaseCommand(Command):
145 146 option_list = ( 147 Option("rawhide_chroot", help="Rawhide chroot name, e.g. fedora-rawhide-x86_64."), 148 Option("dest_chroot", help="Destination chroot, e.g. fedora-24-x86_64."), 149 ) 150
151 - def run(self, rawhide_chroot, dest_chroot):
152 mock_chroot = coprs_logic.MockChrootsLogic.get_from_name(dest_chroot).first() 153 if not mock_chroot: 154 print("Given chroot does not exist. Please run:") 155 print(" sudo python manage.py create_chroot {}".format(dest_chroot)) 156 return 157 158 mock_rawhide_chroot = coprs_logic.MockChrootsLogic.get_from_name(rawhide_chroot).first() 159 if not mock_rawhide_chroot: 160 print("Given rawhide chroot does not exist. Didnt you mistyped?:") 161 print(" {}".format(rawhide_chroot)) 162 return 163 164 for copr in coprs_logic.CoprsLogic.get_all(): 165 if not self.has_rawhide(copr): 166 continue 167 168 data = {"copr": copr.name, 169 "user": copr.user.name, 170 "rawhide_chroot": rawhide_chroot, 171 "dest_chroot": dest_chroot, 172 "builds": []} 173 174 for package in packages_logic.PackagesLogic.get_all(copr.id): 175 last_build = package.last_build(successful=True) 176 if last_build: 177 data["builds"].append(last_build.result_dir_name) 178 179 # rbc means rawhide_build_chroot (we needed short variable) 180 rbc = builds_logic.BuildChrootsLogic.get_by_build_id_and_name(last_build.id, rawhide_chroot).first() 181 dbc = builds_logic.BuildChrootsLogic.get_by_build_id_and_name(last_build.id, dest_chroot).first() 182 if rbc and not dbc: 183 dest_build_chroot = models.BuildChroot(**rbc.to_dict()) 184 dest_build_chroot.mock_chroot_id = mock_chroot.id 185 dest_build_chroot.mock_chroot = mock_chroot 186 db.session.add(dest_build_chroot) 187 188 if len(data["builds"]): 189 actions_logic.ActionsLogic.send_rawhide_to_release(data) 190 self.turn_on_the_chroot_for_copr(copr, rawhide_chroot, mock_chroot) 191 192 db.session.commit()
193
194 - def turn_on_the_chroot_for_copr(self, copr, rawhide_name, mock_chroot):
195 rawhide_chroot = coprs_logic.CoprChrootsLogic.get_by_name_safe(copr, rawhide_name) 196 dest_chroot = coprs_logic.CoprChrootsLogic.get_by_name_safe(copr, mock_chroot.name) 197 198 if not rawhide_chroot or dest_chroot: 199 return 200 201 create_kwargs = { 202 "buildroot_pkgs": rawhide_chroot.buildroot_pkgs, 203 "comps": rawhide_chroot.comps, 204 "comps_name": rawhide_chroot.comps_name, 205 } 206 coprs_logic.CoprChrootsLogic.create_chroot(copr.user, copr, mock_chroot, **create_kwargs)
207
208 - def has_rawhide(self, copr):
209 return any(filter(lambda ch: ch.os_version == "rawhide", copr.mock_chroots))
210 211
212 -class BackendRawhideToReleaseCommand(RawhideToReleaseCommand):
213 214 "Copy backend data of the latest successful rawhide builds into a new chroot" 215
216 - def run(self, rawhide_chroot, dest_chroot):
217 for copr in coprs_logic.CoprsLogic.get_all(): 218 if not self.has_rawhide(copr): 219 continue 220 221 data = {"copr": copr.name, 222 "user": copr.owner_name, 223 "rawhide_chroot": rawhide_chroot, 224 "dest_chroot": dest_chroot, 225 "builds": []} 226 227 for package in packages_logic.PackagesLogic.get_all(copr.id): 228 last_build = package.last_build(successful=True) 229 if last_build: 230 data["builds"].append(last_build.result_dir_name) 231 232 if len(data["builds"]): 233 actions_logic.ActionsLogic.send_rawhide_to_release(data) 234 print("Created copy action from {}/{} to {}/{}" 235 .format(copr.full_name, rawhide_chroot, copr.full_name, dest_chroot)) 236 237 db.session.commit()
238
239 -class AlterChrootCommand(ChrootCommand):
240 241 "Activates or deactivates a chroot" 242
243 - def run(self, chroot_names, action):
244 activate = (action == "activate") 245 for chroot_name in chroot_names: 246 try: 247 coprs_logic.MockChrootsLogic.edit_by_name( 248 chroot_name, activate) 249 db.session.commit() 250 except exceptions.MalformedArgumentException: 251 self.print_invalid_format(chroot_name) 252 except exceptions.NotFoundException: 253 self.print_doesnt_exist(chroot_name)
254 255 option_list = ChrootCommand.option_list + ( 256 Option("--action", 257 "-a", 258 dest="action", 259 help="Action to take - currently activate or deactivate", 260 choices=["activate", "deactivate"], 261 required=True), 262 )
263 264
265 -class DropChrootCommand(ChrootCommand):
266 267 "Activates or deactivates a chroot" 268
269 - def run(self, chroot_names):
278 279
280 -class DisplayChrootsCommand(Command):
281 282 "Displays current mock chroots" 283
284 - def run(self, active_only):
285 for ch in coprs_logic.MockChrootsLogic.get_multiple( 286 active_only=active_only).all(): 287 288 print(ch.name)
289 290 option_list = ( 291 Option("--active-only", 292 "-a", 293 dest="active_only", 294 help="Display only active chroots", 295 required=False, 296 action="store_true", 297 default=False), 298 )
299 300
301 -class AddUserCommand(Command):
302 303 """ 304 You should not use regularly as that user will not be related to FAS account. 305 This should be used only for testing or adding special accounts e.g. proxy user. 306 """ 307
308 - def run(self, name, mail, **kwargs):
309 user = models.User.query.filter(models.User.username == name).first() 310 if user: 311 print("User named {0} already exists.".format(name)) 312 return 313 314 user = create_user_wrapper(name, mail) 315 if kwargs["api_token"]: 316 user.api_token = kwargs["api_token"] 317 if kwargs["api_login"]: 318 user.api_token = kwargs["api_login"] 319 320 db.session.add(user) 321 db.session.commit()
322 323 option_list = ( 324 Option("name"), 325 Option("mail"), 326 Option("--api_token", default=None, required=False), 327 Option("--api_login", default=None, required=False), 328 )
329 330
331 -class AlterUserCommand(Command):
332
333 - def run(self, name, **kwargs):
334 user = models.User.query.filter( 335 models.User.username == name).first() 336 if not user: 337 print("No user named {0}.".format(name)) 338 return 339 340 if kwargs["admin"]: 341 user.admin = True 342 if kwargs["no_admin"]: 343 user.admin = False 344 if kwargs["proven"]: 345 user.proven = True 346 if kwargs["no_proven"]: 347 user.proven = False 348 if kwargs["proxy"]: 349 user.proxy = True 350 if kwargs["no_proxy"]: 351 user.proxy = False 352 353 db.session.add(user) 354 db.session.commit()
355 356 option_list = ( 357 Option("name"), 358 Group( 359 Option("--admin", 360 action="store_true"), 361 Option("--no-admin", 362 action="store_true"), 363 exclusive=True 364 ), 365 Group( 366 Option("--proven", 367 action="store_true"), 368 Option("--no-proven", 369 action="store_true"), 370 exclusive=True 371 ), 372 Group( 373 Option("--proxy", 374 action="store_true"), 375 Option("--no-proxy", 376 action="store_true"), 377 exclusive=True 378 ) 379 )
380 381
382 -class FailBuildCommand(Command):
383 384 """ 385 Marks build as failed on all its non-finished chroots 386 """ 387 388 option_list = [Option("build_id")] 389
390 - def run(self, build_id, **kwargs):
391 try: 392 builds_logic.BuildsLogic.mark_as_failed(build_id) 393 print("Marking non-finished chroots of build {} as failed".format(build_id)) 394 db.session.commit() 395 396 except (sqlalchemy.exc.DataError, sqlalchemy.orm.exc.NoResultFound) as e: 397 print("Error: No such build {}".format(build_id)) 398 return 1
399 400
401 -class UpdateIndexesCommand(Command):
402 """ 403 recreates whoosh indexes for all projects 404 """ 405
406 - def run(self):
407 index = Whooshee.get_or_create_index(app, CoprWhoosheer) 408 409 writer = index.writer() 410 for copr in coprs_logic.CoprsLogic.get_all(): 411 CoprWhoosheer.delete_copr(writer, copr) 412 writer.commit(optimize=True) 413 414 writer = index.writer() 415 writer.schema = CoprWhoosheer.schema 416 writer.commit(optimize=True) 417 418 writer = index.writer() 419 for copr in coprs_logic.CoprsLogic.get_all(): 420 CoprWhoosheer.insert_copr(writer, copr) 421 writer.commit(optimize=True)
422 423
424 -class UpdateIndexesQuickCommand(Command):
425 """ 426 Recreates whoosh indexes for projects for which 427 indexed data were updated in last n minutes. 428 Doesn't update schema. 429 """ 430 431 option_list = [Option("minutes_passed")] 432
433 - def run(self, minutes_passed):
434 index = Whooshee.get_or_create_index(app, CoprWhoosheer) 435 436 writer = index.writer() 437 query = db.session.query(models.Copr).filter( 438 models.Copr.latest_indexed_data_update >= time.time()-int(minutes_passed)*60 439 ) 440 for copr in query.all(): 441 CoprWhoosheer.update_copr(writer, copr) 442 writer.commit()
443 444
445 -class GenerateRepoPackagesCommand(Command):
446 """ 447 go through all coprs and create configuration rpm packages 448 for them, if they don't already have it 449 """ 450
451 - def run(self):
453 454 455 manager = Manager(app) 456 manager.add_command("test", TestCommand()) 457 manager.add_command("create_sqlite_file", CreateSqliteFileCommand()) 458 manager.add_command("create_db", CreateDBCommand()) 459 manager.add_command("drop_db", DropDBCommand()) 460 manager.add_command("create_chroot", CreateChrootCommand()) 461 manager.add_command("alter_chroot", AlterChrootCommand()) 462 manager.add_command("display_chroots", DisplayChrootsCommand()) 463 manager.add_command("drop_chroot", DropChrootCommand()) 464 manager.add_command("alter_user", AlterUserCommand()) 465 manager.add_command("add_user", AddUserCommand()) 466 manager.add_command("fail_build", FailBuildCommand()) 467 manager.add_command("update_indexes", UpdateIndexesCommand()) 468 manager.add_command("update_indexes_quick", UpdateIndexesQuickCommand()) 469 manager.add_command("generate_repo_packages", GenerateRepoPackagesCommand()) 470 manager.add_command("rawhide_to_release", RawhideToReleaseCommand()) 471 manager.add_command("backend_rawhide_to_release", BackendRawhideToReleaseCommand()) 472 473 if __name__ == "__main__": 474 manager.run() 475