1 import math
2 import random
3 import string
4 import html5_parser
5
6 from os.path import normpath
7 from six import with_metaclass
8 from six.moves.urllib.parse import urlparse, parse_qs, urlunparse, urlencode
9 import re
10
11 import flask
12 import posixpath
13 from flask import url_for
14 from dateutil import parser as dt_parser
15 from netaddr import IPAddress, IPNetwork
16 from redis import StrictRedis
17 from sqlalchemy.types import TypeDecorator, VARCHAR
18 import json
19
20 from copr_common.enums import EnumType
21 from copr_common.rpm import splitFilename
22 from coprs import constants
23 from coprs import app
27 """ Generate a random string used as token to access the API
28 remotely.
29
30 :kwarg: size, the size of the token to generate, defaults to 30
31 chars.
32 :return: a string, the API token for the user.
33 """
34 return ''.join(random.choice(string.ascii_lowercase) for x in range(size))
35
36
37 REPO_DL_STAT_FMT = "repo_dl_stat::{copr_user}@{copr_project_name}:{copr_name_release}"
38 CHROOT_REPO_MD_DL_STAT_FMT = "chroot_repo_metadata_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}"
39 CHROOT_RPMS_DL_STAT_FMT = "chroot_rpms_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}"
40 PROJECT_RPMS_DL_STAT_FMT = "project_rpms_dl_stat:hset::{copr_user}@{copr_project_name}"
41
42
43 FINISHED_STATUSES = ["succeeded", "forked", "canceled", "skipped", "failed"]
48
51
52 vals = {"nothing": 0, "request": 1, "approved": 2}
53
54 @classmethod
56 return [(n, k) for k, n in cls.vals.items() if n != without]
57
60 vals = {"unset": 0,
61 "link": 1,
62 "upload": 2,
63 "pypi": 5,
64 "rubygems": 6,
65 "scm": 8,
66 "custom": 9,
67 }
68
71 """Represents an immutable structure as a json-encoded string.
72
73 Usage::
74
75 JSONEncodedDict(255)
76
77 """
78
79 impl = VARCHAR
80
82 if value is not None:
83 value = json.dumps(value)
84
85 return value
86
88 if value is not None:
89 value = json.loads(value)
90 return value
91
94 - def __init__(self, query, total_count, page=1,
95 per_page_override=None, urls_count_override=None,
96 additional_params=None):
97
98 self.query = query
99 self.total_count = total_count
100 self.page = page
101 self.per_page = per_page_override or constants.ITEMS_PER_PAGE
102 self.urls_count = urls_count_override or constants.PAGES_URLS_COUNT
103 self.additional_params = additional_params or dict()
104
105 self._sliced_query = None
106
107 - def page_slice(self, page):
108 return (self.per_page * (page - 1),
109 self.per_page * page)
110
111 @property
113 if not self._sliced_query:
114 self._sliced_query = self.query[slice(*self.page_slice(self.page))]
115 return self._sliced_query
116
117 @property
119 return int(math.ceil(self.total_count / float(self.per_page)))
120
122 if start:
123 if self.page - 1 > self.urls_count // 2:
124 return self.url_for_other_page(request, 1), 1
125 else:
126 if self.page < self.pages - self.urls_count // 2:
127 return self.url_for_other_page(request, self.pages), self.pages
128
129 return None
130
132 left_border = self.page - self.urls_count // 2
133 left_border = 1 if left_border < 1 else left_border
134 right_border = self.page + self.urls_count // 2
135 right_border = self.pages if right_border > self.pages else right_border
136
137 return [(self.url_for_other_page(request, i), i)
138 for i in range(left_border, right_border + 1)]
139
140 - def url_for_other_page(self, request, page):
141 args = request.view_args.copy()
142 args["page"] = page
143 args.update(self.additional_params)
144 return flask.url_for(request.endpoint, **args)
145
148 """
149 Get a git branch name from chroot. Follow the fedora naming standard.
150 """
151 os, version, arch = chroot.rsplit("-", 2)
152 if os == "fedora":
153 if version == "rawhide":
154 return "master"
155 os = "f"
156 elif os == "epel" and int(version) <= 6:
157 os = "el"
158 elif os == "mageia" and version == "cauldron":
159 os = "cauldron"
160 version = ""
161 elif os == "mageia":
162 os = "mga"
163 return "{}{}".format(os, version)
164
167 """
168 Parse package name from possibly incomplete nvra string.
169 """
170
171 if pkg.count(".") >= 3 and pkg.count("-") >= 2:
172 return splitFilename(pkg)[0]
173
174
175 result = ""
176 pkg = pkg.replace(".rpm", "").replace(".src", "")
177
178 for delim in ["-", "."]:
179 if delim in pkg:
180 parts = pkg.split(delim)
181 for part in parts:
182 if any(map(lambda x: x.isdigit(), part)):
183 return result[:-1]
184
185 result += part + "-"
186
187 return result[:-1]
188
189 return pkg
190
214
217 """
218 Ensure that url either has http or https protocol according to the
219 option in app config "ENFORCE_PROTOCOL_FOR_BACKEND_URL"
220 """
221 if app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] == "https":
222 return url.replace("http://", "https://")
223 elif app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] == "http":
224 return url.replace("https://", "http://")
225 else:
226 return url
227
230 """
231 Ensure that url either has http or https protocol according to the
232 option in app config "ENFORCE_PROTOCOL_FOR_FRONTEND_URL"
233 """
234 if app.config["ENFORCE_PROTOCOL_FOR_FRONTEND_URL"] == "https":
235 return url.replace("http://", "https://")
236 elif app.config["ENFORCE_PROTOCOL_FOR_FRONTEND_URL"] == "http":
237 return url.replace("https://", "http://")
238 else:
239 return url
240
243
245 """
246 Usage:
247
248 SQLAlchObject.to_dict() => returns a flat dict of the object
249 SQLAlchObject.to_dict({"foo": {}}) => returns a dict of the object
250 and will include a flat dict of object foo inside of that
251 SQLAlchObject.to_dict({"foo": {"bar": {}}, "spam": {}}) => returns
252 a dict of the object, which will include dict of foo
253 (which will include dict of bar) and dict of spam.
254
255 Options can also contain two special values: __columns_only__
256 and __columns_except__
257
258 If present, the first makes only specified fields appear,
259 the second removes specified fields. Both of these fields
260 must be either strings (only works for one field) or lists
261 (for one and more fields).
262
263 SQLAlchObject.to_dict({"foo": {"__columns_except__": ["id"]},
264 "__columns_only__": "name"}) =>
265
266 The SQLAlchObject will only put its "name" into the resulting dict,
267 while "foo" all of its fields except "id".
268
269 Options can also specify whether to include foo_id when displaying
270 related foo object (__included_ids__, defaults to True).
271 This doesn"t apply when __columns_only__ is specified.
272 """
273
274 result = {}
275 if options is None:
276 options = {}
277 columns = self.serializable_attributes
278
279 if "__columns_only__" in options:
280 columns = options["__columns_only__"]
281 else:
282 columns = set(columns)
283 if "__columns_except__" in options:
284 columns_except = options["__columns_except__"]
285 if not isinstance(options["__columns_except__"], list):
286 columns_except = [options["__columns_except__"]]
287
288 columns -= set(columns_except)
289
290 if ("__included_ids__" in options and
291 options["__included_ids__"] is False):
292
293 related_objs_ids = [
294 r + "_id" for r, _ in options.items()
295 if not r.startswith("__")]
296
297 columns -= set(related_objs_ids)
298
299 columns = list(columns)
300
301 for column in columns:
302 result[column] = getattr(self, column)
303
304 for related, values in options.items():
305 if hasattr(self, related):
306 result[related] = getattr(self, related).to_dict(values)
307 return result
308
309 @property
312
316 self.host = config.get("REDIS_HOST", "127.0.0.1")
317 self.port = int(config.get("REDIS_PORT", "6379"))
318
320 return StrictRedis(host=self.host, port=self.port)
321
324 """
325 Creates connection to redis, now we use default instance at localhost, no config needed
326 """
327 return StrictRedis()
328
331 if v is None:
332 return False
333 return v.lower() in ("yes", "true", "t", "1")
334
337 """
338 Examine given copr and generate proper URL for the `view`
339
340 Values of `username/group_name` and `coprname` are automatically passed as the first two URL parameters,
341 and therefore you should *not* pass them manually.
342
343 Usage:
344 copr_url("coprs_ns.foo", copr)
345 copr_url("coprs_ns.foo", copr, arg1='bar', arg2='baz)
346 """
347 if copr.is_a_group_project:
348 return url_for(view, group_name=copr.group.name, coprname=copr.name, **kwargs)
349 return url_for(view, username=copr.user.name, coprname=copr.name, **kwargs)
350
357
361
362
363 from sqlalchemy.engine.default import DefaultDialect
364 from sqlalchemy.sql.sqltypes import String, DateTime, NullType
365
366
367 PY3 = str is not bytes
368 text = str if PY3 else unicode
369 int_type = int if PY3 else (int, long)
370 str_type = str if PY3 else (str, unicode)
374 """Teach SA how to literalize various things."""
387 return process
388
399
402 """NOTE: This is entirely insecure. DO NOT execute the resulting strings.
403 This can be used for debuggin - it is not and should not be used in production
404 code.
405
406 It is useful if you want to debug an sqlalchemy query, i.e. copy the
407 resulting SQL query into psql console and try to tweak it so that it
408 actually works or works faster.
409 """
410 import sqlalchemy.orm
411 if isinstance(statement, sqlalchemy.orm.Query):
412 statement = statement.statement
413 return statement.compile(
414 dialect=LiteralDialect(),
415 compile_kwargs={'literal_binds': True},
416 ).string
417
420 app.update_template_context(context)
421 t = app.jinja_env.get_template(template_name)
422 rv = t.stream(context)
423 rv.enable_buffering(2)
424 return rv
425
433
436 """
437 Expands variables and sanitize repo url to be used for mock config
438 """
439 parsed_url = urlparse(repo_url)
440 query = parse_qs(parsed_url.query)
441
442 if parsed_url.scheme == "copr":
443 user = parsed_url.netloc
444 prj = parsed_url.path.split("/")[1]
445 repo_url = "/".join([
446 flask.current_app.config["BACKEND_BASE_URL"],
447 "results", user, prj, chroot
448 ]) + "/"
449
450 elif "priority" in query:
451 query.pop("priority")
452 query_string = urlencode(query, doseq=True)
453 parsed_url = parsed_url._replace(query=query_string)
454 repo_url = urlunparse(parsed_url)
455
456 repo_url = repo_url.replace("$chroot", chroot)
457 repo_url = repo_url.replace("$distname", chroot.rsplit("-", 2)[0])
458 return repo_url
459
462 """
463 :param repo: str repo from Copr/CoprChroot/Build/...
464 :param supported_keys list of supported optional parameters
465 :return: dict of optional parameters parsed from the repo URL
466 """
467 supported_keys = supported_keys or ["priority"]
468 params = {}
469 qs = parse_qs(urlparse(repo).query)
470 for k, v in qs.items():
471 if k in supported_keys:
472
473
474 value = int(v[0]) if v[0].isnumeric() else v[0]
475 params[k] = value
476 return params
477
480 """ Return dict with proper build config contents """
481 chroot = None
482 for i in copr.copr_chroots:
483 if i.mock_chroot.name == chroot_id:
484 chroot = i
485 if not chroot:
486 return {}
487
488 packages = "" if not chroot.buildroot_pkgs else chroot.buildroot_pkgs
489
490 repos = [{
491 "id": "copr_base",
492 "baseurl": copr.repo_url + "/{}/".format(chroot_id),
493 "name": "Copr repository",
494 }]
495
496 if not copr.auto_createrepo:
497 repos.append({
498 "id": "copr_base_devel",
499 "baseurl": copr.repo_url + "/{}/devel/".format(chroot_id),
500 "name": "Copr buildroot",
501 })
502
503 def get_additional_repo_views(repos_list):
504 repos = []
505 for repo in repos_list:
506 params = parse_repo_params(repo)
507 repo_view = {
508 "id": generate_repo_name(repo),
509 "baseurl": pre_process_repo_url(chroot_id, repo),
510 "name": "Additional repo " + generate_repo_name(repo),
511 }
512 repo_view.update(params)
513 repos.append(repo_view)
514 return repos
515
516 repos.extend(get_additional_repo_views(copr.repos_list))
517 repos.extend(get_additional_repo_views(chroot.repos_list))
518
519 return {
520 'project_id': copr.repo_id,
521 'additional_packages': packages.split(),
522 'repos': repos,
523 'chroot': chroot_id,
524 'use_bootstrap_container': copr.use_bootstrap_container,
525 'with_opts': chroot.with_opts.split(),
526 'without_opts': chroot.without_opts.split(),
527 }
528
536
539 if not url:
540 return None
541
542 return re.sub(r'(\.git)?/*$', '', url)
543
546 if not url:
547 return False
548
549 url = trim_git_url(url)
550 return urlparse(url)
551
554 """
555 We cannot really switch to the new
556 copr:{hostname}:{owner}:{project} format yet, because it is implemented in
557 dnf-plugins-core-3.x which is only on F29+
558
559 Since the F29+ plugin is able to work with both old and new formats, we can
560 safely stay with the old one until F28 is still supported. Once it goes EOL,
561 we can migrate to the new format.
562
563 New format is:
564
565 return "copr:{0}:{1}:{2}".format(app.config["PUBLIC_COPR_HOSTNAME"].split(":")[0],
566 copr_dir.copr.owner_name.replace("@", "group_"),
567 copr_dir.name)
568
569 """
570 return copr_dir.repo_id
571
575 if not subdir:
576 self.subdir = '.'
577 else:
578 self.subdir = normpath(subdir).strip('/')
579
581 if not path:
582 return False
583
584 changed = normpath(path).strip('/')
585 if changed == '.':
586 return False
587
588 if self.subdir == '.':
589 return True
590
591 return changed.startswith(self.subdir + '/')
592
595 parsed = html5_parser.parse(str(html_string))
596 elements = parsed.xpath(
597 "//section[contains(@class, 'commit_diff')]"
598 "//div[contains(@class, 'card-header')]"
599 "//a[contains(@class, 'font-weight-bold')]"
600 "/text()")
601
602 return set([str(x) for x in elements])
603
606 changes = set()
607 for line in text.split('\n'):
608 match = re.search(r'^(\+\+\+|---) [ab]/(.*)$', line)
609 if match:
610 changes.add(str(match.group(2)))
611 match = re.search(r'^diff --git a/(.*) b/(.*)$', line)
612 if match:
613 changes.add(str(match.group(1)))
614 changes.add(str(match.group(2)))
615 print(changes)
616
617 return changes
618