-
Notifications
You must be signed in to change notification settings - Fork 32
Expand file tree
/
Copy pathbot.py
More file actions
1971 lines (1646 loc) · 70.3 KB
/
bot.py
File metadata and controls
1971 lines (1646 loc) · 70.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
import asyncio
import itertools
import os
import random
import string
import tempfile
import xml.etree.ElementTree as ET
from dataclasses import dataclass
from dataclasses import field
from datetime import datetime
from datetime import timedelta
from functools import reduce
from typing import Callable
from typing import ClassVar
from typing import Coroutine
from typing import Generator
from typing import Iterable
from typing import Literal
from typing import TypedDict
import aiofiles.os
import aiofiles.tempfile
import aiohttp
import git
from bci_build.logger import LOGGER
from bci_build.package import ALL_CONTAINER_IMAGE_NAMES
from bci_build.package import BaseContainerImage
from bci_build.package import OsVersion
from dotnet.updater import DOTNET_IMAGES
from dotnet.updater import DotNetBCI
from obs_package_update.util import CommandError
from obs_package_update.util import CommandResult
from obs_package_update.util import retry_async_run_cmd
from obs_package_update.util import run_cmd
from obs_package_update.util import RunCommand
from staging.build_result import Arch
from staging.build_result import PackageBuildResult
from staging.build_result import PackageStatusCode
from staging.build_result import RepositoryBuildResult
from staging.user import User
from staging.util import ensure_absent
from staging.util import get_obs_project_url
_CONFIG_T = Literal["meta", "prjconf"]
_CONF_TO_ROUTE: dict[_CONFIG_T, str] = {"meta": "_meta", "prjconf": "_config"}
_DEFAULT_REPOS = ["images", "containerfile"]
#: environment variable name from which the osc username for the bot is read
OSC_USER_ENVVAR_NAME = "OSC_USER"
BRANCH_NAME_ENVVAR_NAME = "BRANCH_NAME"
OS_VERSION_ENVVAR_NAME = "OS_VERSION"
#: environment variable from which the password of the bot's user is taken
OSC_PASSWORD_ENVVAR_NAME = "OSC_PASSWORD"
_GIT_COMMIT_ENV = {
"GIT_COMMITTER_NAME": "SUSE Update Bot",
"GIT_COMMITTER_EMAIL": "bci-internal@suse.de",
}
#: tuple of OsVersion that need the base container to be linked into the staging
#: project
#: this is usually only necessary during the early stages of a new SLE service
#: pack when autobuild has not yet synced the binaries of the container images
#: from IBS to OBS
OS_VERSION_NEEDS_BASE_CONTAINER = (OsVersion.SP6,)
def _get_base_image_prj_pkg(os_version: OsVersion) -> tuple[str, str]:
if os_version == OsVersion.TUMBLEWEED:
return "openSUSE:Factory", "opensuse-tumbleweed-image"
if os_version == OsVersion.BASALT:
raise ValueError("The Basalt base container is provided by BCI")
return f"SUSE:SLE-15-SP{os_version}:Update", "sles15-image"
def _get_bci_project_name(os_version: OsVersion) -> str:
prj_suffix = (
os_version
if os_version in (OsVersion.TUMBLEWEED, OsVersion.BASALT)
else "SLE-15-SP" + str(os_version)
)
return f"devel:BCI:{prj_suffix}"
async def _fetch_bci_devel_project_config(
os_version: OsVersion, config_type: _CONFIG_T = "prjconf"
) -> str:
"""Fetches the prjconf for the specified ``os_version``"""
prj_name = _get_bci_project_name(os_version)
route = f"https://api.opensuse.org/public/source/{prj_name}/{_CONF_TO_ROUTE[config_type]}"
async with aiohttp.ClientSession() as session:
async with session.get(route) as response:
return await response.text()
class _ProjectConfigs(TypedDict):
meta: ET.Element
prjconf: str
@dataclass
class StagingBot:
"""Bot that creates a staging project for the BCI images in the Open Build
Service via the git scm bridge.
This bot creates a new worktree based on the "deployment branch" (see
:py:attr:`deployment_branch_name`) and writes all build recipes into it. If
this results in a change, then the changes are committed and pushed to
github.
A new staging project with the name :py:attr:`staging_project_name` is then
created where all packages that were **changed** are inserted via the scm
bridge.
The bot needs to know the name of the user as whom it should act. The
username has to be set via the attribute :py:attr:`osc_username`. This is
unfortunately necessary, as :command:`osc` does not provide a
straightforward way how to get the default username...
Additionally, the bot stores its current settings in an environment file
(with the file name :py:attr:`DOTENV_FILE_NAME`), which can be either
sourced directly in bash or used to create an instance of the bot via
:py:func:`StagingBot.from_env_file`.
The bot supports running all actions as a user that is not configured in
:command:`osc`'s configuration file. All you have to do is to set the
environment variable :py:const:`OSC_PASSWORD_ENVVAR_NAME` to the password of
the user that is going to be used. The :py:meth:`setup` function will then
create a temporary configuration file for :command:`osc` and also set
``XDG_STATE_HOME`` to a temporary directory so that your local osc
:file:`cookiejar` is not modified. Both files are cleaned up via
:py:meth:`teardown`
"""
#: The operating system for which this instance has been created
os_version: OsVersion
#: Name of the branch to which the changes are pushed. If none is provided,
#: then :py:attr:`deployment_branch_name` is used with a few random
#: ASCII characters appended.
branch_name: str = ""
#: username of the user that will be used to perform the actions by the bot.
#:
#: This value must be provided, otherwise the post initialization function
#: raises an exception.
osc_username: str = ""
repositories: list[str] = field(default_factory=lambda: _DEFAULT_REPOS)
_packages: list[str] | None = None
_osc_conf_file: str = ""
_xdg_state_home_dir: tempfile.TemporaryDirectory | None = None
#: Maximum time to wait for a build to finish.
#: github actions will run for 6h at most, no point in waiting longer
MAX_WAIT_TIME_SEC: ClassVar[int] = 6 * 3600
#: filename of the environment file used to store the bot's settings
DOTENV_FILE_NAME: ClassVar[str] = "test-build.env"
_run_cmd: RunCommand = field(default_factory=lambda: RunCommand(logger=LOGGER))
def __post_init__(self) -> None:
if not self.branch_name:
self.branch_name = (
self.deployment_branch_name
+ "-"
+ "".join(random.choice(string.ascii_letters) for _ in range(5))
)
if not self.osc_username:
raise RuntimeError("osc_username is not set, cannot continue")
@property
def _bcis(self) -> Generator[BaseContainerImage, None, None]:
"""Generator yielding all
:py:class:`~bci_build.package.BaseContainerImage` that have the same
:py:attr:`~bci_build.package.BaseContainerImage.os_version` as this bot
instance.
"""
return (
bci
for bci in list(ALL_CONTAINER_IMAGE_NAMES.values()) + DOTNET_IMAGES
if bci.os_version == self.os_version
)
def _generate_project_name(self, prefix: str) -> str:
assert self.osc_username
res = f"home:{self.osc_username}:{prefix}:"
if self.os_version in (OsVersion.TUMBLEWEED, OsVersion.BASALT):
res += str(self.os_version)
else:
res += f"SLE-15-SP{str(self.os_version)}"
return res
@property
def continuous_rebuild_project_name(self) -> str:
"""The name of the continuous rebuild project on OBS."""
return self._generate_project_name("BCI:CR")
@property
def staging_project_name(self) -> str:
"""The name of the staging project on OBS.
It is constructed as follows:
``home:$OSC_USER:BCI:Staging:$OS_VER:$BRANCH`` where:
- ``OSC_USER``: :py:attr:`osc_username`
- ``OS_VER``: :py:attr:`os_version`
- ``BRANCH``: :py:attr:`branch_name`
"""
return self._generate_project_name("BCI:Staging") + ":" + self.branch_name
@property
def staging_project_url(self) -> str:
"""URL to the staging project."""
return get_obs_project_url(self.staging_project_name)
@property
def deployment_branch_name(self) -> str:
"""The name of the branch for this :py:attr:`~StagingBot.os_version`
where the build recipes are checked out by default.
"""
return (
str(self.os_version)
if self.os_version in (OsVersion.TUMBLEWEED, OsVersion.BASALT)
else f"sle15-sp{str(self.os_version)}"
)
@property
def package_names(self) -> list[str] | None:
"""Name of the packages in the staging project or ``None`` if the
staging project has not been setup yet.
"""
return self._packages
@package_names.setter
def package_names(self, pkgs: list[str] | None) -> None:
bci_pkg_names = [bci.package_name for bci in self._bcis]
if pkgs is not None:
for pkg in pkgs:
if pkg not in bci_pkg_names:
raise ValueError(
f"Invalid package name {pkg}, does not belong to the current os_version ({self.os_version})"
)
self._packages = pkgs
@property
def bcis(self) -> Generator[BaseContainerImage, None, None]:
"""Generator for creating an iterable yielding all
:py:class:`~bci_build.package.BaseContainerImage` that are in the bot's
staging project.
"""
return (
bci
for bci in self._bcis
if (
bci.package_name in self._packages
if self._packages is not None
else True
)
)
@staticmethod
def from_github_comment(comment_text: str, osc_username: str) -> "StagingBot":
if comment_text == "":
raise ValueError("Received empty github comment, cannot create the bot")
# comment_text looks like this:
# Created a staging project on OBS for 4: [home:defolos:BCI:Staging:SLE-15-SP4:sle15-sp4-HsmtR](url/to/proj)
# Changes pushed to branch [`sle15-sp4-HsmtR`](url/to/branch)
lines = comment_text.strip().splitlines()
proj_line = lines[0]
CREATED_TEXT = "Created a staging project on OBS for "
if CREATED_TEXT not in proj_line:
raise ValueError(f"Invalid first line in the comment: {comment_text}")
os_ver, prj_markdown_link = proj_line.replace(CREATED_TEXT, "").split(": ")
CHANGES_TEXT = "Changes pushed to branch "
branch_line = lines[1]
if CHANGES_TEXT not in branch_line:
raise ValueError(f"Invalid second line in the comment: {comment_text}")
branch_link_markdown = branch_line.replace(CHANGES_TEXT, "")
branch = branch_link_markdown.split("`]")[0].replace("[`", "")
bot = StagingBot(
os_version=OsVersion.parse(os_ver),
branch_name=branch,
osc_username=osc_username,
)
assert bot.staging_project_name == (
prj := prj_markdown_link.split("]")[0].replace("[", "")
), f"Mismatch between the constructed project name ({bot.staging_project_name}) and the project name from the comment ({prj})"
return bot
@staticmethod
async def from_env_file() -> "StagingBot":
"""Read the last saved settings from the environment file
(:py:attr:`~StagingBot.DOTENV_FILE_NAME`) in the current working
directory and create a :py:class:`StagingBot` from them.
"""
async with aiofiles.open(StagingBot.DOTENV_FILE_NAME, "r") as dot_env:
env_file = await dot_env.read()
branch, os_version, _, osc_username, _, _, _, repos, pkgs = [
line.split("=")[1] for line in env_file.strip().splitlines()
]
packages: list[str] | None = None if pkgs == "None" else pkgs.split(",")
stg_bot = StagingBot(
os_version=OsVersion.parse(os_version),
osc_username=osc_username,
branch_name=branch,
repositories=repos.split(","),
)
stg_bot.package_names = packages
return stg_bot
@property
def obs_workflows_yml(self) -> str:
"""The contents of :file:`.obs/workflows.yml` for branching each package
from the continuous rebuild project
(:py:attr:`~StagingBot.continuous_rebuild_project_name`) to the staging
sub-project.
"""
workflows = """---
staging_build:
steps:
"""
source_project = self.continuous_rebuild_project_name
for bci in self._bcis:
workflows += f""" - branch_package:
source_project: {source_project}
source_package: {bci.package_name}
target_project: {source_project}:Staging
"""
workflows += """ filters:
event: pull_request
refresh_devel_BCI:
steps:
"""
devel_prj = _get_bci_project_name(self.os_version)
for bci in self._bcis:
workflows += f""" - trigger_services:
project: {devel_prj}
package: {bci.package_name}
"""
workflows += f""" filters:
event: push
branches:
only:
- {self.deployment_branch_name}
"""
return workflows
@property
def changelog_check_github_action(self) -> str:
return (
r"""---
name: Check the changelogs
on:
pull_request:
jobs:
changelog-check:
name: changelog check
runs-on: ubuntu-22.04
container: ghcr.io/dcermak/bci-ci:latest
steps:
- uses: actions/checkout@v3
with:
ref: main
fetch-depth: 0
- uses: actions/cache@v3
with:
path: ~/.cache/pypoetry/virtualenvs
key: poetry-${{ hashFiles('poetry.lock') }}
- name: install python dependencies
run: poetry install
- name: fix the file permissions of the repository
run: chown -R $(id -un):$(id -gn) .
- name: fetch all branches
run: git fetch
- name: check the changelog
run: |
poetry run scratch-build-bot \
--os-version """
+ str(self.os_version)
+ r""" -vvvv \
changelog_check \
--base-ref origin/${{ github.base_ref }} \
--head-ref ${{ github.event.pull_request.head.sha }}
env:
OSC_USER: "irrelevant"
"""
)
@property
def find_missing_packages_action(self) -> str:
return (
r"""---
name: Check whether packages are missing on OBS
on:
push:
branches:
- '"""
+ self.deployment_branch_name
+ """'
jobs:
create-issues-for-dan:
name: create an issue for Dan to create the packages in devel:BCI
runs-on: ubuntu-latest
container: ghcr.io/dcermak/bci-ci:latest
strategy:
fail-fast: false
steps:
# we need all branches for the build checks
- uses: actions/checkout@v3
with:
fetch-depth: 0
ref: main
token: ${{ secrets.CHECKOUT_TOKEN }}
- uses: actions/cache@v3
with:
path: ~/.cache/pypoetry/virtualenvs
key: poetry-${{ hashFiles('poetry.lock') }}
- name: fix the file permissions of the repository
run: chown -R $(id -un):$(id -gn) .
- name: install python dependencies
run: poetry install
- name: find the packages that are missing
run: |
pkgs=$(poetry run scratch-build-bot --os-version """
+ str(self.os_version)
+ """ find_missing_packages)
if [[ ${pkgs} = "" ]]; then
echo "missing_pkgs=false" >> $GITHUB_ENV
else
echo "missing_pkgs=true" >> $GITHUB_ENV
echo "pkgs=${pkgs}" >> $GITHUB_ENV
fi
cat test-build.env >> $GITHUB_ENV
env:
OSC_PASSWORD: ${{ secrets.OSC_PASSWORD }}
OSC_USER: "defolos"
- uses: JasonEtco/create-an-issue@v2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
update_existing: true
filename: ".github/create-package.md"
if: env.missing_pkgs == 'true'
"""
)
async def setup(self) -> None:
if pw := os.getenv(OSC_PASSWORD_ENVVAR_NAME):
osc_conf = tempfile.NamedTemporaryFile("w", delete=False)
osc_conf.write(
f"""[general]
apiurl = https://api.opensuse.org
[https://api.opensuse.org]
user = {self.osc_username}
pass = {pw}
aliases = obs
"""
)
osc_conf.flush()
self._osc_conf_file = osc_conf.name
self._xdg_state_home_dir = tempfile.TemporaryDirectory()
self._run_cmd = RunCommand(
logger=LOGGER, env={"XDG_STATE_HOME": self._xdg_state_home_dir.name}
)
await self.write_env_file()
async def write_env_file(self):
async with aiofiles.open(self.DOTENV_FILE_NAME, "w") as dot_env:
await dot_env.write(
f"""{BRANCH_NAME_ENVVAR_NAME}={self.branch_name}
{OS_VERSION_ENVVAR_NAME}={self.os_version}
OS_VERSION_PRETTY={self.os_version.pretty_print}
{OSC_USER_ENVVAR_NAME}={self.osc_username}
DEPLOYMENT_BRANCH_NAME={self.deployment_branch_name}
PROJECT_NAME={self.staging_project_name}
PROJECT_URL={self.staging_project_url}
REPOSITORIES={','.join(self.repositories)}
PACKAGES={','.join(self.package_names) if self.package_names else None}
"""
)
async def teardown(self) -> None:
"""Local cleanup after the bot (nothing on OBS is removed and the git
repository itself is left untouched).
"""
if self._osc_conf_file:
await aiofiles.os.remove(self._osc_conf_file)
assert self._xdg_state_home_dir is not None
tasks = []
for suffix in ("", ".lock"):
tasks.append(
ensure_absent(
os.path.join(
self._xdg_state_home_dir.name, "osc", f"cookiejar{suffix}"
)
)
)
await asyncio.gather(*tasks)
await ensure_absent(os.path.join(self._xdg_state_home_dir.name, "osc"))
self._xdg_state_home_dir.cleanup()
@property
def _osc(self) -> str:
"""command to invoke osc (may include a CLI flag for the custom config file)"""
return (
"osc" if not self._osc_conf_file else f"osc --config={self._osc_conf_file}"
)
async def _generate_test_project_meta(self, target_project_name: str) -> ET.Element:
bci_devel_meta = ET.fromstring(
await _fetch_bci_devel_project_config(self.os_version, "meta")
)
# write the same project meta as devel:BCI, but replace the 'devel:BCI:*'
# with the target project name in the main element and in all repository
# path entries
bci_devel_meta.attrib["name"] = target_project_name
# we will remove the helmchartsrepo as we do not need it
repo_names = []
repos_to_remove = []
# ppc64le & s390x are mostly busted on TW and just cause pointless
# build failures, so we don't build them
# Also, we don't use the local architecture, so drop that one always
arches_to_drop = [str(Arch.LOCAL)]
arches_to_drop.extend(
[str(Arch.PPC64LE), str(Arch.S390X)]
if self.os_version == OsVersion.TUMBLEWEED
else []
)
for elem in bci_devel_meta:
if elem.tag == "repository":
if "name" in elem.attrib:
if (name := elem.attrib["name"]) in ("helmcharts", "standard"):
if name == "helmcharts":
repos_to_remove.append(elem)
continue
repo_names.append(name)
else:
raise ValueError(
f"Invalid <repository> element, missing 'name' attribute: {ET.tostring(elem).decode()}"
)
for repo_elem in elem.iter(tag="path"):
if (
"project" in repo_elem.attrib
and "devel:BCI:" in repo_elem.attrib["project"]
):
repo_elem.attrib["project"] = target_project_name
arch_entries = list(elem.iter(tag="arch"))
for arch_entry in arch_entries:
if arch_entry.text in arches_to_drop:
elem.remove(arch_entry)
container_repos = ("containerfile", "images")
if name in container_repos:
for repo_name in container_repos:
(bci_devel_prj_path := ET.Element("path")).attrib[
"project"
] = _get_bci_project_name(self.os_version)
bci_devel_prj_path.attrib["repository"] = repo_name
elem.insert(0, bci_devel_prj_path)
self.repositories = repo_names
for repo_to_remove in repos_to_remove:
bci_devel_meta.remove(repo_to_remove)
person = ET.Element(
"person", {"userid": self.osc_username, "role": "maintainer"}
)
bci_devel_meta.append(person)
return bci_devel_meta
async def _send_prj_meta(
self, target_project_name: str, prj_meta: ET.Element
) -> None:
"""Set the meta of the project on OBS with the name
``target_project_name`` to the config ``prj_meta``.
"""
async with aiofiles.tempfile.NamedTemporaryFile(mode="wb") as tmp_meta:
await tmp_meta.write(ET.tostring(prj_meta))
await tmp_meta.flush()
async def _send_prj_meta():
await self._run_cmd(
f"{self._osc} meta prj --file={tmp_meta.name} {target_project_name}"
)
# obs sometimes dies setting the project meta with SQL errors 🤯
# so we just try again…
await retry_async_run_cmd(_send_prj_meta)
async def write_cr_project_config(self) -> None:
"""Send the configuration of the continuous rebuild project to OBS.
This will create the project if it did not exist already. If it exists,
then its configuration (= ``meta`` in OBS jargon) will be updated.
"""
meta = await self._generate_test_project_meta(
self.continuous_rebuild_project_name
)
(
scmsync := ET.Element("scmsync")
).text = f"https://github.com/SUSE/bci-dockerfile-generator#{self.deployment_branch_name}"
meta.append(scmsync)
await self._send_prj_meta(self.continuous_rebuild_project_name, meta)
async def write_staging_project_configs(self) -> None:
"""Submit the ``prjconf`` and ``meta`` to the test project on OBS.
The ``prjconf`` is taken directly from the development project on OBS
(``devel:BCI:*``).
The ``meta`` has to be modified slightly:
- we remove the ``helmcharts`` repository (we don't create anything for
that repo, so not worth creating it)
- change the path from ``devel:BCI:*`` to the staging project name
- add the bot user as the maintainer (otherwise you can't do anything in
the project anymore…)
Then we send the ``meta`` and then the ``prjconf``.
"""
confs: _ProjectConfigs = {}
async def _fetch_prjconf():
confs["prjconf"] = await _fetch_bci_devel_project_config(
self.os_version, "prjconf"
)
async def _fetch_prj():
confs["meta"] = await self._generate_test_project_meta(
self.staging_project_name
)
await asyncio.gather(_fetch_prj(), _fetch_prjconf())
# First set the project meta! This will create the project if it does not
# exist already, if we do it asynchronously, then the prjconf might be
# written before the project exists, which fails
await self._send_prj_meta(self.staging_project_name, confs["meta"])
async with aiofiles.tempfile.NamedTemporaryFile(mode="w") as tmp_prjconf:
await tmp_prjconf.write(confs["prjconf"])
await tmp_prjconf.flush()
await self._run_cmd(
f"{self._osc} meta prjconf --file={tmp_prjconf.name} {self.staging_project_name}"
)
def _osc_fetch_results_cmd(self, extra_osc_flags: str = "") -> str:
return (
f"{self._osc} results --xml {extra_osc_flags} "
+ " ".join("--repo=" + repo_name for repo_name in self.repositories)
+ f" {self.staging_project_name}"
)
async def remote_cleanup(
self, branches: bool = True, obs_project: bool = True
) -> None:
"""Deletes the branch with the test commit locally and on the remote and
removes the staging project.
All performed actions are permitted to fail without raising an exception
to ensure that a partially setup test run is cleaned up as much as
possible.
Args:
branches: if ``True``, removes the branch locally and on the remote
(defaults to ``True``)
obs_project: if ``True``, removes the staging project on OBS
(defaults to ``True``)
"""
async def remove_branch():
await self._run_cmd(
f"git branch -D {self.branch_name}", raise_on_error=False
)
await self._run_cmd(
f"git push origin -d {self.branch_name}", raise_on_error=False
)
tasks = []
if branches:
tasks.append(remove_branch())
if obs_project:
tasks.append(
self._run_cmd(
f"{self._osc} rdelete -m 'cleanup' --recursive --force {self.staging_project_name}",
raise_on_error=False,
)
)
await asyncio.gather(*tasks)
async def _write_pkg_meta(
self, bci_pkg: BaseContainerImage, target_obs_project: str, git_branch_name
) -> None:
"""Write the package ``_meta`` of the package with the name of the
``bci_pkg`` in the ``target_obs_project`` to be synced from the git
branch ``git_branch_name``.
"""
(pkg_conf := ET.Element("package")).attrib["name"] = bci_pkg.package_name
(title := ET.Element("title")).text = bci_pkg.title
(descr := ET.Element("description")).text = bci_pkg.description
(
scmsync := ET.Element("scmsync")
).text = f"https://github.com/SUSE/bci-dockerfile-generator?subdir={bci_pkg.package_name}#{git_branch_name}"
for elem in (title, descr, scmsync):
pkg_conf.append(elem)
async with aiofiles.tempfile.NamedTemporaryFile(mode="w") as tmp_pkg_conf:
await tmp_pkg_conf.write(ET.tostring(pkg_conf).decode())
await tmp_pkg_conf.flush()
await self._run_cmd(
f"{self._osc} meta pkg --file={tmp_pkg_conf.name} {target_obs_project} {bci_pkg.package_name}"
)
async def link_base_container_to_staging(self) -> None:
"""Links the base container for this os into
:py:attr:`StagingBot.staging_project_name`. This function does nothing
if the current :py:attr:`~StagingBot.os_version` is not in the list
:py:const:`OS_VERSION_NEEDS_BASE_CONTAINER`.
"""
if self.os_version not in OS_VERSION_NEEDS_BASE_CONTAINER:
return
prj, pkg = _get_base_image_prj_pkg(self.os_version)
await self._run_cmd(
f"{self._osc} linkpac {prj} {pkg} {self.staging_project_name}"
)
async def write_pkg_configs(
self,
packages: Iterable[BaseContainerImage],
git_branch_name: str,
target_obs_project: str,
) -> None:
"""Write all package configurations (= :file:`_meta`) for every package
in ``packages`` to the project `target_obs_project` so that the package
is fetched via the scm bridge from the branch `git_branch_name`.
Args:
packages: the BCI packages that should be added
git_branch_name: the name of the git branch from which the sources
will be retrieved
target_obs_project: name of the project on OBS to which the packages
will be added
"""
tasks = [
self._write_pkg_meta(
bci,
git_branch_name=git_branch_name,
target_obs_project=target_obs_project,
)
for bci in packages
]
for bci in packages:
tasks.append(
self._write_pkg_meta(
bci,
git_branch_name=git_branch_name,
target_obs_project=target_obs_project,
)
)
await asyncio.gather(*tasks)
def _get_changed_packages_by_commit(self, commit: str | git.Commit) -> list[str]:
git_commit = (
commit if isinstance(commit, git.Commit) else git.Repo(".").commit(commit)
)
bci_pkg_names = [bci.package_name for bci in self.bcis]
packages = []
# get the diff between the commit and the deployment branch on the remote
# => list of changed files
# each file's first path element is the package name -> save that in
# `packages`
for diff in git_commit.diff(f"origin/{self.deployment_branch_name}"):
# no idea how this could happen, but in theory the diff mode can be
# `C` for conflict => abort if that's the case
assert (
diff.a_mode != "C" and diff.b_mode != "C"
), f"diff must not be a conflict, but got {diff=}"
if (
(a_path := os.path.split(diff.a_path))
and a_path[0] in bci_pkg_names
and (b_path := os.path.split(diff.b_path))
and b_path[0] in bci_pkg_names
):
packages.append(a_path[0])
# account for files getting moved
if b_path[0] != a_path[0]:
packages.append(b_path[0])
res = list(set(packages))
# it can happen that we only update a non-BCI package file,
# e.g. .obs/workflows.yml, then we will have a commit, but the diff will
# not touch any BCI and thus `res` will be an empty list
# => give reduce an initial value (last parameter) as it will otherwise
# fail
assert reduce(lambda l, r: l and r, (p in bci_pkg_names for p in res), True)
return res
async def _run_git_action_in_worktree(
self,
new_branch_name: str,
origin_branch_name: str,
action: Callable[[str], Coroutine[None, None, bool]],
) -> str | None:
assert not origin_branch_name.startswith("origin/")
await self._run_cmd(
f"git worktree add -B {new_branch_name} {new_branch_name} origin/{origin_branch_name}"
)
worktree_dir = os.path.join(os.getcwd(), new_branch_name)
commit = None
try:
if await action(worktree_dir):
commit = (
await self._run_cmd(
"git show -s --pretty=format:%H HEAD", cwd=worktree_dir
)
).stdout.strip()
LOGGER.info("Created commit %s given the current state", commit)
await self._run_cmd(
f"git pull --rebase origin {origin_branch_name}",
cwd=worktree_dir,
env={**_GIT_COMMIT_ENV, **os.environ},
)
await self._run_cmd(
"git push --force-with-lease origin HEAD", cwd=worktree_dir
)
finally:
await self._run_cmd(f"git worktree remove --force {new_branch_name}")
return commit
async def write_all_build_recipes_to_branch(
self, commit_msg: str = ""
) -> str | None:
"""Creates a worktree for the branch based on the deployment branch of
this :py:attr:`~StagingBot.os_version`, writes all image build recipes
into it, commits and then pushes the changes. Afterwards the worktree is
removed.
Returns:
The hash of the commit including all changes by the writing the
build recipes into the worktree. If no changes were made, then
``None`` is returned.
"""
async def _write_build_recipes_in_worktree(worktree_dir: str) -> bool:
files = await self.write_all_image_build_recipes(worktree_dir)
run_in_worktree = RunCommand(cwd=worktree_dir, logger=LOGGER)
worktree = git.Repo(worktree_dir)
# no changes => nothing to do & bail
# note: diff only checks for changes to *existing* packages, but not
# if anything new got added => need to check if the repo is dirty
# for that (= is there any output with `git status`) or there any
# untracked files
if (
not worktree.head.commit.diff()
and not worktree.is_dirty()
and not worktree.untracked_files
):
LOGGER.info("Writing all build recipes resulted in no changes")
return False
await run_in_worktree("git add " + " ".join(files))
await run_in_worktree(
f"git commit -m '{commit_msg or 'Test build'}'",
env={**_GIT_COMMIT_ENV, **os.environ},
)
return True
# we do not want to overwrite any existing changes from the origin which
# could have been pushed there already
# is there a origin/self.branch_name?
branch_commit_hash_on_remote: str | None = None
try:
branch_commit_hash_on_remote = (
git.Repo(".").commit(f"origin/{self.branch_name}").hexsha
)
except git.BadName:
pass
# yes => check if it is newer than the deployment branch
commit_range = None
if branch_commit_hash_on_remote:
try:
commit_range = self._get_commit_range_between_refs(
branch_commit_hash_on_remote,
f"origin/{self.deployment_branch_name}",
)
except RecursionError:
pass
# it is newer? => base our work on origin/branch_name and not the
# deployment_branch
origin_branch = (
self.deployment_branch_name if not commit_range else self.branch_name
)
return await self._run_git_action_in_worktree(
self.branch_name,
origin_branch,
_write_build_recipes_in_worktree,
)
async def write_all_image_build_recipes(
self, destination_prj_folder: str
) -> list[str]:
"""Writes all build recipes into the folder
:file:`destination_prj_folder` and returns a list of files that were
written.
Returns:
A list of all files that were written to
:file:`destination_prj_folder`. The files are listed relative to
:file:`destination_prj_folder` and don't contain any leading path
components.
"""