Update steps to support Buildbot 4.1.0

Signed-off-by: Magnus Granberg <zorry@gentoo.org>

6 files changed, 148 insertions, 30 deletions

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py
index ab80e19..fffa281 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -172,6 +172,8 @@ def parse_build_log():
     f.addStep(logs.setEmergeInfoLog())
     # add package info to log and db
     f.addStep(logs.setPackageInfoLog())
+    # save build steps log to a file
+    f.addStep(logs.writeStepLogsToFile())
     # make the bugreport
     f.addStep(logs.SetupBugReportSteps())
     # set BuildStatus
diff --git a/buildbot_gentoo_ci/steps/bugs.py b/buildbot_gentoo_ci/steps/bugs.py
index 52bd08d..efff7f5 100644
--- a/buildbot_gentoo_ci/steps/bugs.py
+++ b/buildbot_gentoo_ci/steps/bugs.py
@@ -18,7 +18,7 @@ from buildbot.plugins import util
 
 from buildbot_gentoo_ci.utils.regex import stripQuotesAndMore, finishTitle
 
-from bugz.cli import check_bugz_token, login, list_bugs
+from bugz.cli import check_bugz_token, check_auth, list_bugs
 from bugz.cli_argparser import make_arg_parser
 from bugz.configfile import load_config
 from bugz.settings import Settings
@@ -64,7 +64,7 @@ def search_bugz(args):
     for key in params:
         log_info('   {0:<20} = {1}'.format(key, params[key]))
 
-    login(settings)
+    check_auth(settings)
 
     result = settings.call_bz(settings.bz.Bug.search, params)['bugs']
 
diff --git a/buildbot_gentoo_ci/steps/clean.py b/buildbot_gentoo_ci/steps/clean.py
index e4d8573..fb28973 100644
--- a/buildbot_gentoo_ci/steps/clean.py
+++ b/buildbot_gentoo_ci/steps/clean.py
@@ -10,6 +10,7 @@ from buildbot.process.results import FAILURE
 from buildbot.process.results import SKIPPED
 from buildbot.plugins import steps, util
 
+from buildbot_gentoo_ci.db.builds import _db2data_ProjectsBuilds
 class SetupPropertys(BuildStep):
     name = 'Setup propertys for clean db'
     description = 'Running'
@@ -40,7 +41,8 @@ class SetupPropertys(BuildStep):
                 # get build_data for deleted_ebuild_data['uuid']
                 build_data_list = yield self.gentooci.db.builds.getBuildsByVersionUuid(ebuild_data['uuid'])
                 print(build_data_list)
-                for build_data in build_data_list:
+                for build_data_model in build_data_list:
+                    build_data = _db2data_ProjectsBuilds(build_data_model)
                     if build_data['bug_id'] == 0:
                         print(f"Build : {build_data['id']} will be removed")
                         if build_data['buildbot_build_id'] != 0:
@@ -72,6 +74,11 @@ class RemoveEbuildFromDb(BuildStep):
     @defer.inlineCallbacks
     def run(self):
         self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+        build_data_list = yield self.gentooci.db.builds.getBuildsByVersionUuid(self.uuid)
+        print(build_data_list)
+        for build_data_model in build_data_list:
+            build_data = _db2data_ProjectsBuilds(build_data_model)
+            yield self.gentooci.db.builds.removeBuild(build_data['id'])
         yield self.gentooci.db.versions.removeVersionKeyword(self.uuid)
         yield self.gentooci.db.versions.removeVersionMetadata(self.uuid)
         yield self.gentooci.db.versions.removeVersion(self.uuid)
@@ -112,7 +119,12 @@ class RemoveBuildFromDb(BuildStep):
         self.descriptionDone = f"BuildBot Build : {self.build_id} will be removed"
         bb_build_data = yield self.master.db.builds.getBuild(self.build_id)
         # remove steps and logs
-        yield self.master.db.logs.deleteLogChunks(self.build_id)
+        steps_data = yield self.master.db.steps.getSteps(self.build_id)
+        for step_data in steps_data:
+            yield self.master.db.logs.deleteLog(step_data['id'])
+            logs_data = yield self.master.db.logs.getLogs(step_data['id'])
+            for log_data in logs_data:
+                yield self.master.db.logs.deleteLogChunks(log_data['id'])
         # remove propertys
         yield self.master.db.builds.pruneBuildProperties(self.build_id)
         # remove buildset_sourcestamps
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 6b8eba5..fdd0f3e 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -1,6 +1,9 @@
-# Copyright 2023 Gentoo Authors
+# Copyright 2024 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
 import os
 import re
 import gzip
@@ -27,6 +30,39 @@ from buildbot.plugins import util
 from buildbot_gentoo_ci.steps import master as master_steps
 from buildbot_gentoo_ci.steps import bugs
 from buildbot_gentoo_ci.utils.regex import stripQuotesAndMore, finishTitle
+from buildbot_gentoo_ci.db.versions import _db2data_Version
+from buildbot_gentoo_ci.db.packages import _db2data_Package
+
+from buildbot.db.logs import LogModel
+from buildbot.db.steps import StepModel
+
+def _db2data_Log(model: LogModel):
+    if model is None:
+        return None
+    return {
+        'logid': model.id,
+        'name': model.name,
+        'slug': model.slug,
+        'stepid': model.stepid,
+        'complete': model.complete,
+        'num_lines': model.num_lines,
+        'type': model.type,
+    }
+
+def _db2data_Step(model: StepModel):
+    return {
+        'stepid': model.id,
+        'number': model.number,
+        'name': model.name,
+        'buildid': model.buildid,
+        'started_at': model.started_at,
+        'locks_acquired_at': model.locks_acquired_at,
+        'complete_at': model.complete_at,
+        'state_string': model.state_string,
+        'results': model.results,
+        'urls': [{'name': item.name, 'url': item.url} for item in model.urls],
+        'hidden': model.hidden,
+    }
 
 def PersOutputOfLogParser(rc, stdout, stderr):
     build_summery_output = {}
@@ -103,12 +139,14 @@ class SetupPropertys(BuildStep):
     @defer.inlineCallbacks
     def run(self):
         self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+        print(f"Project_build_data:{self.getProperty('project_build_data')}")
         project_data = yield self.gentooci.db.projects.getProjectByUuid(self.getProperty('project_build_data')['project_uuid'])
         default_project_data = yield self.gentooci.db.projects.getProjectByName(self.gentooci.config.project['project']['update_db'])
         version_data = yield self.gentooci.db.versions.getVersionByUuid(self.getProperty('project_build_data')['version_uuid'])
+        print(f"Version_data:{version_data}")
         self.setProperty("project_data", project_data, 'project_data')
         self.setProperty("default_project_data", default_project_data, 'default_project_data')
-        self.setProperty("version_data", version_data, 'version_data')
+        self.setProperty("version_data", _db2data_Version(version_data), 'version_data')
         self.setProperty("status", 'completed', 'status')
         if isinstance(self.getProperty('faild_cpv'), str):
             log_cpv = self.getProperty('faild_cpv')
@@ -119,21 +157,6 @@ class SetupPropertys(BuildStep):
         self.descriptionDone = 'Runing log checker on ' + log_cpv
         logsdir = yield os.path.join(self.master.basedir, 'workers', self.getProperty('build_workername'), str(self.getProperty("project_build_data")['buildnumber']))
         self.setProperty("logsdir", logsdir, 'logsdir')
-        # get steps/log id's from build
-        build_log_steps_data = {}
-        steps_data = yield self.master.db.steps.getSteps(self.getProperty("project_build_data")['buildbot_build_id'])
-        for step in steps_data:
-            log_data = None
-            logs_data = yield self.master.db.logs.getLogs(step['id'])
-            for log_info in logs_data:
-                if log_info['name'] != 'property changes':
-                    log_data = log_info
-            step_info = {}
-            step_info['name'] = step['name']
-            step_info['number'] = step['number']
-            step_info['log_data'] = log_data
-            build_log_steps_data[step['id']] = step_info
-        self.setProperty("build_log_steps_data", build_log_steps_data, 'build_log_steps_data')
         return SUCCESS
 
 class SetupParserBuildLoger(BuildStep):
@@ -539,15 +562,32 @@ class writeStepLogsToFile(BuildStep):
 
     @defer.inlineCallbacks
     def run(self):
+        # get steps/log id's from build
+        build_log_steps_data = {}
+        steps_data = yield self.master.db.steps.getSteps(self.getProperty("project_build_data")['buildbot_build_id'])
+        for dbdict_step in steps_data:
+            step = _db2data_Step(dbdict_step)
+            logs_data = yield self.master.db.logs.getLogs(step['stepid'])
+            log_data = None
+            for dbdict_log_info in logs_data:
+                log_info = _db2data_Log(dbdict_log_info)
+                if log_info['name'] != 'property changes':
+                    log_data = log_info
+            step_info = {}
+            step_info['name'] = step['name']
+            step_info['number'] = step['number']
+            step_info['log_data'] = log_data
+            build_log_steps_data[step['stepid']] = step_info
+        self.setProperty("build_log_steps_data", build_log_steps_data, 'build_log_steps_data')
         buildsteplogfile_path = yield os.path.join(self.getProperty('logsdir'), 'buildsteps.log.xz')
         stepstologlist = ['Run emerge step build', 'Run emerge step build_1']
         logs_texts = []
-        for k, v in self.getProperty('build_log_steps_data').items():
+        for k, v in build_log_steps_data.items():
             if v['name'] in stepstologlist:
                 logs_texts.append(f">>> BEGINING OF STEP: {v['name']}")
                 logs_texts.append('')
                 log_data = v['log_data']
-                log_text = yield self.master.db.logs.getLogLines(log_data['id'], 1, log_data['num_lines'])
+                log_text = yield self.master.db.logs.getLogLines(log_data['logid'], 1, log_data['num_lines'])
                 for line in log_text.split('\n'):
                     if line.startswith('h  BUILDMASTER='):
                         line = 'h  BUILDMASTER=XXX.XXX.XXX.XXX'
@@ -569,11 +609,56 @@ class SetupBugReportSteps(BuildStep):
         super().__init__(**kwargs)
 
     @defer.inlineCallbacks
+    def getCCAndAssigned_to(self):
+        emails_dict = self.getProperty('emails')
+        packages_emails = emails_dict['packages_emails']
+        change_emails = emails_dict['change']
+        if self.getProperty('failed_version_data'):
+            change_id = self.getProperty('failed_version_data')['change_id']
+        else:
+            change_id = self.getProperty('version_data')['change_id']
+        change = yield self.master.db.changes.getChange(change_id)
+        stab_key = False
+        bug_mails = []
+        for word in ['Stabilize', 'stabilize', 'Keyword', 'keyword']:
+            if word in change['comments']:
+                stab_key = True
+        change_author = change_emails['authors'].split(' <')[1][:-1]
+        if 'proxy-maint@gentoo.org' or 'maintainer-needed@gentoo.org' in packages_emails:
+            if not change_author.endswith('@gentoo.org') and not stab_key:
+                bug_mails.append(change_author)
+            for package_email in packages_emails:
+                if not package_email['email'].endswith('@gentoo.org') and package_email['email'] not in bug_mails:
+                    bug_mails.append(package_email['email'])
+            for package_email in packages_emails:
+                if bug_mails == []:
+                    if package_email['mail_type'] == 'project':
+                        bug_mails.append(package_email['email'])
+        elif 'proxy-maint@gentoo.org' or 'maintainer-needed@gentoo.org' not in packages_emails:
+            if change_author in package_email:
+                bug_mails.append(change_author)
+            if not change_author.endswith('@gentoo.org') not in packages_emails:
+                bug_mails.append(change_author)
+            for package_email in packages_emails:
+                if bug_mails == []:
+                    if package_email['mail_type'] == 'project':
+                        bug_mails.append(package_email['email'])
+            if change_author not in bug_mails and not stab_key:
+                bug_mails.append(change_author)
+        for package_email in packages_emails:
+            if not package_email['email'] in bug_mails:
+                bug_mails.append(package_email['email'])
+        print(f"bug_mails: {bug_mails}")
+        self.setProperty("bug_mails", bug_mails, 'bug_mails')
+        return bug_mails
+
+    @defer.inlineCallbacks
     def run(self):
         self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
         bug_config = self.gentooci.config.project['bug_config']
         make_bugreport = False
         print(bug_config)
+        bug_mails = self.getCCAndAssigned_to()
         if bug_config['enable'] and self.getProperty('status') == 'failed' and self.getProperty('error_dict')['title_found'] and not self.getProperty('bgo'):
             # add bug enable profile, repo and project
             bug_settings_data = yield self.gentooci.db.projects.getBugsSettingsByProjectUuid(self.getProperty("project_data")['uuid'])
@@ -581,9 +666,12 @@ class SetupBugReportSteps(BuildStep):
                 version_data = self.getProperty('failed_version_data')
             else:
                 version_data = self.getProperty("version_data")
-            package_data = yield self.gentooci.db.packages.getPackageByUuid(version_data['package_uuid'])
-            project_repository_settings = yield self.gentooci.db.projects.getRepositorySettingByProjUuidAndRepoUuid(self.getProperty("project_data")['uuid'], package_data['repository_uuid'])
-            print(bug_settings_data)
+            print(version_data)
+            dbdict_package_data = yield self.gentooci.db.packages.getPackageByUuid(version_data['package_uuid'])
+            package_data = _db2data_Package(dbdict_package_data)
+            print(package_data)
+            project_repository_settings = yield self.gentooci.db.projects.getRepositorySettingByProjUuidAndRepoUuid(
+                self.getProperty("project_data")['uuid'], package_data['repository_uuid'])
             print(project_repository_settings)
             if bug_settings_data['enabled'] and project_repository_settings['bug']:
                 make_bugreport = True
@@ -643,6 +731,10 @@ class SetupBugReportSteps(BuildStep):
                 bug_params4['filename'] = yield os.path.join(self.getProperty('logsdir'), filename)
                 #bug_params4['comment'] = filename
                 aftersteps_list.append(bugs.Attach(bug_args, bug_params4))
+            bug_params5 = {}
+            #bug_params5['assigned_to'] = assigned_to
+            #bug_params5['cc'] = cc_list
+            #aftersteps_list.append(bugs.Modify(bug_args, bug_params5))
             yield self.build.addStepsAfterCurrentStep(aftersteps_list)
             return SUCCESS
         return SKIPPED
diff --git a/buildbot_gentoo_ci/steps/package.py b/buildbot_gentoo_ci/steps/package.py
index 847ba12..bd42a48 100644
--- a/buildbot_gentoo_ci/steps/package.py
+++ b/buildbot_gentoo_ci/steps/package.py
@@ -1,6 +1,9 @@
-# Copyright 2021 Gentoo Authors
+# Copyright 2024 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
+from __future__ import annotations
+from typing import TYPE_CHECKING
+
 import re
 import os
 
@@ -16,6 +19,8 @@ from buildbot.process.results import SKIPPED
 from buildbot.process import remotecommand
 from buildbot.plugins import steps
 
+from buildbot_gentoo_ci.db.packages import _db2data_Package
+
 class SetupPropertys(BuildStep):
     name = 'Setup propertys for CPV check'
     description = 'Running'
@@ -246,7 +251,7 @@ class CheckP(BuildStep):
             self.setProperty("package", self.package, 'package')
             yield self.build.addStepsAfterCurrentStep([AddPackage()])
         else:
-            self.setProperty("package_data", self.package_data, 'package_data')
+            self.setProperty("package_data", _db2data_Package(self.package_data), 'package_data')
         return SUCCESS
 
 class TriggerCheckForV(BuildStep):
@@ -306,8 +311,10 @@ class TriggerCleanPackageDb(BuildStep):
         days_in_s = 86400 * days
         needcleaning  = 0
         now = int(self.master.reactor.seconds())
+        print(f"Now: {now} Days: {days_in_s} Now - Days: {now - days_in_s} in s Old ebuilds: {len(deleted_ebuilds_data)}")
         for deleted_ebuild_data in deleted_ebuilds_data:
             # check if older the days
+            print(f"Ebuild deleted time in s: {deleted_ebuild_data['deleted_at']} Days old: {divmod(now - deleted_ebuild_data['deleted_at'], 86400)}")
             if deleted_ebuild_data['deleted_at'] < (now - days_in_s):
                 needcleaning = needcleaning  + 1
         print(f"{str(needcleaning)} ebuilds need cleaning in db")
diff --git a/buildbot_gentoo_ci/steps/version.py b/buildbot_gentoo_ci/steps/version.py
index 5f252c7..56f107c 100644
--- a/buildbot_gentoo_ci/steps/version.py
+++ b/buildbot_gentoo_ci/steps/version.py
@@ -1,6 +1,9 @@
-# Copyright 2021 Gentoo Authors
+# Copyright 2024 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
 import os
 
 from portage.versions import cpv_getversion, pkgsplit, catpkgsplit
@@ -17,6 +20,8 @@ from buildbot.process.results import SKIPPED
 from buildbot.process import remotecommand
 from buildbot.plugins import steps
 
+from buildbot_gentoo_ci.db.versions import _db2data_Version
+
 def GetPythonVersion(pyprop):
     py_list = pyprop.replace(' ', '').replace('\n ', '').split('.')
     return '.'.join([py_list[0], py_list[1]]).lower()
@@ -402,7 +407,7 @@ class SetupPropertys(BuildStep):
         print(self.version)
         self.old_version_data = yield self.gentooci.db.versions.getVersionByName(self.version, self.getProperty("package_data")['uuid'])
         print(self.old_version_data)
-        self.setProperty("old_version_data", self.old_version_data, 'old_version_data')
+        self.setProperty("old_version_data", _db2data_Version(self.old_version_data), 'old_version_data')
         self.setProperty("version", self.version, 'version')
         return SUCCESS