diff options
author | 2023-03-10 02:47:15 +0100 | |
---|---|---|
committer | 2023-03-10 02:47:15 +0100 | |
commit | 0b3facc196912a1665b951a570cbafe76038ba03 (patch) | |
tree | 2e6556a0249e24f451f776f56c1edde1b326c791 | |
parent | Change cleantime_package to 1w and disable eclean-dist (diff) | |
download | tinderbox-cluster-0b3facc196912a1665b951a570cbafe76038ba03.tar.gz tinderbox-cluster-0b3facc196912a1665b951a570cbafe76038ba03.tar.bz2 tinderbox-cluster-0b3facc196912a1665b951a570cbafe76038ba03.zip |
Add package metadata support (email)
Signed-off-by: Magnus Granberg <zorry@gentoo.org>
-rw-r--r-- | buildbot_gentoo_ci/config/buildfactorys.py | 3 | ||||
-rw-r--r-- | buildbot_gentoo_ci/db/model.py | 28 | ||||
-rw-r--r-- | buildbot_gentoo_ci/db/packages.py | 147 | ||||
-rw-r--r-- | buildbot_gentoo_ci/steps/package.py | 166 | ||||
-rw-r--r-- | sql/gentoo_ci_schema.sql | 201 |
5 files changed, 521 insertions, 24 deletions
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py index efccc93..2ca2ca9 100644 --- a/buildbot_gentoo_ci/config/buildfactorys.py +++ b/buildbot_gentoo_ci/config/buildfactorys.py @@ -58,13 +58,14 @@ def update_db_cpv(): f.addStep(category.CheckC()) # if package in db # return package_data - # add check package path step at end # else # add package to db step # return package_data f.addStep(package.CheckP()) # Trigger update_db_v f.addStep(package.TriggerCheckForV()) + # update metadata if needed + f.addStep(package.CheckMetadataPackagePath()) return f def update_db_v(): diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py index 4373ebb..6e13ad7 100644 --- a/buildbot_gentoo_ci/db/model.py +++ b/buildbot_gentoo_ci/db/model.py @@ -287,6 +287,34 @@ class Model(base.DBConnectorComponent): sa.Column('deleted_at', sa.Integer, nullable=True), ) + emails = sautils.Table( + "emails", metadata, + # unique uuid per keyword + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('email', sa.String(255), nullable=False), + ) + + packages_emails = sautils.Table( + "packages_emails", metadata, + # unique uuid per keyword + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('package_uuid', sa.String(36), + sa.ForeignKey('packages.uuid', ondelete='CASCADE')), + sa.Column('email_id', sa.Integer, + sa.ForeignKey('emails.id', ondelete='CASCADE')), + sa.Column('mail_type', sa.Enum('project', 'person', 'unknown',), nullable=False, default='unknown'), + sa.Column('proxied', sa.Enum('yes', 'no', 'proxy',), nullable=False, default='no'), + ) + + packages_metadata = sautils.Table( + "packages_metadata", metadata, + # unique uuid per keyword + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('package_uuid', sa.String(36), + sa.ForeignKey('packages.uuid', ondelete='CASCADE')), + sa.Column('sha256', sa.String(255), nullable=False, default='0'), + ) + versions = sautils.Table( "versions", metadata, sa.Column('uuid', sa.String(36), primary_key=True, diff --git a/buildbot_gentoo_ci/db/packages.py b/buildbot_gentoo_ci/db/packages.py index a22b9d8..79ae685 100644 --- a/buildbot_gentoo_ci/db/packages.py +++ b/buildbot_gentoo_ci/db/packages.py @@ -65,7 +65,10 @@ class PackagesConnectorComponent(base.DBConnectorComponent): r = conn.execute(q, dict(name=name, repository_uuid=repository_uuid, category_uuid=category_uuid)) - except (sa.exc.IntegrityError, sa.exc.ProgrammingError): + except Exception as e: + print(type(e)) + print(e.args) + print(e) uuid = None else: uuid = r.inserted_primary_key[0] @@ -80,3 +83,145 @@ class PackagesConnectorComponent(base.DBConnectorComponent): repository_uuid=row.repository_uuid, category_uuid=row.category_uuid ) + + @defer.inlineCallbacks + def getPackageMetadataByPackageUuid(self, package_uuid): + def thd(conn): + tbl = self.db.model.packages_metadata + q = tbl.select() + q = q.where(tbl.c.package_uuid == package_uuid) + res = conn.execute(q) + row = res.fetchone() + if not row: + return None + return self._row2dict_metadata(conn, row) + res = yield self.db.pool.do(thd) + return res + + @defer.inlineCallbacks + def addMetadata(self, package_uuid, sha256): + def thd(conn, no_recurse=False): + try: + tbl = self.db.model.packages_metadata + q = tbl.insert() + r = conn.execute(q, dict(package_uuid=package_uuid, + sha256=sha256 + )) + except Exception as e: + print(type(e)) + print(e.args) + print(e) + res = None + else: + res = r.inserted_primary_key[0] + return res + res = yield self.db.pool.do(thd) + return res + + def _row2dict_metadata(self, conn, row): + return dict( + id=row.id, + package_uuid=row.package_uuid, + sha256=row.sha256, + ) + + @defer.inlineCallbacks + def addEmail(self, email): + def thd(conn, no_recurse=False): + try: + tbl = self.db.model.emails + q = tbl.insert() + r = conn.execute(q, dict(email=email)) + except Exception as e: + print(type(e)) + print(e.args) + print(e) + res = None + else: + res = r.inserted_primary_key[0] + return res + res = yield self.db.pool.do(thd) + return res + + @defer.inlineCallbacks + def addPackageEmail(self, email_id, package_uuid, mail_type, proxied): + def thd(conn, no_recurse=False): + try: + tbl = self.db.model.packages_emails + q = tbl.insert() + r = conn.execute(q, dict(email_id=email_id, + package_uuid=package_uuid, + mail_type=mail_type, + proxied=proxied + )) + except Exception as e: + print(type(e)) + print(e.args) + print(e) + res = None + else: + res = r.inserted_primary_key[0] + return res + res = yield self.db.pool.do(thd) + return res + + @defer.inlineCallbacks + def getEmailIdByEmail(self, email): + def thd(conn): + tbl = self.db.model.emails + q = tbl.select() + q = q.where(tbl.c.email == email) + res = conn.execute(q) + row = res.fetchone() + if not row: + return None + return self._row2dict_email(conn, row) + res = yield self.db.pool.do(thd) + return res + + @defer.inlineCallbacks + def getEmailByEmailId(self, email_id): + def thd(conn): + tbl = self.db.model.emails + q = tbl.select() + q = q.where(tbl.c.id == email_id) + res = conn.execute(q) + row = res.fetchone() + if not row: + return None + return self._row2dict_email(conn, row) + res = yield self.db.pool.do(thd) + return res + + @defer.inlineCallbacks + def getEmailsIdsByPackageUuid(self, package_uuid): + def thd(conn): + tbl = self.db.model.packages_emails + q = tbl.select() + q = q.where(tbl.c.package_uuid == package_uuid) + return [self._row2dict_package_email(conn, row) + for row in conn.execute(q).fetchall()] + res = yield self.db.pool.do(thd) + return res + + @defer.inlineCallbacks + def delPackageEmail(self, package_uuid): + def thd(conn, no_recurse=False): + tbl = self.db.model.packages_emails + conn.execute(tbl.delete( + whereclause=((tbl.c.package_uuid == package_uuid)))) + return self.db.pool.do(thd) + + def _row2dict_email(self, conn, row): + return dict( + id=row.id, + email=row.email, + ) + def _row2dict_package_email(self, conn, row): + return dict( + id=row.id, + email_id=row.email_id, + package_uuid=row.package_uuid, + type=row.type, + proxied=row.proxied, + ) diff --git a/buildbot_gentoo_ci/steps/package.py b/buildbot_gentoo_ci/steps/package.py index 9fc59e8..13eafa2 100644 --- a/buildbot_gentoo_ci/steps/package.py +++ b/buildbot_gentoo_ci/steps/package.py @@ -4,9 +4,7 @@ import re import os -from portage.xml.metadata import MetaDataXML -from portage.checksum import perform_checksum -from portage.versions import catpkgsplit +import xml.etree.ElementTree as etree from twisted.internet import defer from twisted.python import log @@ -14,6 +12,7 @@ from twisted.python import log from buildbot.process.buildstep import BuildStep from buildbot.process.results import SUCCESS from buildbot.process.results import FAILURE +from buildbot.process import remotecommand from buildbot.plugins import steps class SetupPropertys(BuildStep): @@ -67,6 +66,161 @@ class AddPackage(BuildStep): self.setProperty("package_data", self.package_data, 'package_data') return SUCCESS +class UpdateEmails(BuildStep): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + name = 'UpdateEmails' + description = 'Running' + descriptionDone = 'Ran' + descriptionSuffix = None + haltOnFailure = True + flunkOnFailure = True + + @defer.inlineCallbacks + def run(self): + self.gentooci = self.master.namedServices['services'].namedServices['gentooci'] + packages_emails = yield self.gentooci.db.packages.getEmailsIdsByPackageUuid(self.getProperty("package_data")['uuid']) + if packages_emails != []: + yield self.gentooci.db.packages.delPackageEmail(self.getProperty("package_data")['uuid']) + # update email + for maintainer in self.getProperty("maintainers"): + print(maintainer) + email_data = yield self.gentooci.db.packages.getEmailIdByEmail(maintainer['email']) + print(email_data) + if email_data is None: + email_id = yield self.gentooci.db.packages.addEmail(maintainer['email']) + print(email_id) + else: + email_id = email_data['id'] + if maintainer['proxied'] is None: + proxied = 'no' + else: + proxied = maintainer['proxied'] + res = yield self.gentooci.db.packages.addPackageEmail( + email_id, + self.getProperty("package_data")['uuid'], + maintainer['type'], + proxied + ) + print(res) + return SUCCESS + +class ReadMetadataXML(BuildStep): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + name = 'ReadMetadataXML' + description = 'Running' + descriptionDone = 'Ran' + descriptionSuffix = None + haltOnFailure = True + flunkOnFailure = True + + @defer.inlineCallbacks + def run(self): + self._xml_tree = etree.fromstring(self.getProperty("metadata_xml")) + # get maintainer + self.maintainers = [] + for maintainer in self._xml_tree.findall('maintainer'): + self.maintainers.append(dict( + type=maintainer.get('type'), + proxied=maintainer.get('proxied'), + email=maintainer.find('email').text, + )) + print(self.maintainers) + self.setProperty("maintainers", self.maintainers, 'maintainers') + yield self.build.addStepsAfterCurrentStep([UpdateEmails()]) + # update email + return SUCCESS + +class CheckSha256Metadata(BuildStep): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + name = 'CheckSha256Metadata' + description = 'Running' + descriptionDone = 'Ran' + descriptionSuffix = None + haltOnFailure = True + flunkOnFailure = True + + @defer.inlineCallbacks + def run(self): + # if checksum diff update + self.gentooci = self.master.namedServices['services'].namedServices['gentooci'] + sha256sum_metadata = self.getProperty("sha256sum_metadata").split()[0] + print(sha256sum_metadata) + package_metadata_data_updated = False + self.package_metadata_data = yield self.gentooci.db.packages.getPackageMetadataByPackageUuid(self.getProperty("package_data")['uuid']) + print(self.package_metadata_data) + if self.package_metadata_data is None: + res = yield self.gentooci.db.packages.addMetadata( + self.getProperty("package_data")['uuid'], + sha256sum_metadata + ) + print(res) + package_metadata_data_updated = True + elif self.package_metadata_data['sha256'] != sha256sum_metadata: + # update sha256 + package_metadata_data_updated = True + if package_metadata_data_updated: + yield self.build.addStepsAfterCurrentStep([ + steps.SetPropertyFromCommand( + name = 'CatMetadataFile', + haltOnFailure = True, + flunkOnFailure = True, + command=['cat', self.getProperty("metadata_file")], + strip=False, + property='metadata_xml' + ), + ReadMetadataXML(), + ]) + return SUCCESS + +class CheckMetadataPackagePath(BuildStep): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + name = 'CheckMetadataPackagePath' + description = 'Running' + descriptionDone = 'Ran' + descriptionSuffix = None + haltOnFailure = True + flunkOnFailure = True + + @defer.inlineCallbacks + def run(self): + # set metadata.xml path + self.repository_basedir = yield os.path.join(self.getProperty("worker_basedir"), self.getProperty('portage_repos_path')) + print(self.repository_basedir) + self.repository_path = yield os.path.join(self.repository_basedir, self.getProperty("repository_data")['name']) + print(self.repository_path) + self.cp_path = yield os.path.join(self.repository_path, self.getProperty("change_data")['cp']) + print(self.cp_path) + self.metadata_file = yield '/'.join([self.cp_path, 'metadata.xml']) + print(self.metadata_file) + # check file exist + cmd = remotecommand.RemoteCommand('stat', {'file': self.metadata_file}) + yield self.runCommand(cmd) + if cmd.didFail(): + return FAILURE + self.setProperty("metadata_file", self.metadata_file, 'metadata_file') + # checksum on file + yield self.build.addStepsAfterCurrentStep([ + steps.SetPropertyFromCommand( + name = 'ChecksumMetadataPackage', + haltOnFailure = True, + flunkOnFailure = True, + command=['sha256sum', self.metadata_file], + workdir=self.repository_path, + strip=False, + property='sha256sum_metadata' + ), + CheckSha256Metadata(), + ]) + return SUCCESS + class CheckP(BuildStep): def __init__(self, **kwargs): super().__init__(**kwargs) @@ -90,10 +244,8 @@ class CheckP(BuildStep): if self.package_data is None: self.setProperty("package", self.package, 'package') yield self.build.addStepsAfterCurrentStep([AddPackage()]) - #yield self.build.addStepsAfterLastStep([AddMetadataPackage()]) - return SUCCESS - self.setProperty("package_data", self.package_data, 'package_data') - #yield self.build.addStepsAfterLastStep([CheckPathPackage()]) + else: + self.setProperty("package_data", self.package_data, 'package_data') return SUCCESS class TriggerCheckForV(BuildStep): diff --git a/sql/gentoo_ci_schema.sql b/sql/gentoo_ci_schema.sql index d26e0c8..7de44ba 100644 --- a/sql/gentoo_ci_schema.sql +++ b/sql/gentoo_ci_schema.sql @@ -17,6 +17,32 @@ SET client_min_messages = warning; SET row_security = off; -- +-- Name: email_proxied; Type: TYPE; Schema: public; Owner: buildbot +-- + +CREATE TYPE public.email_proxied AS ENUM ( + 'no', + 'yes', + 'proxy' +); + + +ALTER TYPE public.email_proxied OWNER TO buildbot; + +-- +-- Name: email_type; Type: TYPE; Schema: public; Owner: buildbot +-- + +CREATE TYPE public.email_type AS ENUM ( + 'person', + 'project', + 'unknown' +); + + +ALTER TYPE public.email_type OWNER TO buildbot; + +-- -- Name: projects_builds_status; Type: TYPE; Schema: public; Owner: buildbot -- @@ -216,27 +242,32 @@ CREATE TYPE public.workers_config_type AS ENUM ( ALTER TYPE public.workers_config_type OWNER TO buildbot; -- --- Name: workers_type; Type: TYPE; Schema: public; Owner: buildbot +-- Name: workers_images_type; Type: TYPE; Schema: public; Owner: buildbot -- -CREATE TYPE public.workers_type AS ENUM ( - 'log', - 'build' +CREATE TYPE public.workers_images_type AS ENUM ( + 'docker' ); -ALTER TYPE public.workers_type OWNER TO buildbot; +ALTER TYPE public.workers_images_type OWNER TO buildbot; -- --- Name: workers_images_type; Type: TYPE; Schema: public; Owner: buildbot +-- Name: workers_type; Type: TYPE; Schema: public; Owner: buildbot -- -CREATE TYPE public.workers_images_type AS ENUM ( - 'docker' +CREATE TYPE public.workers_type AS ENUM ( + 'default', + 'local', + 'latent', + 'chroot', + 'docker', + 'log', + 'build' ); -ALTER TYPE public.workers_images_type OWNER TO buildbot; +ALTER TYPE public.workers_type OWNER TO buildbot; SET default_tablespace = ''; @@ -255,6 +286,39 @@ CREATE TABLE public.categorys ( ALTER TABLE public.categorys OWNER TO buildbot; -- +-- Name: emails; Type: TABLE; Schema: public; Owner: buildbot +-- + +CREATE TABLE public.emails ( + id integer DEFAULT nextval('public.packages_metadata_id_seq'::regclass) NOT NULL, + email character varying(255) +); + + +ALTER TABLE public.emails OWNER TO buildbot; + +-- +-- Name: emails_id_seq; Type: SEQUENCE; Schema: public; Owner: buildbot +-- + +CREATE SEQUENCE public.emails_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.emails_id_seq OWNER TO buildbot; + +-- +-- Name: emails_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: buildbot +-- + +ALTER SEQUENCE public.emails_id_seq OWNED BY public.emails.id; + + +-- -- Name: keywords; Type: TABLE; Schema: public; Owner: buildbot -- @@ -286,9 +350,9 @@ ALTER TABLE public.migrate_version OWNER TO buildbot; CREATE TABLE public.nodes ( name character varying, docker_host_url character varying, - bin_host_url character varying, enable boolean, - uuid character varying(36) NOT NULL + uuid character varying(36) NOT NULL, + bin_host_url character varying ); @@ -324,6 +388,64 @@ CREATE TABLE public.packages ( ALTER TABLE public.packages OWNER TO buildbot; -- +-- Name: packages_emails_id_seq; Type: SEQUENCE; Schema: public; Owner: buildbot +-- + +CREATE SEQUENCE public.packages_emails_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.packages_emails_id_seq OWNER TO buildbot; + +-- +-- Name: packages_emails; Type: TABLE; Schema: public; Owner: buildbot +-- + +CREATE TABLE public.packages_emails ( + id integer DEFAULT nextval('public.packages_emails_id_seq'::regclass) NOT NULL, + package_uuid character varying, + email_id integer, + mail_type public.email_type, + proxied public.email_proxied +); + + +ALTER TABLE public.packages_emails OWNER TO buildbot; + +-- +-- Name: packages_metadata_id_seq; Type: SEQUENCE; Schema: public; Owner: buildbot +-- + +CREATE SEQUENCE public.packages_metadata_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.packages_metadata_id_seq OWNER TO buildbot; + +-- +-- Name: packages_metadata; Type: TABLE; Schema: public; Owner: buildbot +-- + +CREATE TABLE public.packages_metadata ( + id integer DEFAULT nextval('public.packages_metadata_id_seq'::regclass) NOT NULL, + package_uuid character varying(36), + sha256 character varying +); + + +ALTER TABLE public.packages_metadata OWNER TO buildbot; + +-- -- Name: portages_makeconf; Type: TABLE; Schema: public; Owner: buildbot -- @@ -614,7 +736,7 @@ ALTER SEQUENCE public.projects_portages_makeconf_id_seq OWNED BY public.projects -- CREATE TABLE public.projects_portages_package ( - id bigint NOT NULL, + id integer NOT NULL, project_uuid character varying(36), directory public.projects_portage_package_directorys, package character varying(50), @@ -669,7 +791,7 @@ ALTER SEQUENCE public.projects_repositorys_id_seq OWNED BY public.projects_repos -- CREATE TABLE public.projects_workers ( - id bigint NOT NULL, + id integer NOT NULL, project_uuid character varying(36), worker_uuid character varying(36) ); @@ -897,6 +1019,7 @@ CREATE TABLE public.workers_images ( ALTER TABLE public.workers_images OWNER TO buildbot; + -- -- Name: portages_makeconf id; Type: DEFAULT; Schema: public; Owner: buildbot -- @@ -984,6 +1107,14 @@ ALTER TABLE ONLY public.categorys -- +-- Name: emails email_pkey; Type: CONSTRAINT; Schema: public; Owner: buildbot +-- + +ALTER TABLE ONLY public.emails + ADD CONSTRAINT email_pkey PRIMARY KEY (id); + + +-- -- Name: keywords keywords_pkey; Type: CONSTRAINT; Schema: public; Owner: buildbot -- @@ -1024,6 +1155,22 @@ ALTER TABLE ONLY public.nodes_workers -- +-- Name: packages_emails packages_emails_pkey; Type: CONSTRAINT; Schema: public; Owner: buildbot +-- + +ALTER TABLE ONLY public.packages_emails + ADD CONSTRAINT packages_emails_pkey PRIMARY KEY (id); + + +-- +-- Name: packages_metadata packages_metadata_pkey; Type: CONSTRAINT; Schema: public; Owner: buildbot +-- + +ALTER TABLE ONLY public.packages_metadata + ADD CONSTRAINT packages_metadata_pkey PRIMARY KEY (id); + + +-- -- Name: packages packages_pkey; Type: CONSTRAINT; Schema: public; Owner: buildbot -- @@ -1328,6 +1475,14 @@ ALTER TABLE ONLY public.packages -- +-- Name: packages_emails email_id; Type: FK CONSTRAINT; Schema: public; Owner: buildbot +-- + +ALTER TABLE ONLY public.packages_emails + ADD CONSTRAINT email_id FOREIGN KEY (email_id) REFERENCES public.emails(id); + + +-- -- Name: projects keywords_fkey; Type: FK CONSTRAINT; Schema: public; Owner: buildbot -- @@ -1360,6 +1515,22 @@ ALTER TABLE ONLY public.projects_portages_makeconf -- +-- Name: packages_emails package_uuid; Type: FK CONSTRAINT; Schema: public; Owner: buildbot +-- + +ALTER TABLE ONLY public.packages_emails + ADD CONSTRAINT package_uuid FOREIGN KEY (package_uuid) REFERENCES public.packages(uuid); + + +-- +-- Name: packages_metadata package_uuid; Type: FK CONSTRAINT; Schema: public; Owner: buildbot +-- + +ALTER TABLE ONLY public.packages_metadata + ADD CONSTRAINT package_uuid FOREIGN KEY (package_uuid) REFERENCES public.packages(uuid) NOT VALID; + + +-- -- Name: versions packages_fkey; Type: FK CONSTRAINT; Schema: public; Owner: buildbot -- @@ -1488,10 +1659,10 @@ ALTER TABLE ONLY public.versions_metadata -- --- Name: SCHEMA public; Type: ACL; Schema: -; Owner: postgres +-- Name: SCHEMA public; Type: ACL; Schema: -; Owner: buildbot -- -REVOKE ALL ON SCHEMA public FROM postgres; +REVOKE ALL ON SCHEMA public FROM buildbot; GRANT ALL ON SCHEMA public TO buildbot; |