From 7ba5f957c082545a9eb097839fdc7eba826a7d80 Mon Sep 17 00:00:00 2001 From: Antanas Uršulis Date: Tue, 30 Jul 2013 01:38:23 +0300 Subject: Implement additional metadata storage for logs received from Portage Flags like 'pkg_failed', 'test_failed', etc. are stored once per group. This is a discussion item, as it might make sense to store some data once per file instead. --- database.py | 2 +- portage_database.py | 12 ++++++++++++ portage_processor.py | 18 +++++++++++------- schema_portage.sql | 10 ++++++++++ 4 files changed, 34 insertions(+), 8 deletions(-) create mode 100644 portage_database.py create mode 100644 schema_portage.sql diff --git a/database.py b/database.py index 79ae693..9927627 100644 --- a/database.py +++ b/database.py @@ -1,7 +1,7 @@ from contextlib import closing import MySQLdb -class DatabaseConnection: +class DatabaseConnection(object): def __init__(self, conn): self.conn = conn diff --git a/portage_database.py b/portage_database.py new file mode 100644 index 0000000..ca7831c --- /dev/null +++ b/portage_database.py @@ -0,0 +1,12 @@ +from contextlib import closing +from database import DatabaseConnection + +class PortageDatabaseConnection(DatabaseConnection): + def __init__(self, db): + super(PortageDatabaseConnection, self).__init__(db.conn) + + # TODO: consider passing these arguments around in a dictionary or kwargs + def insert_group_extra(self, group_id, pkg_name, matches, pkg_failed, test_failed, collision, bug_assignee, bug_cc): + with closing(self.conn.cursor()) as c: + c.execute("insert into `groups_portage` values (%s, %s, %s, %s, %s, %s, %s, %s)", (group_id, pkg_name, matches, pkg_failed, test_failed, collision, bug_assignee, bug_cc)) + self.conn.commit() diff --git a/portage_processor.py b/portage_processor.py index 32ca9c4..252209e 100644 --- a/portage_processor.py +++ b/portage_processor.py @@ -1,4 +1,5 @@ import os, re, StringIO, time +from portage_database import PortageDatabaseConnection class PortageProcessor: _r = { @@ -14,16 +15,17 @@ class PortageProcessor: self.storage = storage def process(self, request, source, db): + db = PortageDatabaseConnection(db) group_id = db.insert_group(source, request.group_name, 'portage', int(time.time())) - for f in request.files: - matches = 0 - pkg_failed = False - test_failed = False - collision = False - bug_assignee = 'bug-wranglers@gentoo.org' - bug_cc = '' + matches = 0 + pkg_failed = False + test_failed = False + collision = False + bug_assignee = 'bug-wranglers@gentoo.org' + bug_cc = '' + for f in request.files: # TODO: look at proper HTML generation methods: # (*) either XHTML via xml.etree # (*) or Jinja2 (is it possible to parse and generate in one pass?) @@ -74,3 +76,5 @@ class PortageProcessor: self.storage.save_file(source, f.filename, output.getvalue()) file_id = db.insert_file(os.path.join(source, f.filename), group_id) + + db.insert_group_extra(group_id, 'TODO', matches, pkg_failed, test_failed, collision, bug_assignee, bug_cc) diff --git a/schema_portage.sql b/schema_portage.sql new file mode 100644 index 0000000..4b9c80d --- /dev/null +++ b/schema_portage.sql @@ -0,0 +1,10 @@ +create table if not exists `groups_portage` ( + `id` int primary key, + `pkg_name` varchar(255) not null, + `matches` int not null, + `pkg_failed` bool not null, + `test_failed` bool not null, + `collision` bool not null, + `bug_assignee` text not null, + `bug_cc` text not null +); -- cgit v1.2.3-65-gdbad