Return-Path: X-Original-To: apmail-incubator-bloodhound-commits-archive@minotaur.apache.org Delivered-To: apmail-incubator-bloodhound-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id A1159EC58 for ; Fri, 25 Jan 2013 14:31:52 +0000 (UTC) Received: (qmail 27358 invoked by uid 500); 25 Jan 2013 14:31:52 -0000 Delivered-To: apmail-incubator-bloodhound-commits-archive@incubator.apache.org Received: (qmail 27286 invoked by uid 500); 25 Jan 2013 14:31:51 -0000 Mailing-List: contact bloodhound-commits-help@incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: bloodhound-dev@incubator.apache.org Delivered-To: mailing list bloodhound-commits@incubator.apache.org Received: (qmail 27267 invoked by uid 99); 25 Jan 2013 14:31:50 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 25 Jan 2013 14:31:50 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 25 Jan 2013 14:31:49 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 283D023888FE; Fri, 25 Jan 2013 14:31:30 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1438538 - in /incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_multiproduct: multiproduct/api.py multiproduct/dbcursor.py tests/dbcursor.py Date: Fri, 25 Jan 2013 14:31:29 -0000 To: bloodhound-commits@incubator.apache.org From: jure@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20130125143130.283D023888FE@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: jure Date: Fri Jan 25 14:31:29 2013 New Revision: 1438538 URL: http://svn.apache.org/viewvc?rev=1438538&view=rev Log: #288, properly handle translated table INSERTs w/o column names specified, schema upgrade rewritten, test cases updated and added Modified: incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_multiproduct/multiproduct/api.py incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_multiproduct/multiproduct/dbcursor.py incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_multiproduct/tests/dbcursor.py Modified: incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_multiproduct/multiproduct/api.py URL: http://svn.apache.org/viewvc/incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_multiproduct/multiproduct/api.py?rev=1438538&r1=1438537&r2=1438538&view=diff ============================================================================== --- incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_multiproduct/multiproduct/api.py (original) +++ incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_multiproduct/multiproduct/api.py Fri Jan 25 14:31:29 2013 @@ -21,6 +21,8 @@ from datetime import datetime from genshi.builder import tag +import copy + from pkg_resources import resource_filename from trac.config import PathOption from trac.core import Component, TracError, implements @@ -117,89 +119,57 @@ class MultiProductSystem(Component): if db_installed_version < 3: from multiproduct.dbcursor import DEFAULT_PRODUCT + from multiproduct.model import Product + import trac.db_default migrate_tables = ['enum', 'component', 'milestone', 'version', 'permission', 'wiki'] - table_defs = [ - Table('enum', key=('type', 'name', 'product'))[ - Column('type'), - Column('name'), - Column('value'), - Column('product')], - Table('component', key=('name', 'product'))[ - Column('name'), - Column('owner'), - Column('description'), - Column('product')], - Table('milestone', key=('name', 'product'))[ - Column('name'), - Column('due', type='int64'), - Column('completed', type='int64'), - Column('description'), - Column('product')], - Table('version', key=('name', 'product'))[ - Column('name'), - Column('time', type='int64'), - Column('description'), - Column('product')], - Table('permission', key=('username', 'action', 'product'))[ - Column('username'), - Column('action'), - Column('product')], - Table('wiki', key=('name', 'version', 'product'))[ - Column('name'), - Column('version', type='int'), - Column('time', type='int64'), - Column('author'), - Column('ipnr'), - Column('text'), - Column('comment'), - Column('readonly', type='int'), - Column('product'), - Index(['time'])], - ] + # extend trac default schema by adding product column and extending key with product + table_defs = [copy.deepcopy(t) for t in trac.db_default.schema if t.name in migrate_tables] + for t in table_defs: + t.columns.append(Column('product')) + if isinstance(t.key, list): + t.key = tuple(t.key) + tuple(['product']) + elif isinstance(t.key, tuple): + t.key = t.key + tuple(['product']) + else: + raise TracError("Invalid table '%s' schema key '%s' while upgrading " + "plugin '%s' from version %d to %d'" % + (t.name, t.key, PLUGIN_NAME, db_installed_version, 3)) table_columns = dict() - table_vals = {} for table in table_defs: table_columns[table.name] = filter(lambda column: column != 'product', [column.name for column in list(filter(lambda t: t.name == table.name, table_defs)[0].columns)]) - table_columns['bloodhound_product'] = ['prefix', 'name', 'description', 'owner'] - def fetch_table(table): - table_vals[table] = list(db("SELECT %s FROM %s" % (','.join(table_columns[table]), table))) - for table in table_columns.keys(): - self.log.info("Fetching table '%s'", table) - fetch_table(table) - for table in migrate_tables: - self.log.info("Dropping obsolete table '%s'", table) - db("DROP TABLE %s" % table) - db_connector, _ = DatabaseManager(self.env).get_connector() - for table in table_defs: - self.log.info("Creating table '%s'", table.name) - for sql in db_connector.to_sql(table): - db(sql) self.log.info("Creating default product") - db("""INSERT INTO bloodhound_product (prefix, name, description, owner) - VALUES ('%s', 'Default', 'Default product', '')""" % DEFAULT_PRODUCT) + default_product = Product(self.env) + default_product.update_field_dict({'prefix': DEFAULT_PRODUCT, + 'name': 'Default', + 'description': 'Default product', + 'owner': '', + }) + default_product.insert() + self.log.info("Migrating tickets w/o product to default product") db("""UPDATE ticket SET product='%s' WHERE product=''""" % DEFAULT_PRODUCT) - def insert_with_product(table, product): - cols = table_columns[table] + ['product'] - sql = "INSERT INTO %s (%s) VALUES (%s)" % (table, - ','.join(cols), - ','.join(['%s'] * len(cols))) - for r in table_vals[table]: - vals = list() - for v in list(r): - vals.append(v if v else '') - db(sql, tuple(vals + [product])) + self.log.info("Migrating tables to a new schema") for table in migrate_tables: - self.log.info("Creating tables '%s' for default product", table) - insert_with_product(table, DEFAULT_PRODUCT) - for p in table_vals['bloodhound_product']: - self.log.info("Creating tables '%s' for product '%s' ('%s')", table, p[1], p[0]) - insert_with_product(table, p[0]) + cols = ','.join(table_columns[table]) + self.log.info("Migrating table '%s' to a new schema", table) + db("CREATE TEMPORARY TABLE %s_temp AS SELECT %s FROM %s" % + (table, cols, table)) + db("DROP TABLE %s" % table) + db_connector, _ = DatabaseManager(self.env)._get_connector() + table_schema = filter(lambda t: t.name == table, table_defs)[0] + for sql in db_connector.to_sql(table_schema): + db(sql) + products = Product.select(self.env) + for product in products: + self.log.info("Populating table '%s' for product '%s' ('%s')", table, product.name, product.prefix) + db("INSERT INTO %s (%s, product) SELECT %s,'%s' FROM %s_temp" % + (table, cols, cols, product.prefix, table)) + db("DROP TABLE %s_temp" % table) db_installed_version = self._update_db_version(db, 3) if db_installed_version < 4: Modified: incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_multiproduct/multiproduct/dbcursor.py URL: http://svn.apache.org/viewvc/incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_multiproduct/multiproduct/dbcursor.py?rev=1438538&r1=1438537&r2=1438538&view=diff ============================================================================== --- incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_multiproduct/multiproduct/dbcursor.py (original) +++ incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_multiproduct/multiproduct/dbcursor.py Fri Jan 25 14:31:29 2013 @@ -343,8 +343,11 @@ class BloodhoundProductSQLTranslate(obje current_token, field_lists = self._select_expression_tokens(parent, fields_token, ['FROM'] + self._from_end_words) def handle_insert_table(table_name): if insert_table and insert_table in self._translate_tables: - for keyword in [self._product_column, ',', ' ']: - self._token_insert_before(parent, fields_token, Types.Token(Tokens.Keyword, keyword)) + if not field_lists or not field_lists[-1]: + raise Exception("Invalid SELECT field list") + last_token = list(field_lists[-1][-1].flatten())[-1] + for keyword in [self._product_column, ' ', ',']: + self._token_insert_after(last_token.parent, last_token, Types.Token(Tokens.Keyword, keyword)) return table_name_callback = handle_insert_table if insert_table else None from_token = self._token_next_match(parent, start_token, Tokens.Keyword, 'FROM') @@ -403,14 +406,22 @@ class BloodhoundProductSQLTranslate(obje isinstance(columns_token, Types.Parenthesis): ptoken = self._token_first(columns_token) if not ptoken.match(Tokens.Punctuation, '('): - raise Exception("Invalid INSERT statement") - for keyword in [' ', ',', self._product_column]: - self._token_insert_after(columns_token, ptoken, Types.Token(Tokens.Keyword, keyword)) + raise Exception("Invalid INSERT statement, expected parenthesis around columns") + ptoken = self._token_next(columns_token, ptoken) + last_token = ptoken + while ptoken: + last_token = ptoken + ptoken = self._token_next(columns_token, ptoken) + if not last_token or \ + not last_token.match(Tokens.Punctuation, ')'): + raise Exception("Invalid INSERT statement, unable to find column parenthesis end") + for keyword in [',', ' ', self._product_column]: + self._token_insert_before(columns_token, last_token, Types.Token(Tokens.Keyword, keyword)) return - def insert_extra_column_value(tablename, ptoken, start_token): + def insert_extra_column_value(tablename, ptoken, before_token): if tablename in self._translate_tables: for keyword in [',', "'", self._product_prefix, "'"]: - self._token_insert_after(ptoken, start_token, Types.Token(Tokens.Keyword, keyword)) + self._token_insert_before(ptoken, before_token, Types.Token(Tokens.Keyword, keyword)) return tablename = None table_name_token = self._token_next(parent, token) @@ -440,14 +451,19 @@ class BloodhoundProductSQLTranslate(obje ptoken = self._token_first(token) if not ptoken.match(Tokens.Punctuation, '('): raise Exception("Invalid INSERT statement") - insert_extra_column_value(tablename, token, ptoken) + last_token = ptoken while ptoken: if not ptoken.match(Tokens.Punctuation, separators) and \ not ptoken.match(Tokens.Keyword, separators) and \ not ptoken.is_whitespace(): ptoken = self._expression_token_unwind_hack(token, ptoken, self._token_prev(token, ptoken)) self._eval_expression_value(token, ptoken) + last_token = ptoken ptoken = self._token_next(token, ptoken) + if not last_token or \ + not last_token.match(Tokens.Punctuation, ')'): + raise Exception("Invalid INSERT statement, unable to find column value parenthesis end") + insert_extra_column_value(tablename, token, last_token) elif not token.match(Tokens.Punctuation, separators) and\ not token.match(Tokens.Keyword, separators) and\ not token.is_whitespace(): Modified: incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_multiproduct/tests/dbcursor.py URL: http://svn.apache.org/viewvc/incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_multiproduct/tests/dbcursor.py?rev=1438538&r1=1438537&r2=1438538&view=diff ============================================================================== --- incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_multiproduct/tests/dbcursor.py (original) +++ incubator/bloodhound/branches/bep_0003_multiproduct/bloodhound_multiproduct/tests/dbcursor.py Fri Jan 25 14:31:29 2013 @@ -497,12 +497,12 @@ data = { reporter, cc, version, milestone, status, resolution, summary, description, keywords FROM ticket_old WHERE COALESCE(severity,'') <> 'enhancement'""", -"""INSERT INTO ticket(product, id, type, time, changetime, component, severity, priority, +"""INSERT INTO ticket(id, type, time, changetime, component, severity, priority, owner, reporter, cc, version, milestone, status, resolution, - summary, description, keywords) - SELECT product, id, 'defect', time, changetime, component, severity, priority, owner, + summary, description, keywords, product) + SELECT id, 'defect', time, changetime, component, severity, priority, owner, reporter, cc, version, milestone, status, resolution, summary, - description, keywords FROM (SELECT * FROM PRODUCT_ticket_old) AS ticket_old + description, keywords, product FROM (SELECT * FROM PRODUCT_ticket_old) AS ticket_old WHERE COALESCE(severity,'') <> 'enhancement'""" ), ( @@ -513,12 +513,12 @@ data = { owner, reporter, cc, version, milestone, status, resolution, summary, description, keywords FROM ticket_old WHERE severity = 'enhancement'""", -"""INSERT INTO ticket(product, id, type, time, changetime, component, severity, priority, +"""INSERT INTO ticket(id, type, time, changetime, component, severity, priority, owner, reporter, cc, version, milestone, status, resolution, - summary, description, keywords) - SELECT product, id, 'enhancement', time, changetime, component, 'normal', priority, + summary, description, keywords, product) + SELECT id, 'enhancement', time, changetime, component, 'normal', priority, owner, reporter, cc, version, milestone, status, resolution, summary, - description, keywords FROM (SELECT * FROM PRODUCT_ticket_old) AS ticket_old + description, keywords, product FROM (SELECT * FROM PRODUCT_ticket_old) AS ticket_old WHERE severity = 'enhancement'""" ), ( @@ -669,9 +669,17 @@ data = { """INSERT INTO wiki(version, name, time, author, ipnr, text) SELECT 1 + COALESCE(max(version), 0), %s, %s, 'trac', '127.0.0.1', %s FROM wiki WHERE name=%s""", -"""INSERT INTO wiki(product, version, name, time, author, ipnr, text) - SELECT product, 1 + COALESCE(max(version), 0), %s, %s, 'trac', - '127.0.0.1', %s FROM (SELECT * FROM wiki WHERE product="PRODUCT") AS wiki WHERE name=%s""" +"""INSERT INTO wiki(version, name, time, author, ipnr, text, product) + SELECT 1 + COALESCE(max(version), 0), %s, %s, 'trac', + '127.0.0.1', %s, product FROM (SELECT * FROM wiki WHERE product="PRODUCT") AS wiki WHERE name=%s""" + ), + ( +"""INSERT INTO permission VALUES ('dev','WIKI_VIEW')""", +"""INSERT INTO permission VALUES ('dev','WIKI_VIEW','PRODUCT')""" + ), + ( +"""INSERT INTO permission (username, action) VALUES ('dev','WIKI_VIEW')""", +"""INSERT INTO permission (username, action, product) VALUES ('dev','WIKI_VIEW','PRODUCT')""" ), ],