# configure_bugzilla.py -- Build P4DTI configuration for replication # to Bugzilla. # # Richard Brooksby, Ravenbrook Limited, 2000-12-07 # # $Id: //info.ravenbrook.com/project/p4dti/version/1.2/code/replicator/configure_bugzilla.py#1 $ # # # 1. INTRODUCTION # # This is the automatic configurator for the Bugzilla integration of the # Perforce Defect Tracking Integration (P4DTI). # # This code takes a basic set of configuration parameters [RB 2000-08-10, 5.1] # and generates a full set of parameters for the "dt_bugzilla", "p4", and # "replicator" classes, as well as a Perforce jobspec. # # --- # # This file is copyright (c) 2001 Perforce Software, Inc. All rights # reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in # the documentation and/or other materials provided with the # distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT # HOLDERS AND CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS # OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR # TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE # USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH # DAMAGE. import MySQLdb import bugzilla import catalog import check_config import dt_bugzilla import logger import message import os import socket import string import translator import types error = "Bugzilla configuration error" # Force MySQLdb to use plain strings as date/time types try: from DateTime import DateTimeType, DateTimeDeltaType # types of values coming out of MySQL for t in [MySQLdb.FIELD_TYPE.DATETIME, MySQLdb.FIELD_TYPE.DATE, MySQLdb.FIELD_TYPE.TIME, MySQLdb.FIELD_TYPE.TIMESTAMP]: if MySQLdb.type_conv.has_key(t): del MySQLdb.type_conv[t] # types of values going into MySQL for t in [DateTimeType, DateTimeDeltaType]: if MySQLdb.quote_conv.has_key(t): del MySQLdb.quote_conv[t] except ImportError: # e.g. no DateTime module; MySQLdb will revert to plain strings anyway. pass # perforce keyword translator. keyword_translator = translator.keyword_translator() # enum translator (just like keyword translator except '' <-> 'NONE') enum_translator = dt_bugzilla.enum_translator(keyword_translator) # make_state_pairs: (strings * string) -> (string * string) list. Make a # list of pairs of state names (Bugzilla state, Perforce state). This list # will be used to translate between states, and also to generate the possible # values for the Status field in Perforce. # # The closed_state argument is the Bugzilla state which maps to the special # state 'closed' in Perforce, or None if there is no such state. See # requirement 45. See decision decision [RB 2000-11-28 14:44:36 GMT]. # # The case of state names in these pairs is normalized for usability in # Perforce: see design decision [RB 2000-11-28 14:24:32 GMT]. def make_state_pairs(states, closed_state): state_pairs = [] state_p4_to_dt = {} found_closed_state = 0 if closed_state != None: p4_closed_state = keyword_translator.translate_0_to_1( string.lower(closed_state)) # Perforce jobs can't have state "new" (this indicates a fresh job and # Perforce changes the state to "open"). Nor can they have state # "ignore", because that is used in the submit form to indicate that a # job shouldn't be fixed by the change. # # Unfortunately, "new" and "ignore" are common names for states in # the defect tracker (the former is in the default workflow in # TeamTrack), so we don't disallow them, but prefix them with # 'bugzilla_'. Then we quit if two Bugzilla states map to the # same state in Perforce, ruling out the unlikely situation that # someone has a Bugzilla status of 'BUGZILLA_CLOSED'. See job000141. prohibited_states = ['new', 'ignore'] prohibited_state_prefix = 'bugzilla_' for dt_state in states: p4_state = keyword_translator.translate_0_to_1(string.lower(dt_state)) if closed_state != None: if p4_state == p4_closed_state: p4_state = 'closed' found_closed_state = 1 elif p4_state == 'closed': p4_state = prohibited_state_prefix + p4_state if p4_state in prohibited_states: p4_state = prohibited_state_prefix + p4_state if (state_p4_to_dt.has_key(p4_state) and state_p4_to_dt[p4_state] != dt_state): # "Two Bugzilla states '%s' and '%s' map to the same Perforce state # '%s'." raise error, catalog.msg(300, (dt_state, state_p4_to_dt[p4_state], p4_state)) state_p4_to_dt[p4_state] = dt_state pair = (dt_state, p4_state) if pair not in state_pairs: state_pairs.append(pair) if closed_state != None and not found_closed_state: # "You specified the closed_state '%s', but there's no such Bugzilla # state." raise error, catalog.msg(301, closed_state) return state_pairs # Given the name of an enum column, calculate the values and default. def translate_enum(bugs_types, column): if not bugs_types[column]['type'] == 'enum': # "The '%s' column of Bugzilla's 'bugs' table is not an enum type." raise error, catalog.msg(302, column) values = bugs_types[column]['values'] values = map(enum_translator.translate_0_to_1, bugs_types[column]['values']) default = bugs_types[column]['default'] if default != None: default = enum_translator.translate_0_to_1(default) values = string.join(values,'/') return (values, default) # Some Bugzilla fields should not be changed from Perforce, either # because they can not be changed from Bugzilla # (e.g. 'creation_ts', 'delta_ts', 'lastdiffed'), or because they # can only be changed in Bugzilla in very restricted ways # (e.g. 'groupset', 'product', 'version', 'component', # 'target_milestone', 'everconfirmed'), or because changing them # in Bugzilla has complex side-effects which can't be sensibly # reproduced here (e.g. 'votes', 'keywords'). read_only_fields = ['bug_id', 'groupset', 'creation_ts', 'delta_ts', 'product', 'version', 'component', 'target_milestone', 'votes', 'keywords', 'lastdiffed', 'everconfirmed'] # Some Bugzilla fields can only be appended to. # In particular, the 'Descriptions' section of a Bugzilla bug # is actually a number of rows from another table, to which # the Bugzilla web interface allows one to add a row. # We fake this as if it's a regular field of the table, but # retain the append-only restriction. append_only_fields = ['longdesc'] # Map bugzilla field name to a name we can use in Perforce and a # comment. bz_field_map = { 'longdesc': ('Description', "Description and comments."), 'assigned_to': ('Assigned_To', "User to which the bug is assigned."), 'groupset': ('Groupset', None), 'bug_file_loc': ('URL', "The bug's URL."), 'bug_severity': ('Severity', "The bug's severity."), 'bug_status': ('Status', "The bug's status."), 'creation_ts': ('Creation_Timestamp', "Time created."), 'delta_ts': ('Update_Timestamp', "Time last updated."), 'short_desc': ('Summary', "The bug's summary."), 'op_sys': ('OS', "The OS to which the bug applies."), 'priority': ('Priority', "The bug's priority."), 'product': ('Product', "The product."), 'rep_platform': ('Platform', "The hardware platform to which the bug applies."), 'reporter': ('Reporter', "The Bugzilla user who reported the bug."), 'version': ('Version', "The product version."), 'component': ('Component', "The product component."), 'resolution': ('Resolution', "The manner in which the bug was resolved."), 'target_milestone': ('TargetMilestone', "The bug's target milestone."), 'qa_contact': ('QA_Contact', "The Bugzilla user who is the QA contact for this bug."), 'status_whiteboard': ('StatusWhiteboard', "The bug's status whiteboard."), 'votes': ('Votes', "The number of votes for this bug."), 'keywords': ('Keywords', "Keywords for this bug."), 'lastdiffed': ('LastDiffed', "Time last compared for automated e-mail."), 'everconfirmed': ('EverConfirmed', "Has this bug ever been confirmed?"), 'bug_id': ('Bug_number', "Bug number."), 'reporter_accessible': ('Reporter_accessible', "Accessible to the bug reporter."), 'assignee_accessible': ('Assignee_accessible', "Accessible to the assignee."), 'qacontact_accessible': ('QAContact_accessible', "Accessible to the QA Contact."), 'cclist_accessible': ('CCList_accessible', "Accessible to the CC List."), } def configuration(config): # Check Bugzilla specific configuration parameters. check_config.check_host(config.dbms_host, 'dbms_host') check_config.check_int(config.dbms_port, 'dbms_port') check_config.check_string(config.dbms_database, 'dbms_database') check_config.check_string(config.dbms_user, 'dbms_user') check_config.check_string(config.dbms_password, 'dbms_password') check_config.check_string_or_none(config.bugzilla_directory, 'bugzilla_directory') check_config.check_list_of(config.replicated_fields, 'replicated_fields', types.StringType, 'strings') if config.bugzilla_directory != None: # strip any trailing / character if (len(config.bugzilla_directory) > 1 and config.bugzilla_directory[-1:] == '/'): config.bugzilla_directory = config.bugzilla_directory[:-1] if not os.path.isdir(config.bugzilla_directory): # "Configuration parameter 'bugzilla_directory' does not name a # directory." raise error, catalog.msg(303) processmail = config.bugzilla_directory + '/processmail' if not os.access(processmail, os.X_OK): # "Configuration parameter 'bugzilla_directory' does not name a # directory containing a processmail script." raise error, catalog.msg(304) # Handle logger. We need a list of logger objects: loggers = [logger.file_logger(priority = config.log_level), # to stdout logger.sys_logger(priority = config.log_level)] # to syslog if config.log_file != None: # to config.log_file loggers.append(logger.file_logger(open(config.log_file, "a"), priority = config.log_level)) # now a single logger object which logs to all of the logger objects in # our list: config.logger = logger.multi_logger(loggers) # Open a connection to the Bugzilla database. This makes a DB-API # v2.0 connection object. To work with a database other than # MySQL, change this to make an appropriate connection object. # Note that in that case changes are also needed in bugzilla.py # where we deal with MySQL-specific types such as tinyint. db = MySQLdb.connect(host = config.dbms_host, port = config.dbms_port, db = config.dbms_database, user = config.dbms_user, passwd = config.dbms_password) # Make a Bugzilla DB object. Note that this same object is used # subsequently by the replicator itself. config.bugzilla = bugzilla.bugzilla(db, config) # Get the types of the 'bugs' table from Bugzilla bugs_types = config.bugzilla.get_types('bugs') # Get the types of the 'profiles' table from Bugzilla. In particular we # need to know the size of the 'login_name' column. profiles_types = config.bugzilla.get_types('profiles') if not profiles_types.has_key('login_name'): # "Bugzilla's table 'profiles' does not have a 'login_name' column." raise error, catalog.msg(305) if profiles_types['login_name']['type'] != 'text': # "The 'login_name' column of Bugzilla's 'profiles' table does not have # a 'text' type." raise error, catalog.msg(306) user_name_length = profiles_types['login_name']['length'] required_columns = ['bug_status', 'resolution', 'assigned_to', 'short_desc'] for column in required_columns: if not bugs_types.has_key(column): # "Bugzilla's table 'bugs' does not have a '%s' column." raise error, catalog.msg(307, column) if not bugs_types['bug_status']['type'] == 'enum': # "The 'bug_status' column of Bugzilla's 'bugs' table is not an enum type." raise error, catalog.msg(308) if not bugs_types['resolution']['type'] == 'enum': # "The 'resolution' column of Bugzilla's 'bugs' table is not an enum type." raise error, catalog.msg(309) if not 'FIXED' in bugs_types['resolution']['values']: # "The 'resolution' column of Bugzilla's 'bugs' table does not have a # 'FIXED' value." raise error, catalog.msg(310) # Make a list of (Bugzilla state, Perforce state) pairs. state_pairs = make_state_pairs(bugs_types['bug_status']['values'], config.closed_state) # Make a list of possible resolutions. (resolutions,default_resolution) = translate_enum(bugs_types, 'resolution') user_translator = dt_bugzilla.user_translator(config.replicator_address, config.p4_user, allow_unknown = 0) # Work out the legal values of the State field in the jobspec. Note that # "closed" must be a legal state because "p4 fix -c CHANGE JOBNAME" always # sets the State to "closed" even if "closed" is not a legal value. See # job000225. legal_states = map((lambda x: x[1]), state_pairs) if 'closed' not in legal_states: legal_states.append('closed') state_values = string.join(legal_states, '/') # p4_fields maps Bugzilla field name to the jobspec data (number, name, # type, length, dispositon, preset, values, help text, translator). The # five fields 101 to 105 are predefined because they are needed by both # Bugzilla and Perforce and the replicator. The fields Job and Date are # special: they are not replicated from TeamTrack but are required by # Perforce, so we have them here. Note that their help text is given (the # other help texts will be fetched from TeamTrack). p4_fields = { \ '(JOB)': ( 101, 'Job', 'word', 32, 'required', None, None, "The job name.", None ), 'bug_status': ( 102, bz_field_map['bug_status'][0], 'select', bugs_types['bug_status']['length'], 'required', state_pairs[0][1], state_values, bz_field_map['bug_status'][1], dt_bugzilla.status_translator(state_pairs) ), 'assigned_to': ( 103, bz_field_map['assigned_to'][0], 'word', user_name_length, 'required', '$user', None, bz_field_map['assigned_to'][1], user_translator), '(DATE)': ( 104, 'Date', 'date', 20, 'always', '$now', None, "The date this job was last modified.", None ), 'short_desc': ( 105, bz_field_map['short_desc'][0], 'text', bugs_types['short_desc']['length'], 'required', '$blank', None, bz_field_map['short_desc'][1], dt_bugzilla.text_translator() ), 'resolution': ( 106, bz_field_map['resolution'][0], 'select', bugs_types['resolution']['length'], 'required', default_resolution, resolutions, bz_field_map['resolution'][1], enum_translator), '(FILESPECS)': ( 191, 'P4DTI-filespecs', 'text', 0, 'optional', None, None, "Associated filespecs.", None ), '(RID)': ( 192, 'P4DTI-rid', 'word', 32, 'required', 'None', None, "P4DTI replicator identifier. Do not edit!", None ), '(ISSUE)': ( 193, 'P4DTI-issue-id', 'word', 32, 'required', 'None', None, "Bugzilla issue database identifier. Do not edit!", None ), '(USER)': ( 194, 'P4DTI-user', 'word', 32, 'always', '$user', None, "Last user to edit this job. You can't edit this!", None ), } # The other replicated fields will be sequential from this field id. p4_field_id = 110 # Go through the replicated_fields list, build structures and add them to # p4_fields. for dt_field_name in config.replicated_fields: # Field must not be already in the p4_fields structure. if p4_fields.has_key(dt_field_name): if dt_field_name in ['bug_status', 'assigned_to', 'short_desc', 'resolution']: # "Field '%s' specified in 'replicated_fields' is a system # field: leave it out!" raise error, catalog.msg(311, dt_field_name) else: # "Field '%s' appears twice in 'replicated_fields'." raise error, catalog.msg(312, dt_field_name) # Field must exist in Bugzilla if not bugs_types.has_key(dt_field_name): # "Field '%s' specified in 'replicated_fields' list not in Bugzilla # 'bugs' table." raise error, catalog.msg(313, dt_field_name) type = bugs_types[dt_field_name] dt_field_type = type['type'] p4_field_values = None p4_field_length = None if bz_field_map.has_key(dt_field_name): p4_field_name = bz_field_map[dt_field_name][0] p4_field_comment = bz_field_map[dt_field_name][1] else: p4_field_name = "bugzilla_" + dt_field_name p4_field_comment = None if p4_field_comment == None: p4_field_comment = ("Bugzilla's '%s' field" % dt_field_name) # if there is a default, use it. if (type.has_key('default') and type['default'] != None and type['default'] != ''): p4_field_class = 'default' p4_field_default = type['default'] else: p4_field_class = 'optional' p4_field_default = None if dt_field_type == 'other': # "Field '%s' specified in 'replicated_fields' list has type '%s': # this is not yet supported by P4DTI." raise error, catalog.msg(314, (dt_field_name, type['sql_type'])) elif dt_field_type == 'float': # "Field '%s' specified in 'replicated_fields' list has # floating-point type: this is not yet supported by P4DTI." raise error, catalog.msg(315, dt_field_name) elif dt_field_type == 'user': p4_field_type = 'word' p4_field_length = user_name_length trans = user_translator elif dt_field_type == 'enum': p4_field_type = 'select' (p4_field_values, p4_field_default) = translate_enum(bugs_types, dt_field_name) trans = translator.translator() elif dt_field_type == 'int': p4_field_type = 'word' trans = dt_bugzilla.int_translator() elif dt_field_type == 'date': p4_field_type = 'date' p4_field_length = 20 trans = dt_bugzilla.date_translator() elif dt_field_type == 'timestamp': p4_field_type = 'date' p4_field_length = 20 trans = dt_bugzilla.timestamp_translator() elif dt_field_type == 'text': p4_field_type = 'text' trans = dt_bugzilla.text_translator() # "p4 -G" uses the field "code" to indicate whether the Perforce # command succeeded or failed. See job000003. if p4_field_name == 'code': # "You can't have a field called 'code' in the Perforce jobspec." raise error, catalog.msg(316) # Fixed-length fields get the length from Bugzilla. if p4_field_length == None: p4_field_length = type['length'] if dt_field_name in read_only_fields: p4_field_comment = (p4_field_comment + " DO NOT MODIFY.") if dt_field_name in append_only_fields: p4_field_comment = (p4_field_comment + " ONLY MODIFY BY APPENDING.") p4_fields[dt_field_name] = \ ( p4_field_id, p4_field_name, p4_field_type, p4_field_length, p4_field_class, p4_field_default, p4_field_values, p4_field_comment, trans ) p4_field_id = p4_field_id + 1 if p4_field_id >= 191: # "Too many fields to replicate: Perforce jobs can contain only 99 # fields." raise error, catalog.msg(317) # Build a jobspec, suitable for "p4 -G jobspec -i". jobspec = {} p4_fields_list = p4_fields.values() def p4_fields_less(x,y): if x[0] < y[0]: return -1 elif x[0] > y[0]: return 1 else: # "Jobspec fields '%s' and '%s' have the same number %d." raise error, catalog.msg(318, (x[1],y[1],x[0])) p4_fields_list.sort(p4_fields_less) # We really do want this code to do this. # See the result of p4.p4.run("p4 jobspec -o"). i = 0 for field in p4_fields_list: jobspec['Fields%d' % i] = "%s %s %s %s %s" % field[0:5] i = i + 1 i = 0 for field in p4_fields_list: if field[6] != None: jobspec['Values%d' % i] = "%s %s" % (field[1], field[6]) i = i + 1 i = 0 for field in p4_fields_list: if field[5] != None: jobspec['Presets%d' % i] = "%s %s" % (field[1], field[5]) i = i + 1 def comment(field): return "# %s: %s\n" % (field[1], field[7]) jobspec['Comments'] = ("# DO NOT CREATE NEW JOBS IN PERFORCE. " "USE BUGZILLA.\n#\n" + string.join(map(comment, p4_fields_list), "")) # Set configuration parameters needed by dt_bugzilla. config.append_only_fields = append_only_fields config.read_only_fields = read_only_fields config.jobname_function = lambda bug: 'bug%d' % bug['bug_id'] # Set configuration parameters needed by the replicator. config.date_translator = dt_bugzilla.date_translator() config.job_owner_field = 'Assigned_To' config.job_status_field = 'Status' config.job_date_field = 'Date' config.text_translator = dt_bugzilla.text_translator() config.user_translator = \ dt_bugzilla.user_translator(config.replicator_address, config.p4_user, allow_unknown = 1) # The field_map parameter is a list of triples (Bugzilla database field # name, Perforce field name, translator) required by the replicator. We # use the filter to remove the fields that aren't replicated: these have no # translator. config.field_map = \ map(lambda item: (item[0], item[1][1], item[1][8]), filter(lambda item: item[1][8] != None, p4_fields.items())) return (jobspec, config) # A. REFERENCES # # [RB 2000-08-10] "Perforce Defect Tracking Integration Administrator's # Guide"; Richard Brooksby; Ravenbrook Limited; 2000-08-10. # # # B. DOCUMENT HISTORY # # 2000-12-07 RB Branched and adapted from configure_teamtrack.py. # # 2000-12-08 GDR Translate state name "ignore" to "_ignore". # # 2000-12-13 NB State names need to be lower case in Perforce. Also logger # object needed for the bugzilla.bugzilla object. # # 2000-12-15 NB Added verbosity support. # # 2001-01-11 NB Added support for replicated_fields. Also support # closed_state, and move MySQL connection code here (out of bugzilla.py) # so that we only open one connection when starting up. Also sorted out # the size of the jobspec fields. # # 2001-01-12 NB Moved configuration of read-only and append-only fields here # from dt_bugzilla.py. Added configuration of fields not recorded in the # bugs_activity table. Added comments for read-only and append-only fields. # # 2001-01-15 NB Added a table for field names and comments, because the # automatically-generated ones were terrible. Added validation for # config parameters. # # 2001-01-18 NB Removed bugzilla_user. Moved configuration checks out to # check_config.py. Pass replicator_user to Bugzilla. # # 2001-01-19 NB closed_state and log_file may be None. Use system logger. # # 2001-01-25 NB Added bugzilla_directory to support processmail. # # 2001-01-26 NB Pass p4_server_description to Bugzilla. # # 2001-02-04 GDR Added start_date parameter. # # 2001-02-08 NB Prevent the existence of DateTime from changing our behaviour. # (job000193). # # 2001-02-09 NB Added checks for bugzilla_directory. # # 2001-02-13 GDR Allow administrator_address and smtp_server to be None. # # 2001-02-16 NB Added replicate_p configuration parameter. # # 2001-02-19 NB Moved keyword translation to p4.py. # # 2001-02-22 GDR Moved keyword translation to keyword.py. Made sure that # 'closed' is a legal job state in Perforce. Added text translator to # replicator config. # # 2001-02-26 GDR Refer to "Bugzilla" explicitly in messages, not just "defect # tracker". # # 2001-03-02 RB Transferred copyright to Perforce under their license. # # 2001-03-12 GDR Use messages for errors. # # 2001-03-13 GDR Removed verbose parameter; added log_level. Removed # P4DTI-action field. Made P4DTI-filespec field optional. Get # keyword_translator from translator, not keyword. # # 2001-03-15 GDR Store configuration in config module. # # 2001-03-23 GDR Added job-date-field to replicator configuration. # # 2001-09-19 NB Bugzilla 2.14 (job000390): new fields, some # functionality moved to bugzilla.py