# Perforce Defect Tracking Integration Project # # # REPLICATOR.PY -- P4DTI REPLICATOR # # Gareth Rees, Ravenbrook Limited, 2000-08-09 # # # 1. INTRODUCTION # # This Python module implements the P4DTI replicator: the component of # the P4DTI that copies data from the defect tracker to Perforce and # vice versa [RB 2000-08-10], in order to keep the defect tracker state # consistent with the Perforce state [Requirements, 1] and to provide # the ability to ask questions involving both the defect tracking system # and Perforce [Requirements, 5]. # # The replicator is independent of any particular defect tracker: it # interacts with the defect tracker through the abstract interfaces # declared in the dt_interface module and documented in [GDR 2000-10-16, # 7]. This is to make it possible to integrate Perforce with new defect # tracking systems [Requirements, 20, 21] and to simplify the design of # the replicator so that it is modifiable [Requirements, 25], stable # [Requirements, 27] and maintainable [Requirements, 30]. # # See [GDR 2000-09-13] for the design of the replicator, its algorithms, # and the specification of the data it stores in Perforce. # # The intended readership of this document is project developers. # # This document is not confidential. import catalog import dt_interface import logger import message import p4 import re import smtplib import socket import string import sys import time import stacktrace import types # 2. CURSOR WRAPPER FOR LISTS # # list_cursor is a class that wraps up a list as a cursor with a # fetchone() method. # # This class is used because the specification of the 'all_issues' and # 'changed_entities' methods in the defect_tracker class has changed # since P4DTI 1.1.1 was released. In release 1.1.1 they were documented # to return lists of issues. Now they are documented to return cursors. # We still want to support people who wrote code that returned lists, so # this module examines the results of these methods and wraps lists with # this class. class list_cursor: def __init__(self, list): self.list = list def fetchone(self): if self.list: result = self.list[0] self.list = self.list[1:] return result else: return None # 3. DEFECT TRACKER INTERFACE TO PERFORCE # # The replicator attempts to be as symmetric as possible, for simplicity # of design. It treats Perforce as much as possible in the same way as # the defect tracker. # # It should be possible to develop a dt_perforce module that implements # a full defect tracker interface to Perforce. However, there will need # to be some changes to the abstract interface [GDR 2000-10-16, 7] # because the situation is not 100% symmetric (for example, changelists # are only replicated in one direction). # # We haven't had the time to develop the revised interface and the # implementation, so for the moment the dt_perforce class is a # placeholder. Eventually it will be fully functional and take over all # Perforce operations from the replicator, which can then be simplified # and made fully symmetric. class dt_perforce(dt_interface.defect_tracker): error = "Perforce interface error" p4 = None def __init__(self, p4_interface, config): assert isinstance(p4_interface, p4.p4) self.p4 = p4_interface # 4. REPLICATOR class replicator: # 4.1. Data # Configuration module. config = None # Defect tracker. dt = None # Defect tracker interface to Perforce. This is a placeholder; at # the moment the implementation is incomplete, but eventually it # will be there. dt_p4 = None # Replicator identifier. rid = None # Interface to Perforce. p4 = None # The number of columns to format e-mail messages to. columns = 80 # The replicator's counter on the Perforce server. counter = None # Error object for fatal errors raised by the replicator. error = 'P4DTI Replicator error' # Do we support migration of Perforce jobs to defect tracker issues? support_migrate_jobs = 0 # Do we support migration of Perforce users to the defect tracker? support_migrate_users = 0 # Do we support replication of new jobs in Perforce to the defect # tracker? support_replicate_new_jobs = 0 # Missing migration parameters (this allows us to better explain why # we don't support migration). missing_migration_parameters = None # 4.2. Initialization def __init__(self, dt, p4_interface, config): assert isinstance(dt, dt_interface.defect_tracker) assert isinstance(p4_interface, p4.p4) self.dt = dt self.config = config self.rid = config.rid # This is incomplete. Eventually there will be a real defect # tracker interface to Perforce. self.dt_p4 = dt_perforce(p4_interface, config) self.p4 = p4_interface # Replicator ids must match. if self.rid != self.dt.rid: # "The replicator's RID ('%s') doesn't match the defect # tracker's RID ('%s')." raise self.error, catalog.msg(833, (self.rid, self.dt.rid)) # Make a counter name for this replicator. if not self.counter: self.counter = 'P4DTI-%s' % self.rid # Make a client for the replicator. self.p4.run('client -i', self.p4.run('client -o')[0]) # Initialize the defect tracking system. dt_interface.defect_tracker_issue.replicate_p = self.config.replicate_p self.dt.init() # Determine if certain features are supported: we do this simply # by checking to see if the defect tracker and the configuration # offer the required interface. self.support_migrate_jobs = 1 self.missing_migration_parameters = [] for m in ['new_issue', 'new_issues_start', 'new_issues_end']: if getattr(self.dt, m, None) == None: self.support_migrate_jobs = 0 self.support_replicate_new_jobs = self.support_migrate_jobs for p in ['pre_migrate_issue', 'prepare_issue_advanced', 'migrated_jobspec_description', 'migrate_p']: if not self.config.__dict__.has_key(p): self.support_migrate_jobs = 0 self.missing_migration_parameters.append(p) self.support_migrate_users = 1 for m in ['add_user']: if getattr(self.dt, m, None) == None: self.support_migrate_users = 0 # check_jobspec(). Check that P4DTI-* fields are present in the # jobspec. def check_jobspec(self): if not self.p4.jobspec_has_p4dti_fields(self.p4.get_jobspec()): # "P4DTI fields not found in Perforce jobspec." raise self.error, catalog.msg(836) # check_migration(). Check that the migration from Perforce jobs to # the defect tracker is possible. If not, raise an error. def check_migration(self): if self.support_migrate_jobs: return if self.missing_migration_parameters: # "Your configuration doesn't support migration. These # parameters need values: %s." raise self.error, catalog.msg(907, self.missing_migration_parameters) else: # "The P4DTI does not support migration of Perforce jobs to # issues in the defect tracker '%s'." raise self.error, catalog.msg(905, self.config.dt_name) # start_logger(). Has the logger been started? If not, start it. # (We must be careful not to set the logger counter to 0 more than # once; this will confuse Perforce [Seiwald 2000-09-11].) def start_logger(self): logger_started = 0 counters = self.p4.run('counters') for c in counters: if c.get('counter') == 'logger': return self.p4.run('counter logger 0') # 4.3. Logging # log(msg, args = ()). Write the message to the replicator's log. # The msg argument can be a message instance, the number of a # message in the catalog (with the remaining arguments used to fill # in the message parameters), or a string. def log(self, msg, args = ()): if isinstance(msg, message.message): self.config.logger.log(msg) elif isinstance(msg, types.IntType): self.config.logger.log(catalog.msg(msg, args)) else: # "%s" self.config.logger.log(catalog.msg(910, str(msg))) # 4.4. Perforce interface # # These methods provide the replicator with an interface to # Perforce. In time they can be moved to the dt_perforce class # (section 3). # all_jobs(). Return a list of all jobs. def all_jobs(self): return self.p4.run('jobs') # changed_entities(). Return a 3-tuple consisting of (a) changed # jobs, (b) changed changelists, and (c) the last log entry that # was considered. The changed jobs are those that are due for # replication by this replicator (that is, the P4DTI-rid field of # the job matches the replicator id), or new jobs which pass the # replicate_job_p check. The last log entry will be passed to # mark_changes_done. def changed_entities(self): # Get all entries from the log since the last time we updated # the counter. log_entries = self.p4.run('logger -t %s' % self.counter) jobs = {} changelists = [] last_log_entry = None # The last entry number in the log. for e in log_entries: last_log_entry = int(e['sequence']) if e['key'] == 'job': jobname = e['attr'] if jobname == 'new': # "Perforce has a job called 'new', which is # illegal and will stop the P4DTI from working." raise self.error, catalog.msg(896) if not jobs.has_key(jobname): job = self.job(jobname) p4dti_rid = job.get('P4DTI-rid', 'None') if (p4dti_rid == self.rid or (p4dti_rid == 'None' and self.config.replicate_job_p(job))): # We ought to make sure not to return jobs that # were last updated by the replicator, by # looking at the P4DTI-user field in each job. # But this doesn't work yet: see job000014. # and job.has_key('P4DTI-user') # and job['P4DTI-user']!=self.config.p4_user jobs[jobname] = job elif e['key'] == 'change': # Collect new and updated changelists here. A # changelist can change (using p4 change -f) without any # related jobs changing, so we need to replicate # changelists as well as replicating the fixes of # changed jobs. change_number = e['attr'] try: changelist = self.p4.run('change -o %s' % change_number)[0] changelists.append(changelist) except p4.error: # The changelist might not exist any more: it might # have been a pending changelist that's been # renumbered. So don't replicate it. Should it be # deleted from the defect tracker? GDR 2000-11-02. pass return jobs, changelists, last_log_entry # mark_changes_done(log_entry). Update the Perforce database to # record the fact that the replicator has replicated all changes up # to log_entry. def mark_changes_done(self, log_entry): assert log_entry == None or isinstance(log_entry, types.IntType) # Update counter to last entry number in the log that we've # replicated. If this is the last entry in the log, it has the # side-effect of deleting the log (see "p4 help undoc"). if log_entry: self.p4.run('logger -t %s -c %d' % (self.counter, log_entry)) # clear_logger(). Clear the logger. def clear_logger(self): last_log_entry = self.p4.run('counter logger')[0]['data'] self.p4.run('logger -t %s -c %s' % (self.counter, last_log_entry)) # job(jobname). Return the Perforce job with the given name if it # exists, or an empty job specification (otherwise). def job(self, jobname): assert isinstance(jobname, types.StringType) jobs = self.p4.run('job -o %s' % jobname) if len(jobs) != 1 or not jobs[0].has_key('Job'): # "Expected a job but found %s." raise self.error, catalog.msg(837, str(jobs)) # Compare job names case-insensitively (see job000313). elif string.lower(jobs[0]['Job']) != string.lower(jobname): # "Asked for job '%s' but got job '%s'." raise self.error, catalog.msg(838, (jobname, jobs[0]['Job'])) else: return jobs[0] # job_exists(job). Return true if the job exists, false if it is # new. We do this by looking for the field 'P4DTI-user'. This is # an 'always' field, so it does not appear in new jobs. This really # is an effective way of determining if a job exists: see [Seiwald # 2000-11-28]. def job_exists(self, job): return job.has_key('P4DTI-user') # job_filespecs(job). Return a list of filespecs for the given job. # Each element of the list is a filespec, as a string. def job_filespecs(self, job): assert isinstance(job, types.DictType) if job.has_key('P4DTI-filespecs'): filespecs = string.split(job['P4DTI-filespecs'], '\n') else: filespecs = [] # Since Perforce text fields are terminated with a newline, the # last item of the list should be empty. Remove it. if filespecs: if filespecs[-1] != '': # "P4DTI-filespecs field has value '%s': this should end # in a newline." raise self.error, catalog.msg(839, job['P4DTI-filespecs']) filespecs = filespecs[:-1] return filespecs # job_fixes(job). Return a list of fixes for the given job. Each # element of the list is a dictionary with keys Change, Client, # User, Job, and Status. def job_fixes(self, job): assert isinstance(job, types.DictType) return self.p4.run('fixes -j %s' % job['Job']) # job_format(job). Format a job so that people can read it. Also, # indent the first line of the job so that it can be included in the # body of a mail message without being wrapped; see mail(). def job_format(self, job): def format_item(i): key, value = i if '\n' in value: if value[-1] == '\n': value = value[0:-1] value = string.join(string.split(value,'\n'),'\n\t') return "%s:\n\t%s" % (key, value) else: return "%s: %s" % (key, value) items = job.items() # Remove special Perforce system fields. items = filter(lambda i: i[0] not in ['code','specdef'], items) # Sort into lexical order. items.sort() return string.join(map(format_item, items), '\n') # job_mail_recipients(job). Work out the people associated with the # job who should receive e-mail when there's a problem with that job # (namely the job's owner and the last person to edit the job, # unless either of these is the replicator). def job_mail_recipients(self, job): recipients = [] # Owner of the job, if any. owner = None if (job.has_key(self.config.job_owner_field)): owner = job[self.config.job_owner_field] owner_address = self.user_email_address(owner) if owner_address: recipients.append(('Job owner', owner_address)) # Last person to change the job, if neither replicator nor # owner. if (job.has_key('P4DTI-user') and job['P4DTI-user'] != self.config.p4_user): changer = self.user_email_address(job['P4DTI-user']) if changer and changer != owner: recipients.append(('Job changer', changer)) return recipients # job_modifier(job). Return our best guess at who last modified the # job. # # The reason we have this method is that (as of Perforce 2000.2) the # "always" fields in a job don't get modified when a job is fixed # (or a fix is deleted for a job). This means that the P4DTI-user # field (which is always set to $user) may not be accurate, since # there may have been fixes added later. # # So our strategy for finding the owner is as follows: # # 1. Is there a fix record, submitted more recently than the job # has been modified, by someone other than the replicator? If so, # take the person who submitted the most recent such fix as the # modifier. # # 2. If not, does the P4DTI-user field contain a user other than # the replicator? If so, take them as the modifier. # # 3. If not, take the job owner as the modifier. # # Note that this doesn't give a 100% accurate answer (for example, # if you fix a job and then delete the fix), but it's right in all # but a few exceptional cases. # # See job000133 and job000270 for the motivation. def job_modifier(self, job): if job['P4DTI-user'] == self.config.p4_user: modifier = job[self.config.job_owner_field] else: modifier = job['P4DTI-user'] # Dates in job fields look like 2000/12/31 23:59:59, but dates # in fixes are seconds since 1970-01-01 00:00:00, so convert the # job modification time to an integer for comparison. match = re.match('^(\d{4})/(\d{2})/(\d{2}) ' '(\d{2}):(\d{2}):(\d{2})$', job[self.config.job_date_field]) if not match: # "Job '%s' has a date field in the wrong format: %s." raise self.error, catalog.msg(889, (job['Job'], job)) date = time.mktime(tuple(map(int, match.groups()) + [0,0,-1])) fixes = self.job_fixes(job) for f in fixes: if (int(f['Date']) > date and f['User'] != self.config.p4_user): modifier = f['User'] date = int(f['Date']) return modifier # update_job(job, changes). Update the job in Perforce by applying # the given changes. Also update the "job" dictionary to reflect # these changes, and also any changes made by Perforce, such as # picking up the new jobname (if job['Job'] is 'new'). update_job_re = re.compile('^Job ([^ ]+)') def update_job(self, job, changes = {}): assert isinstance(job, types.DictType) assert isinstance(changes, types.DictType) for key, value in changes.items(): job[key] = value results = self.p4.run('job -i', job) # Check that the results of the 'job -i' command are as # expected: Perforce should say something like 'Job job012345 # saved.' or 'Job job012345 not changed.' If the jobname was # 'new', then record the jobname that Perforce gave the new # job so that we can call setup_for_replication() in # replicate_many(). if (len(results) == 1 and results[0].has_key('data')): match = self.update_job_re.match(results[0]['data']) if not match or match.group(1) == 'new': # "Expected Perforce output of 'job -i' to say 'Job # jobname ...', but found '%s'." raise self.error, catalog.msg(897, results[0]['data']) elif job['Job'] == 'new': job['Job'] = match.group(1) elif job['Job'] != match.group(1): # "Tried to update job '%s', but Perforce replied '%s'." raise self.error, catalog.msg(899, (job['Job'], results[0]['data'])) else: # "Unexpected output from Perforce command 'job -i': %s." raise self.error, catalog.msg(898, results) # Return the e-mail address of a Perforce user, or None if the # address can't be found. def user_email_address(self, user): assert isinstance(user, types.StringType) # Even though "p4 user -o foo" doesn't actually create a user, # it does fail if the user foo doesn't exist and there are no # spare licences. So trap that case. See job000204. try: u = self.p4.run('user -o %s' % user) except: return None # Existing users have Access and Update fields in the returned # structure; non-existing users don't. if (len(u) == 1 and u[0].has_key('Access') and u[0].has_key('Update') and u[0].has_key('Email')): return u[0]['Email'] else: return None # 4.5. Entry points # check_consistency(). Run a consistency check on the two # databases, reporting any inconsistencies. def check_consistency(self): # "Checking consistency for replicator '%s'." self.log(871, self.rid) self.check_jobspec() inconsistencies = 0 # Number of inconsistencies found. # Get issues and jobs. issues_cursor = self.dt.all_issues() if getattr(issues_cursor, 'fetchone', None) == None: issues_cursor = list_cursor(issues_cursor) issue_id_to_job = {} jobs = {} for j in self.p4.run('jobs -e P4DTI-rid=%s' % self.rid): jobs[j['Job']] = j while 1: issue = issues_cursor.fetchone() if issue == None: break id = issue.id() jobname = issue.corresponding_id() # "Checking issue '%s' against job '%s'." self.log(890, (id, jobname)) # Report if issue has no corresponding job. if issue.rid() != self.rid: if issue.replicate_p(): # "Issue '%s' should be replicated but is not." self.log(872, id) inconsistencies = inconsistencies + 1 continue issue_id_to_job[id] = jobname if not jobs.has_key(jobname): # "Issue '%s' should be replicated to job '%s' but that # job either does not exists or is not replicated." self.log(873, (id, jobname)) inconsistencies = inconsistencies + 1 continue # Get corresponding job. job = jobs[jobname] del jobs[jobname] # Report if mapping is in error. if job['P4DTI-issue-id'] != id: # "Issue '%s' is replicated to job '%s' but that job is # replicated to issue '%s'." self.log(874, (id, jobname, job['P4DTI-issue-id'])) inconsistencies = inconsistencies + 1 # Report if job and issue don't match. changes = self.translate_issue_dt_to_p4(issue, job) if changes: # "Job '%s' would need the following set of changes in # order to match issue '%s': %s." self.log(875, (jobname, id, str(changes))) inconsistencies = inconsistencies + 1 # Report if the sets of filespecs differ. p4_filespecs = self.job_filespecs(job) dt_filespecs = issue.filespecs() diffs = self.filespecs_differences(dt_filespecs, p4_filespecs) for p4_filespec, dt_filespec in diffs: if p4_filespec and not dt_filespec: # "Job '%s' has associated filespec '%s' but there # is no corresponding filespec for issue '%s'." self.log(876, (jobname, p4_filespec, id)) inconsistencies = inconsistencies + 1 elif not p4_filespec and dt_filespec: # "Issue '%s' has associated filespec '%s' but there # is no corresponding filespec for job '%s'." self.log(877, (id, dt_filespec.name(), jobname)) inconsistencies = inconsistencies + 1 else: # Corresponding filespecs can't differ (since their # only attribute is their name). assert 0 # Report if the sets of fixes differ. p4_fixes = self.job_fixes(job) dt_fixes = issue.fixes() diffs = self.fixes_differences(dt_fixes, p4_fixes) for p4_fix, dt_fix in diffs: if p4_fix and not dt_fix: # "Change %s fixes job '%s' but there is no # corresponding fix for issue '%s'." self.log(878, (p4_fix['Change'], jobname, id)) inconsistencies = inconsistencies + 1 elif not p4_fix and dt_fix: # "Change %d fixes issue '%s' but there is no # corresponding fix for job '%s'." self.log(879, (dt_fix.change(), id, jobname)) inconsistencies = inconsistencies + 1 else: # "Change %s fixes job '%s' with status '%s', but # change %d fixes issue '%s' with status '%s'." self.log(880, (p4_fix['Change'], jobname, p4_fix['Status'], dt_fix.change(), id, dt_fix.status())) inconsistencies = inconsistencies + 1 # There should be no remaining jobs, so any left are in error. for job in jobs.values(): if issue_id_to_job.has_key(job['P4DTI-issue-id']): # "Job '%s' is marked as being replicated to issue '%s' # but that issue is being replicated to job '%s'." self.log(881, (job['Job'], job['P4DTI-issue-id'], issue_id_to_job[job['P4DTI-issue-id']])) inconsistencies = inconsistencies + 1 else: # "Job '%s' is marked as being replicated to issue '%s' # but that issue either doesn't exist or is not being # replicated by this replicator." self.log(882, (job['Job'], job['P4DTI-issue-id'])) inconsistencies = inconsistencies + 1 # Report on success/failure. if len(issue_id_to_job) == 1: # "Consistency check completed. 1 issue checked." self.log(883) else: # "Consistency check completed. %d issues checked." self.log(884, len(issue_id_to_job)) if inconsistencies == 0: # "Looks all right to me." self.log(885) elif inconsistencies == 1: # "1 inconsistency found." self.log(886) else: # "%d inconsistencies found." self.log(887, inconsistencies) # migrate_users() ensures that there is a defect tracker user # corresponding to each Perforce user. def migrate_users(self): if not self.support_migrate_users: # "The P4DTI does not support migration of Perforce users to # the defect tracker '%s'." raise self.error, catalog.msg(906, self.config.dt_name) p4_users = self.p4.run("users") for user in p4_users: self.dt.add_user(user['User'], user['Email'], user['FullName']) # migrate() migrates all existing Perforce jobs to the DT. Note # that we can't just call replicate_new_issue_p4_to_dt here because # that method assumes we have the new jobspec in place (so that we # can replicate backwards) which we don't until migration is # finished. Instead, we replicate backwards in a bunch after # migration succeeds. def migrate(self): self.check_migration() if self.p4.jobspec_has_p4dti_fields(self.p4.get_jobspec()): # "It looks as if migration has already been run (the P4 # jobspec has P4DTI fields). Please revert the Perforce and # defect tracker databases before attempting to run # migration again. A future P4DTI release will have a # migration script which will handle this better." raise self.error, catalog.msg(903) pairs=[] jobs = self.all_jobs() try: self.dt.new_issues_start() for job in jobs: if self.config.migrate_p(job): self.config.pre_migrate_issue(self.config, self.dt, self.dt_p4, job) issue = self.create_issue(job) # "Migrated job '%s' to issue '%s'." self.log(892, (job['Job'], issue.readable_name())) # Migrate information about this issue only if # we're going to replicate it. if issue.replicate_p(): # Save for post-migration replication. Note # that we save the issue ID, not the issue, # for scalability reasons. (for some defect # trackers, each issue object may be very # large). pairs.append((job, issue.id())) # replicate fixes and changelists self.replicate_fixes_p4_to_dt(issue, job) # replicate filespecs self.replicate_filespecs_p4_to_dt(issue, job) finally: self.dt.new_issues_end() # consistency checks which could go in here: have all the # jobs been migrated? (i.e. are they in the defect tracker # with the correct P4DTI meta-data) if self.config.migrated_jobspec_description: self.p4.install_jobspec(self.config.migrated_jobspec_description) # "Installed post-migration jobspec." self.log(893) for pair in pairs: job, issue_id = pair issue = self.dt.issue(issue_id) job['P4DTI-rid'] = self.rid job['P4DTI-issue-id'] = issue_id # "Post-migration replication of issue '%s' to job # '%s'." self.log(894, (issue.readable_name(), job['Job'])) self.replicate_issue_dt_to_p4(issue, job) # "Migration completed." self.log(895) # poll(). Poll the defect tracker and Perforce, replicate changes, # then stop. def poll(self): self.check_jobspec() self.start_logger() self.poll_databases() # refresh_perforce_jobs(). Delete all Perforce jobs then replicate # all issues, filespecs and fixes from the defect tracker. def refresh_perforce_jobs(self): self.check_jobspec() self.start_logger() jobs = self.p4.run('jobs') for job in jobs: self.p4.run('job -d %s' % job['Job']) self.replicate_all_dt_to_p4() self.clear_logger() # carefully_poll_databases(). Poll once, handling exceptions def carefully_poll_databases(self): try: self.poll_databases() # Reset poll period when the poll was successful. self.poll_period = self.config.poll_period except AssertionError: # Assertions indicate severe bugs in the replicator. It # might cause serious data corruption if we continue. # We also want these failures to be reported, and they # might go unreported if the replicator carried on # going. raise except KeyboardInterrupt: # Allow people to stop the replicator with Control-C. raise except: self.mail_report( # "The replicator failed to poll successfully." catalog.msg(863), # "The replicator failed to poll successfully, # because of the following problem:" [catalog.msg(864)]) # The poll failed; it's likely that it will fail again # for the same reason the next time we poll. Back off # exponentially so as not to mail bomb the admin. See # job000215 and job000135. self.poll_period = self.poll_period * 2 # prepare_to_run(). Invoked once when run() is called, to preform # startup tasks. def prepare_to_run(self): self.check_jobspec() self.start_logger() self.poll_period = self.config.poll_period self.mail_startup_message() # run(). Repeatedly (handling exceptions) poll and replicate # changes. def run(self): self.prepare_to_run() while 1: self.carefully_poll_databases() time.sleep(self.poll_period) # 4.6. E-mail # mail(recipients, subject, body). Send e-mail to the given # recipients (pls the administrator) with the given subject and # body. The recipients argument is a list of pairs (role, address). # The body argument is a list of paragraphs. Paragraphs belonging # to the message.message class will be wrapped to 80 columns. # Ordinary strings will be left alone. Log the contents of the # message. def mail(self, recipients, subject, body): assert isinstance(recipients, types.ListType) assert isinstance(subject, message.message) assert isinstance(body, types.ListType) # Always e-mail the administrator recipients.append(('P4DTI administrator', self.config.administrator_address)) # Build the contents of the RFC822 To: header. to = string.join(map(lambda r: "%s <%s>" % r, recipients), ', ') # "Mailing '%s' re: '%s'." self.log(800, (to, subject)) map(self.log, body) # Don't send e-mail if administrator_address or smtp_server is # None. if (self.config.administrator_address == None or self.config.smtp_server == None): return smtp = smtplib.SMTP(self.config.smtp_server) message_paragraphs = [ ("From: %s\n" "To: %s\n" "Subject: %s" % (self.config.replicator_address, to, subject)), # "This is an automatically generated e-mail from the # Perforce Defect Tracking Integration replicator '%s'." catalog.msg(865, self.rid), ] + body def fmt(s, columns = self.columns): if isinstance(s, message.message): return s.wrap(columns) else: return str(s) message_text = string.join(map(fmt, message_paragraphs), "\n\n") smtp.sendmail(self.config.replicator_address, map(lambda r: r[1], recipients), message_text) smtp.quit() # exception_message(exc_info). Return a message object describing # the given exception, or None if there was no exception. The # exc_info argument must be the results of calling sys.exc_info(). def exception_message(self, exc_info): exc_type, exc_value, _ = exc_info if exc_type == 'TeamShare API error' and not exc_value: # "There was no error message from TeamTrack. The most # likely reasons for this problem are: you don't have # permission to update the issue; the job contained data # that was invalid in TeamTrack; or the job was missing a # field that is required in TeamTrack." return catalog.msg(850) elif isinstance(exc_value, message.message): return exc_value elif exc_type is not None: # "Defect tracker error (%s): %s" return catalog.msg(891, (exc_type, exc_value)) else: # We're not in the context of an exception, so there's # nothing to report. return None # mail_report(subject, intro, extra=[], job=None). Compose and send # e-mail when something's gone wrong. If a job argument is # supplied, it's the job to which the mail applies, and is used to # deduce who to send the e-mail to. If no job argument is supplied, # then mail is to the administrator (only). def mail_report(self, subject, intro, extra=[], job=None): assert isinstance(subject, message.message) assert isinstance(intro, types.ListType) assert isinstance(extra, types.ListType) for m in intro + extra: assert (isinstance(m, types.StringType) or isinstance(m, message.message)) assert job is None or isinstance(job, types.DictType) exc_info = sys.exc_info() msg = self.exception_message(exc_info) if msg is not None: body = intro + [ msg ] + extra + [ # "Here's a full Python traceback:" catalog.msg(852), string.join(apply(stacktrace.format_exception, exc_info), ''), ] else: body = intro + extra if job is None: self.mail([], subject, body) else: # "If you are having continued problems, please contact your # P4DTI administrator <%s>." body.append(catalog.msg(853, self.config.administrator_address)) self.mail(self.job_mail_recipients(job), subject, body) # mail_startup_message(self). Send a message to the administrator # when the replicator starts to run. It exercises the SMTP server, # which is the only way we can really test that part of the # configuration. This is very important, because the replicator may # often be run unattended, so we can't rely on log messages being # read. # # Also this is a good time to tell the administrator about any # unmatched and duplicate user records, as he may wish to take # action to fix them. def mail_startup_message(self): unmatches = self.config.user_translator.unmatched_users(self.dt, self.dt_p4) (unmatched_dt_users, unmatched_p4_users, dt_user_msg, p4_user_msg) = unmatches[0:4] if len(unmatches) >= 8: (duplicate_dt_users, duplicate_p4_users, duplicate_dt_msg, duplicate_p4_msg) = unmatches[4:8] else: duplicate_dt_users = None duplicate_p4_users = None # "The P4DTI replicator has started." subject = catalog.msg(866) body = [ subject ] if unmatched_p4_users: body = body + [ # "The following Perforce users do not correspond to # defect tracker users. The correspondence is based on # the e-mail addresses in the defect tracker and # Perforce user records." catalog.msg(867), p4_user_msg, self.format_email_table(unmatched_p4_users), ] if unmatched_dt_users: body = body + [ # "The following defect tracker users do not correspond # to Perforce users. The correspondence is based on the # e-mail addresses in the defect tracker and Perforce # user records." catalog.msg(870), dt_user_msg, self.format_email_table(unmatched_dt_users), ] if duplicate_p4_users: body = body + [ duplicate_p4_msg, self.format_email_table(duplicate_p4_users), ] if duplicate_dt_users: body = body + [ duplicate_dt_msg, self.format_email_table(duplicate_dt_users), ] self.mail([], subject, body) # format_email_table(self, user_dict). Format a table of users and # e-mail addresses. The users argument is a dictoinary mapping # userid to e-mail address. Return a string containing the table. def format_email_table(self, user_dict): # "User" user_header = catalog.msg(868).text # "E-mail address" email_header = catalog.msg(869).text longest_user = len(user_header) longest_email = len(email_header) users = user_dict.keys() users.sort() for u in users: if len(u) > longest_user: longest_user = len(u) if len(user_dict[u]) > longest_email: longest_email = len(user_dict[u]) spaces = longest_user + 2 - len(user_header) table = [ " %s%s%s" % (user_header, ' ' * spaces, email_header), " " + "-" * (longest_user + 2 + longest_email) ] for u in users: email = user_dict[u] if email == '': email = '' spaces = longest_user + 2 - len(u) table.append(" %s%s%s" % (u, ' ' * spaces, email)) return string.join(table, "\n") # 4.7. Replication # fixes_differences(dt_fixes, p4_fixes). Each argument is a list of # fixes for the same job/issue. Return list of pairs (p4_fix, # dt_fix) of corresponding fixes which differ. Elements of pairs # are None where there is no corresponding fix. def fixes_differences(self, dt_fixes, p4_fixes): assert isinstance(dt_fixes, types.ListType) assert isinstance(p4_fixes, types.ListType) # Make hash from change number to p4 fix. p4_fix_by_change = {} for p4_fix in p4_fixes: assert isinstance(p4_fix, types.DictType) p4_fix_by_change[int(p4_fix['Change'])] = p4_fix # Make pairs (dt fix, corresponding p4 fix or None). pairs = [] for dt_fix in dt_fixes: assert isinstance(dt_fix, dt_interface.defect_tracker_fix) if not p4_fix_by_change.has_key(dt_fix.change()): pairs.append((None, dt_fix)) else: p4_fix = p4_fix_by_change[dt_fix.change()] del p4_fix_by_change[dt_fix.change()] if dt_fix.status() != p4_fix['Status']: pairs.append((p4_fix, dt_fix)) # Remaining p4 fixes are unpaired. for p4_fix in p4_fix_by_change.values(): pairs.append((p4_fix, None)) return pairs # filespecs_differences(dt_filespecs, p4_filespecs). Each argument # is a list of filespecs for the same job/issue. Return list of # pairs (p4_filespec, dt_filespec) of filespecs which differ. # Elements of pairs are None where there is no corresponding # filespec (this is always the case since there is no associated # information with a filespec; the function is like this for # consistency with fixes_differences, and so that it is easy to # extend if there is ever a way to associate information with a # filespec, for example the nature of the association -- see # requirement 55). def filespecs_differences(self, dt_filespecs, p4_filespecs): assert isinstance(dt_filespecs, types.ListType) assert isinstance(p4_filespecs, types.ListType) # Make hash from name to p4 filespec. p4_filespec_by_name = {} for p4_filespec in p4_filespecs: assert isinstance(p4_filespec, types.StringType) p4_filespec_by_name[p4_filespec] = p4_filespec # Make pairs (dt filespec, None). pairs = [] for dt_filespec in dt_filespecs: assert isinstance(dt_filespec, dt_interface.defect_tracker_filespec) if not p4_filespec_by_name.has_key(dt_filespec.name()): pairs.append((None, dt_filespec)) else: del p4_filespec_by_name[dt_filespec.name()] # Make pairs (None, p4 filespec). for p4_filespec in p4_filespec_by_name.values(): pairs.append((p4_filespec, None)) return pairs # conflict_policy(issue, job). This method is called when both the # issue and the corresponding job have changed since the last time # they were consistent. Return 'p4' if the Perforce job is correct # and should be replicated to the defect tracker. Return 'dt' if # the defect tracking issue is correct and should be replicated to # Perforce. Return anything else to indicate that the replicator # should take no further action. # # The default policy is to return 'dt'. This is because we're # treating the Perforce jobs database as a scratch copy of the real # data in the defect tracker. So when there's a conflict the defect # tracker is correct. See job000102 for details. def conflict_policy(self, issue, job): assert isinstance(issue, dt_interface.defect_tracker_issue) assert isinstance(job, types.DictType) return 'dt' # poll_databases(). Poll the DTS for changed issues. Poll Perforce # for changed jobs and changelists. Replicate all of these # entities. def poll_databases(self): # "Poll starting." self.log(911) if getattr(self.dt, 'poll_start', None): self.dt.poll_start() try: # Get the changed issues (ignore changed changelists if any # since we only replicate changelists from Perforce to the # defect tracker). changed_issues_cursor, _, dt_marker = self.dt.changed_entities() if getattr(changed_issues_cursor, 'fetchone', None) == None: changed_issues_cursor = list_cursor(changed_issues_cursor) changed_jobs, changelists, p4_marker = self.changed_entities() # Replicate the issues and the jobs. self.replicate_many(changed_issues_cursor, changed_jobs) # Replicate the affected changelists. for c in changelists: self.replicate_changelist_p4_to_dt(c) # Tell the defect tracker and Perforce that we've finished # replicating these changes. self.dt.mark_changes_done(dt_marker) self.mark_changes_done(p4_marker) finally: if getattr(self.dt, 'poll_end', None): self.dt.poll_end() # "Poll finished." self.log(912) # replicate_all_dt_to_p4(). Go through all the issues in the defect # tracker, set them up for replication if necessary, and replicate # them to Perforce. def replicate_all_dt_to_p4(self): all_issues_cursor = self.dt.all_issues() if getattr(all_issues_cursor, 'fetchone', None) == None: all_issues_cursor = list_cursor(all_issues_cursor) self.replicate_many(all_issues_cursor, {}) def replicate_changelist_p4_to_dt(self, changelist): assert isinstance(changelist, types.DictType) change = int(changelist['Change']) client = changelist['Client'] date = self.config.date_translator.translate_1_to_0(changelist['Date'], self.dt, self.dt_p4) description = self.config.text_translator.translate_1_to_0(changelist['Description'], self.dt, self.dt_p4) status = changelist['Status'] user = self.config.user_translator.translate_1_to_0(changelist['User'], self.dt, self.dt_p4) if self.dt.replicate_changelist(change, client, date, description, status, user): # "Replicated changelist %d." self.log(802, change) # replicate_many(issues_cursor, jobs). Replicate the issues and # jobs. The issues argument is a list of issues (which must belong # to a subclass of defect_tracker_issue; the jobs list is a hash # from jobname to job). # # The reason why the arguments have different conventions (list vs # hash) is that the algorithm for getting the changed jobs from the # p4 logger outpt involves constructing a hash from jobname to job, # and it seems silly to turn this hash back into a list only to # immediately turn it back into a hash again. def replicate_many(self, issues_cursor, jobs): assert getattr(issues_cursor, 'fetchone') assert isinstance(jobs, types.DictType) while 1: issue = issues_cursor.fetchone() if issue == None: break assert isinstance(issue, dt_interface.defect_tracker_issue) # Is this the first time the issue is being replicated? first_time = not issue.rid() # Only replicate issues which pass replicate_p. (But if # the issue is already set up for replication, don't ask # again.) if first_time and not issue.replicate_p(): continue # Set the jobname. If we've replicated this issue before, # then the defect tracker knows the jobname. Otherwise, # if the administrator has specified # 'use_perforce_jobnames', use 'new'. if first_time and self.config.use_perforce_jobnames: jobname = 'new' else: jobname = issue.corresponding_id() if jobs.has_key(jobname): job = jobs[jobname] if job['P4DTI-rid'] != self.rid: assert job['P4DTI-rid'] == 'None' # "Issue '%s' is marked as being replicated to job # '%s' but that job is marked as not being # replicated (P4DTI-rid = None)." self.log(900, (issue.id(), jobname)) self.replicate(issue, job, 'both') del jobs[jobname] else: job = self.job(jobname) self.replicate(issue, job, 'dt') if first_time: # If we started out with jobname == 'new', then by # this time it must have been set to the new jobname # by the update_job() method. if job['Job'] == 'new': # "Replicated issue '%s' to Perforce, but didn't # get a jobname for it (the 'Job' field is still # 'new')." raise self.error, catalog.msg(904, (issue.id())) issue.setup_for_replication(job['Job']) # "Set up issue '%s' to replicate to job '%s'." self.log(803, (issue.id(), job['Job'])) # Now go through the remaining changed jobs. for job in jobs.values(): assert isinstance(job, types.DictType) issue_id = job['P4DTI-issue-id'] if issue_id != 'None': issue = self.dt.issue(issue_id) if not issue: # "Asked for issue '%s' but got an error instead." raise self.error, catalog.msg(888, issue_id) self.replicate(issue, job, 'p4') elif self.support_replicate_new_jobs: # Job is new in Perforce, so create new issue in the # defect tracker. self.replicate_new_issue_p4_to_dt(job) # Replicate newly-created job over to defect tracker def replicate_new_issue_p4_to_dt(self, job): jobname = job['Job'] try: issue = self.create_issue(job) except: self.mail_report( # "Job '%s' could not be replicated to the defect # tracker." catalog.msg(908, jobname), # "The replicator failed to replicate Perforce job '%s' # to the defect tracker, because of the following # problem:" [catalog.msg(909, jobname)], [], job) return issuename = issue.readable_name() # "Migrated job '%s' to issue '%s'." self.log(892, (jobname, issuename)) if issue.replicate_p(): try: # The result of replicating back may be different from # the original job. job['P4DTI-rid'] = self.rid job['P4DTI-issue-id'] = issue.id() # "Post-migration replication of issue '%s' to job # '%s'." self.log(894, (issue.readable_name(), job['Job'])) changes = self.translate_issue_dt_to_p4(issue, job) if changes: # "-- Defect tracker made changes as a result of # the update: %s." self.log(826, changes) self.update_job(job, changes) self.replicate_fixes_p4_to_dt(issue, job) self.replicate_filespecs_p4_to_dt(issue, job) except: # Undo our half-completed work: delete the issue and # revert the job. issue.delete() self.update_job(job, { 'P4DTI-rid': 'None', 'P4DTI-issue-id': 'None' }) self.mail_report( # "Job '%s' could not be replicated to issue '%s'." catalog.msg(848, (jobname, issuename)), # "The replicator failed to replicate Perforce job # '%s' to defect tracker issue '%s', because of the # following problem:" [catalog.msg(851, (jobname, issuename))], [], job) # replicate(issue, job, changed). Replicate an issue to or from the # corresponding job. The changed argument is 'dt' if the defect # tracking issue has changed but not the Perforce job; 'p4' if vice # versa; 'both' if both have changed. # # Basically this method is a series of conditions that end in one of # the following cases: # # 1. Replicate the issue to the job or vice versa (the normal mode # of operation). # # 2. Overwrite the job with the issue or vice versa (if they have # both changed and the conflict policy says to overwrite). This is # just like replication, except that the old version of the # overwritten entity gets mailed to its owner as a record in case # data was lost. # # 3. Do nothing (if both have changed and the conflict policy says # to do nothing). # # 4. Revert the job from the issue (if we tried to replicate the # job to the issue but it failed, probably due to lack of privileges # or invalid data). def replicate(self, issue, job, changed): assert isinstance(issue, dt_interface.defect_tracker_issue) assert isinstance(job, types.DictType) assert changed in ['dt','p4','both'] issuename = issue.readable_name() jobname = job['Job'] # Figure out what to do with this issue and job. Do nothing? # Overwrite issue with job? Overwrite job with issue? # Only the defect tracker issue has changed. if changed == 'dt': # "Replicating issue '%s' to job '%s'." self.log(804, (issuename, jobname)) self.replicate_issue_dt_to_p4(issue, job) # Only the Perforce job has changed. elif changed == 'p4': # "Replicating job '%s' to issue '%s'." self.log(805, (jobname, issuename)) try: self.replicate_issue_p4_to_dt(issue, job) except: self.revert_issue_dt_to_p4(issue, job) # Both have changed. Apply the conflict resolution policy. else: assert changed == 'both' # "Issue '%s' and job '%s' have both changed. Consulting # conflict resolution policy." self.log(806, (issuename, jobname)) decision = self.conflict_policy(issue, job) if decision == 'dt': # "Defect tracker issue '%s' and Perforce job '%s' # have both changed since the last time the replicator # polled the databases. The replicator's conflict # resolution policy decided to overwrite the job with # the issue." reason = [ catalog.msg(841, (issuename, jobname)) ] self.overwrite_issue_dt_to_p4(issue, job, reason) elif decision == 'p4': # "Defect tracker issue '%s' and Perforce job '%s' # have both changed since the last time the replicator # polled the databases. The replicator's conflict # resolution policy decided to overwrite the issue # with the job." reason = [ catalog.msg(842, (issuename, jobname)) ] self.overwrite_issue_p4_to_dt(issue, job, reason) else: # "Conflict resolution policy decided: no action." self.log(807) # revert_issue_dt_to_p4(self, issue, job). This is called when an # error has occurred in replicating from Perforce to the defect # tracker. The most likely reason for this is a privilege failure # (the user is not allowed to edit that issue in that way) or a # failure to put valid values in the job fields. In this case, set # the job back to a copy of the issue. def revert_issue_dt_to_p4(self, issue, job): assert isinstance(issue, dt_interface.defect_tracker_issue) assert isinstance(job, types.DictType) exc_info = sys.exc_info() issuename = issue.readable_name() jobname = job['Job'] try: # Get the issue again, since it might have been changed in # memory in the course of the failed replication. Note new # variable name so as not to overwrite the old issue. (Can # we avoid all this nonsense by keeping better track of old # and new issues?) GDR 2000-10-31. issue_2 = self.dt.issue(issue.id()) if not issue_2: # "Issue '%s' not found." raise self.error, catalog.msg(840, issue.id()) self.overwrite_issue_dt_to_p4( issue_2, job, [ # "The replicator failed to replicate Perforce job '%s' # to defect tracker issue '%s', because of the following # problem:" catalog.msg(851, (jobname, issuename)), ]) except: # Replicating back to Perforce failed. Report both errors # to the administrator. self.mail_report( # "Job '%s' could not be replicated to issue '%s'." catalog.msg(848, (jobname, issuename)), [ # "The replicator failed to replicate Perforce job '%s' # to defect tracker issue '%s' because of this problem:" catalog.msg(854, (jobname, issuename)), self.exception_message(exc_info), # "Here's a full Python traceback:" catalog.msg(852), string.join(apply(stacktrace.format_exception, exc_info), ''), # "The replicator attempted to restore the job to a copy # of the issue, but this failed too, because of the # following problem:" catalog.msg(855), ], [ # "The replicator has now given up." catalog.msg(856), ]) # overwrite_issue_p4_to_dt(self, issue, job, reason). As # replicate_issue_p4_to_dt, but e-mails an old copy of the issue to # the owner of the job and the administrator. The reason argument # is a list of message objects giving the reason for the # overwriting. def overwrite_issue_p4_to_dt(self, issue, job, reason): assert isinstance(issue, dt_interface.defect_tracker_issue) assert isinstance(job, types.DictType) assert isinstance(reason, types.ListType) issuename = issue.readable_name() jobname = job['Job'] # "Overwrite issue '%s' with job '%s'." self.log(810, (issuename, jobname)) # Build e-mail before overwriting so we get the old issue. # "Issue '%s' overwritten by job '%s'." subject = catalog.msg(857, (issuename, jobname)) extra = [ # "The replicator has therefore overwritten defect tracker # issue '%s' with Perforce job '%s'." catalog.msg(858, (issuename, jobname)), # "The defect tracker issue looked like this before being # overwritten:" catalog.msg(859), str(issue), ] self.replicate_issue_p4_to_dt(issue, job) self.mail_report(subject, reason, extra, job) # overwrite_issue_dt_to_p4(self, issue, job, reason). # As replicate_issue_dt_to_p4, but e-mails an old copy of the # issue to the owner of the job and the administrator. The reason # argument is a list of strings or message objects given a reason # for the overwriting. The appendix argument (also a list of # strings or message objects) gives extra information about the # reason that should come at the end of the message. def overwrite_issue_dt_to_p4(self, issue, job, reason): assert isinstance(issue, dt_interface.defect_tracker_issue) assert isinstance(job, types.DictType) assert isinstance(reason, types.ListType) issuename = issue.readable_name() jobname = job['Job'] # "Overwrite job '%s' with issue '%s'." self.log(811, (jobname, issuename)) # Build e-mail before overwriting so we get the old job. # "Job '%s' overwritten by issue '%s'." subject = catalog.msg(860, (jobname, issuename)) extra = [ # "The replicator has therefore overwritten Perforce job # '%s' with defect tracker issue '%s'. See section 2.2 of # the P4DTI User Guide for more information." catalog.msg(861, (jobname, issuename)), # "The job looked like this before being overwritten:" catalog.msg(862), self.job_format(job), ] self.replicate_issue_dt_to_p4(issue, job) self.mail_report(subject, reason, extra, job) # replicate_issue_dt_to_p4(issue, old_job). Replicate the given # issue from the defect tracker to Perforce. def replicate_issue_dt_to_p4(self, issue, job): assert isinstance(issue, dt_interface.defect_tracker_issue) assert isinstance(job, types.DictType) # Transform the issue into a job. This has to be done first # because the job might be new, and we won't be able to # replicate fixes or filespecs until the job's been created # (p4 fix won't accept non-existent jobnames). I suppose I # could create a dummy job to act as a placeholder here, but # that's not easy at all -- you have to know quite a lot about # the jobspec to be able to create a job. changes = self.translate_issue_dt_to_p4(issue, job) if changes: # "-- Changed fields: %s." self.log(812, changes) self.update_job(job, changes) else: # "-- No issue fields were replicated." self.log(813) # Replicate filespecs. dt_filespecs = issue.filespecs() p4_filespecs = self.job_filespecs(job) if self.filespecs_differences(dt_filespecs, p4_filespecs): names = map(lambda(f): f.name(), dt_filespecs) self.update_job(job, { 'P4DTI-filespecs': string.join(names,'\n') }) # "-- Filespecs changed to '%s'." self.log(814, string.join(names)) # Replicate fixes. p4_fixes = self.job_fixes(job) dt_fixes = issue.fixes() job_status = job[self.config.job_status_field] diffs = self.fixes_differences(dt_fixes, p4_fixes) for p4_fix, dt_fix in diffs: if p4_fix and not dt_fix: self.p4.run('fix -d -c %s %s' % (p4_fix['Change'], p4_fix['Job'])) # "-- Deleted fix for change %s." self.log(815, p4_fix['Change']) elif not p4_fix and dt_fix: try: self.p4.run('fix -s %s -c %d %s' % (dt_fix.status(), dt_fix.change(), issue.corresponding_id())) except p4.error, message: # We get an error here if the changelist was somehow # deleted. In this case there's not much we can do # except log the error. See job000128. self.log(message) else: job_status = dt_fix.status() # "-- Added fix for change %d with status %s." self.log(816, (dt_fix.change(), dt_fix.status())) elif p4_fix['Status'] != dt_fix.status(): self.p4.run('fix -s %s -c %d %s' % (dt_fix.status(), dt_fix.change(), issue.corresponding_id())) job_status = dt_fix.status() # "-- Fix for change %d updated to status %s." self.log(817, (dt_fix.change(), dt_fix.status())) else: # This should't happen, since fixes_differences returns # only a list of pairs which differ. assert 0 # It might be the case that the job status has been changed by # replicating a fix from the defect tracker. But this changed # status won't be right. So restore the correct status if # necessary. if job_status != job[self.config.job_status_field]: self.update_job(job, { 'Status': job[self.config.job_status_field] }) # replicate_fixes_p4_to_dt(issue, job). Replicate fixes for the # given job from Perforce to the defect tracker. Ensures that the # changelists for the fixes are also replicated. Raise an # exception on failure. # If you change this function, you may have to change the regression # test for job000385; see test_p4dti.py. def replicate_fixes_p4_to_dt(self, issue, job, failed_before = 0): p4_fixes = self.job_fixes(job) dt_fixes = issue.fixes() fix_diffs = self.fixes_differences(dt_fixes, p4_fixes) for p4_fix, dt_fix in fix_diffs: if dt_fix and not p4_fix: dt_fix.delete() # "-- Deleted fix for change %d." self.log(818, dt_fix.change()) else: # p4 fix has changed # "-- Considering Perforce fix %s." self.log(819, p4_fix) (change, client, date, status, user) = self.translate_fix_p4_to_dt(p4_fix) # make sure changelist is replicated try: changelist = self.p4.run('change -o %s' % change)[0] except p4.error: # The changelist might have been renumbered since we # called job_fixes; see job000385. If it has, then # try again. But don't get stuck in an infinite # loop. if failed_before: raise else: self.replicate_fixes_p4_to_dt(issue, job, failed_before = 1) return self.replicate_changelist_p4_to_dt(changelist) if not dt_fix: # new fix; add to DT issue.add_fix(change, client, date, status, user) # "-- Added fix for change %s with status %s." self.log(820, (p4_fix['Change'], p4_fix['Status'])) elif dt_fix.status() != p4_fix['Status']: # status changed dt_fix.update(change, client, date, status, user) # "-- Fix for change %s updated to status %s." self.log(821, (p4_fix['Change'], p4_fix['Status'])) else: # This should't happen, since fixes_differences # returns only a list of pairs which differ. assert 0 # replicate_filespecs_p4_to_dt(issue, job). Replicate fixes for # the given job from Perforce to the defect tracker. Raise an # exception on failure. def replicate_filespecs_p4_to_dt(self, issue, job): p4_filespecs = self.job_filespecs(job) dt_filespecs = issue.filespecs() filespec_diffs = self.filespecs_differences(dt_filespecs, p4_filespecs) for p4_filespec, dt_filespec in filespec_diffs: if dt_filespec and not p4_filespec: dt_filespec.delete() # "-- Deleted filespec %s." self.log(822, dt_filespec.name()) elif not dt_filespec: issue.add_filespec(p4_filespec) # "-- Added filespec %s." self.log(823, p4_filespec) else: # This should't happen, since filespecs_differences # returns only a list of pairs which differ. assert 0 # replicate_issue_p4_to_dt(issue, job). Replicate the given job # from Perforce to the defect tracker. def replicate_issue_p4_to_dt(self, issue, job): assert isinstance(issue, dt_interface.defect_tracker_issue) assert isinstance(job, types.DictType) # Replicate fixes. self.replicate_fixes_p4_to_dt(issue, job) # Replicate filespecs. self.replicate_filespecs_p4_to_dt(issue, job) # Transform the job into an issue and update the issue. changes = self.translate_issue_p4_to_dt(issue, job) if changes: # "-- Changed fields: %s." self.log(824, repr(changes)) p4_user = self.job_modifier(job) dt_user = self.config.user_translator.translate_1_to_0(p4_user, self.dt, self.dt_p4) issue.update(dt_user, changes) else: # "-- No job fields were replicated." self.log(825) # The issue may have changed as a consequence of updating it. # For example, in TeamTrack the issue's owner changes when an # issue goes through a transition. So we fetch the issue again, # check for changes and replicate them back to the job if we # find them. See job000053. new_issue = self.dt.issue(issue.id()) if not new_issue: # "Issue '%s' not found." raise self.error, catalog.msg(840, issue.id()) new_changes = self.translate_issue_dt_to_p4(new_issue, job) if new_changes: # "-- Defect tracker made changes as a result of the update: # %s." self.log(826, new_changes) self.update_job(job, new_changes) def replicate_changelists(self): # Replicate all the changelists. # "Checking changelists to see if they need replicating..." self.log(827) changelists = self.p4.run('changes') # "-- %d changelists to check." self.log(828, len(changelists)) for c in changelists: c2 = self.p4.run('change -o %s' % c['change'])[0] self.replicate_changelist_p4_to_dt(c2) # translate_fix_p4_to_dt(p4_fix). def translate_fix_p4_to_dt(self, p4_fix): assert isinstance(p4_fix, types.DictType) change = int(p4_fix['Change']) client = p4_fix['Client'] date = self.config.date_translator.translate_1_to_0(p4_fix['Date'], self.dt, self.dt_p4) status = p4_fix['Status'] user = self.config.user_translator.translate_1_to_0(p4_fix['User'], self.dt, self.dt_p4) return (change, client, date, status, user) # translate_issue_dt_to_p4(issue, job). Return changes as a # dictionary but don't apply them yet. def translate_issue_dt_to_p4(self, issue, job): assert isinstance(issue, dt_interface.defect_tracker_issue) assert isinstance(job, types.DictType) changes = { } # Do the P4DTI fields need to be changed? If so, record in # changes. for key, value in [('P4DTI-rid', self.rid), ('P4DTI-issue-id', issue.id())]: if job[key] != value: changes[key] = value # What about the replicated fields? for dt_field, p4_field, trans in self.config.field_map: p4_value = trans.translate_0_to_1(issue[dt_field], self.dt, self.dt_p4, issue, job) # A field not appearing in the job is equivalent to the # field having a value of the empty string. See job000181. if ((job.has_key(p4_field) and p4_value != job[p4_field]) or (not job.has_key(p4_field) and p4_value != '')): changes[p4_field] = p4_value return changes # translate_issue_p4_to_dt(issue, job). Return changes as a # dictionary but don't apply them yet. def translate_issue_p4_to_dt(self, issue, job): assert isinstance(issue, dt_interface.defect_tracker_issue) assert isinstance(job, types.DictType) changes = {} for dt_field, p4_field, trans in self.config.field_map: if job.has_key(p4_field): p4_value = job[p4_field] else: # Missing fields indicate optional fields without a # value -- this happens when the empty string has been # supplied for the value. So supply the empty string # ourselves. See job000181. p4_value = '' dt_value = trans.translate_1_to_0(p4_value, self.dt, self.dt_p4, issue, job) if dt_value != issue[dt_field]: changes[dt_field] = dt_value return changes # create_issue(job). Makes a new issue corresponding to the # job. Returns the new issue. def create_issue(self, job): assert isinstance(job, types.DictType) dict = {} for dt_field, p4_field, trans in self.config.field_map: if job.has_key(p4_field): p4_value = job[p4_field] else: # Missing fields indicate optional fields without a # value -- this happens when the empty string has been # supplied for the value. So supply the empty string # ourselves. See job000181. When migrating, this # will also happen for fields which we are about to # add to the jobspec. p4_value = '' dt_value = trans.translate_1_to_0(p4_value, self.dt, self.dt_p4, None, job) dict[dt_field] = dt_value self.config.prepare_issue_advanced(self.config, self.dt, self.dt_p4, dict, job) return self.dt.new_issue(dict, job['Job']) # A. REFERENCES # # [GDR 2000-09-13] "Replicator design"; Gareth Rees; Ravenbrook Limited; # 2000-09-13. # # [GDR 2000-10-16] "Perforce Defect Tracking Integration Integrator's # Guide" (living document); Gareth Rees; Ravenbrook Limited; 2000-10-16. # # [RB 2000-08-10] "Perforce Defect Tracking Integration Architecture"; # Richard Brooksby; Ravenbrook Limited; 2000-08-10. # # [Requirements] "Perforce Defect Tracking Integration Project # Requirements" (living document); Gareth Rees; Ravenbrook Limited; # 2000-05-24; . # # [Seiwald 2000-09-11] "Re: Is 'p4 counter logger 0' idempotent?" # (e-mail message); Christopher Seiwald; Perforce Software; # ; # 2000-09-11. # # [Seiwald 2000-11-28] "Re: Can we rely on 'always' fields not appearing # in 'job -o newjob'?" (e-mail message); Christopher Seiwald; Perforce # Software; ; # 2000-11-28. # # # B. DOCUMENT HISTORY # # 2000-12-05 NB addess -> address # # 2000-12-05 GDR Imported p4 module so replicator can catch p4.error. # Added replicator method mail_concerning_job() for e-mailing people # about a job. There were several places where the owner of a job was # been fetched and e-mailed, some of which were buggy. This method # replaces all those instances, hopefully correctly. # # 2000-12-06 GDR Fixed the replicator's user_email_address method so # that it really returns None when there is no such user. # # 2000-12-06 GDR Updated supported Perforce changelevel to 18974 (this # is the changelevel we document against). # # 2000-12-06 GDR Fixing job000133 (replicator gets wrong user when a job # is fixed): When the last person who changed the job is the replicator # user, update the issue on behalf of the job owner instead. # # 2000-12-06 GDR If the owner of a job and the person who last changed # it are the same (a common occurrence), include them only once in any # e-mail sent by the replicator about that job. # # 2000-12-06 GDR E-mail messages from the replicator concerning # overwritten jobs are much improved. # # 2000-12-06 GDR The overwriting methods now send e-mail with the new # issue/job in them, not the old issue/job. # # 2000-12-07 GDR When there's no error message (typically in the case of # assertion failure), say so. Format the job properly in all messages # (including the one sent by the conflict method). Use "Perforce job" # and "defect tracker issue" for clarity. (Even better would be to have # a defect_tracker.name so it could say "TeamTrack issue".) # # 2000-12-07 GDR Created new class dt_perforce; a placeholder for an # eventual full implementation of a defect_tracker subclass that # interfaces to Perforce. # # 2001-01-19 NB Better stack traces. # # 2001-01-19 GDR Handle empty fields. Fix comments. # # 2001-01-23 NB SMTP server test (unmatched users). # # 2001-02-04 GDR Updated definition of defect_tracker.all_issues() # method. # # 2001-02-12 GDR Fixed bug in check_consisteney. # # 2001-02-13 GDR Don't send e-mail if administrator_address or # smtp_server is None. # # 2001-02-14 GDR user_email_address returns None if the user doesn't # exist, even when there are no spare licences (see job000204). # # 2001-02-16 NB Added replicate-p configuration parameter. # # 2001-02-21 GDR The replicator backs off exponentially if it fails to # poll successfully, so as not to mailbomb the administrator. # # 2001-02-22 GDR replicate_changelist_p4_to_dt applies a text translator # to the change description, since a change description can have several # lines of text. # # 2001-02-23 GDR Added a corresponding_id method to the # defect_tracker_issue class. # # 2001-03-02 RB Transferred copyright to Perforce under their license. # # 2001-03-11 GDR Use messages for errors, logging and e-mail. # # 2001-03-13 GDR Removed the recording of conflicts. Conflict # resolution is always immediate. Moved translator class to translator # module. Moved defect tracker interface classes to dt_interface.py. # # 2001-03-14 GDR Use messages when consistency checking. # # 2001-03-15 GDR Get configuration from config module. # # 2001-03-16 GDR Added refresh_perforce_jobs() method. # # 2001-03-21 GDR The setup_for_replication() method takes a jobname # argument. # # 2001-03-23 GDR New method job_modifier returns a best guess at who # last modified the job, to fix job000270. # # 2001-03-24 GDR Check supported Perforce server changelevel in p4.py, # not replicator.__init__ (so that we find out if p4 -G jobspec -i will # work before actually trying it in init.py). # # 2001-03-25 RB Moved message 889 to catalog due to merge from version # 1.0 sources. # # 2001-05-17 GDR Defect tracker methods 'add_issue' and # 'changed_entities' may return cursors as well as lists. # # 2001-05-19 GDR Added progress report for consistency checking. # # 2001-05-22 GDR Compare job names case-insensitively when fetching a # job to work around job000313. # # 2001-06-14 GDR The reason argument to overwrite_issue_dt_to_p4 must # consist only of messages, since they are logged as well as mailed. # Each call to the defect_tracker.issue() method now has error checking. # # 2001-06-15 NB Moved functionality out of init because it's not all # required by all the scripts. Creation of client and calling # defect_tracker.init moved to __init__ method. Checking of jobspec # moved to new method check_jobspec and called from check, refresh and # run. # # 2001-06-25 NB post-migration replication now works! # # 2001-06-25 NB Now support the 'use_perforce_jobnames' configuration # parameter by specifying 'new' for the jobname in replicate_many() # and then recording the jobname when we find out what it is in # update_job(). # # 2001-06-26 NB Now support the creation of new jobs in Perforce. # Also moved the replication of changelists, and changed the interface # to changed_entities (so that changelists are replicated iff the # matching fixes are replicated). # # 2001-06-27 NB Moved code from new_issue out to the defect tracker # (changed new_issue interface). # # 2001-06-29 NB Produce full traceback if migration fails. # # 2001-06-30 GDR The replicator doesn't stop if it can't replicate a fix # because the changelist has been deleted (see job000128). # # 2001-07-04 NB Changed issue creation system so we use the regular # field map. # # 2001-08-06 GDR Specify -1 for DST argument to mktime(). # # 2001-10-02 GDR Include users with duplicate e-mail addresses in # startup message. See job000308. # # 2001-10-03 GDR Handle renumbered changelist race condition during # replication of fixes from Perforce to the defect tracker; see # job000385. # # 2001-10-07 GDR Reformatted as a document. # # 2001-10-23 GDR Renamed poll() as poll_databases(); added poll() entry # point. Report error if jobname is still new after update_job. Don't # call replicate_issue_p4_to_dt after migrating; just replicate fixes # and filespecs. Wrap migration code with checks that the feature is # supported. Protect new_issues_end with a try ... finally. # # 2001-10-29 GDR Send e-mail if create_issue fails. Always log e-mail # even if it doesn't get sent. # # 2001-11-01 NB Add calls to poll_start and poll_end, for job000306. # # 2001-11-05 GDR Rename migrate_issue as prepare_issue_advanced; new # configuration parameter replicate_job_p. # # 2001-11-07 NDL Extracted contents of run() into smaller functions # to make them acessible to NT service code. # # 2001-11-09 NDL Added debug messages at start and end of # poll_databases(). # # # C. COPYRIGHT AND LICENCE # # This file is copyright (c) 2001 Perforce Software, Inc. All rights # reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in # the documentation and/or other materials provided with the # distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT # HOLDERS AND CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS # OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR # TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE # USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH # DAMAGE. # # # $Id: //info.ravenbrook.com/project/p4dti/branch/2001-11-05/nt-service/code/replicator/replicator.py#3 $