# replicator.py -- P4DTI replicator.
# Gareth Rees, Ravenbrook Limited, 2000-08-09.
# $Id: //info.ravenbrook.com/project/p4dti/branch/2001-04-20/migrate-bugzilla/code/replicator/replicator.py#20 $
#
# See "Perforce Defect Tracking Integration Architecture"
# for the architecture of the integration;
# "Replicator design" for the design of the
# replicator; and "Replicator classes in Python"
# for the class organization of the
# replicator.
#
# This file is copyright (c) 2001 Perforce Software, Inc. All rights
# reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# HOLDERS AND CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
# OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
# TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
# DAMAGE.
import catalog
import dt_interface
import logger
import message
import p4
import re
import smtplib
import socket
import string
import sys
import time
import stacktrace
import types
# Cursor wrapper for lists
#
# list_cursor is a class that wraps up a list as a cursor with a
# fetchone() method.
#
# This class is used because the specification of the 'all_issues' and
# 'changed_entities' methods in the defect_tracker class has changed
# since P4DTI 1.1.1 was released. In release 1.1.1 they were documented
# to return lists of issues. Now they are documented to return cursors.
# We still want to support people who wrote code that returned lists, so
# this module examines the results of these methods and wraps lists with
# this class.
class list_cursor:
def __init__(self, list):
self.list = list
def fetchone(self):
if self.list:
result = self.list[0]
self.list = self.list[1:]
return result
else:
return None
# dt_perforce is a defect_tracker subclass that interfaces to Perforce as if it
# were a defect tracker.
#
# For the moment this is a placeholder; the idea is that eventually it will be
# fully functional and take over all Perforce operations from the replicator,
# which can then be simplified and made symmetric.
class dt_perforce(dt_interface.defect_tracker):
error = "Perforce interface error"
p4 = None
def __init__(self, p4_interface, config):
assert isinstance(p4_interface, p4.p4)
self.p4 = p4_interface
# The replicator class is a generic replicator.
class replicator:
# Configuration module.
config = None
# Defect tracker.
dt = None
# Defect tracker interface to Perforce. This is a placeholder; at the
# moment there is no separate dt_perforce class to use, but eventually
# there will be.
dt_p4 = None
# Replicator identifier.
rid = None
# Interface to Perforce.
p4 = None
# The number of columns to format e-mail messages to.
columns = 80
# The replicator's counter on the Perforce server.
counter = None
# Error object for fatal errors raised by the replicator.
error = 'P4DTI Replicator error'
def __init__(self, dt, p4_interface, config):
assert isinstance(dt, dt_interface.defect_tracker)
assert isinstance(p4_interface, p4.p4)
self.dt = dt
self.config = config
self.rid = config.rid
# This is a placeholder. Eventually there will be a real defect
# tracker interface to Perforce.
self.dt_p4 = dt_perforce(p4_interface, config)
self.p4 = p4_interface
# Replicator ids must match.
if self.rid != self.dt.rid:
# "The replicator's RID ('%s') doesn't match the defect tracker's
# RID ('%s')."
raise self.error, catalog.msg(833, (self.rid, self.dt.rid))
# Make a counter name for this replicator.
if not self.counter:
self.counter = 'P4DTI-%s' % self.rid
dt_interface.defect_tracker_issue.replicate_p = self.config.replicate_p
# Make a client for the replicator.
self.p4.run('client -i', self.p4.run('client -o')[0])
# Initialize the defect tracking system.
self.dt.init()
# changed_entities(). Return a 3-tuple consisting of (a) changed
# jobs, (b) changed changelists, and (c) the last log entry that
# was considered. The changed jobs are those that are due for
# replication by this replicator (that is, the P4DTI-rid field of
# the job matches the replicator id), or new jobs which pass the
# migrate_p check. The last log entry will be passed to
# mark_changes_done.
def changed_entities(self):
# Get all entries from the log since the last time we updated the
# counter.
log_entries = self.p4.run('logger -t %s' % self.counter)
jobs = {}
changelists = []
last_log_entry = None # The last entry number in the log.
for e in log_entries:
last_log_entry = int(e['sequence'])
if e['key'] == 'job':
jobname = e['attr']
if jobname == 'new':
# "Perforce has a job called 'new', which is
# illegal and will stop the P4DTI from working."
raise self.error, catalog.msg(896)
if not jobs.has_key(jobname):
job = self.job(jobname)
# new jobs and jobs replicated by us.
if job.has_key('P4DTI-rid'):
p4dti_rid = job['P4DTI-rid']
if (p4dti_rid == self.rid or
(p4dti_rid == 'None' and
self.config.migrate_p(job))):
# We ought to make sure not to return jobs
# that were last updated by the
# replicator, by looking at the P4DTI-user
# field in each job. But this doesn't
# work yet: see job000014.
# and job.has_key('P4DTI-user')
# and job['P4DTI-user']!=self.config.p4_user
jobs[jobname] = job
elif e['key'] == 'change':
# collect changed changelists here. A changelist can
# change (e.g. with p4 change -f) without the related
# job changing, so we need to replicate them as well
# as replicating the fixes of changed jobs.
change_number = e['attr']
try:
changelist = self.p4.run('change -o %s' % change_number)[0]
changelists.append(changelist)
except p4.error:
# The changelist might not exist, because it might have
# been a pending changelist that's been renumbered. So
# don't replicate it. Should it be deleted? GDR
# 2000-11-02.
pass
return jobs, changelists, last_log_entry
# all_jobs(). Return a list of all jobs.
def all_jobs(self):
return self.p4.run('jobs')
# mark_changes_done(log_entry). Update the Perforce database to
# record the fact that the replicator has replicated all changes up to
# log_entry.
def mark_changes_done(self, log_entry):
assert log_entry == None or isinstance(log_entry, types.IntType)
# Update counter to last entry number in the log that we've replicated.
# If this is the last entry in the log, it has the side-effect of
# deleting the log (see "p4 help undoc").
if log_entry:
self.p4.run('logger -t %s -c %d' % (self.counter, log_entry))
# check_consistency(). Run a consistency check on the two databases,
# reporting any inconsistencies.
def check_consistency(self):
# "Checking consistency for replicator '%s'."
self.log(871, self.rid)
self.check_jobspec()
inconsistencies = 0 # Number of inconsistencies found.
# Get issues and jobs.
issues_cursor = self.dt.all_issues()
if getattr(issues_cursor, 'fetchone', None) == None:
issues_cursor = list_cursor(issues_cursor)
issue_id_to_job = {}
jobs = {}
for j in self.p4.run('jobs -e P4DTI-rid=%s' % self.rid):
jobs[j['Job']] = j
while 1:
issue = issues_cursor.fetchone()
if issue == None:
break
id = issue.id()
jobname = issue.corresponding_id()
# "Checking issue '%s' against job '%s'."
self.log(890, (id, jobname))
# Report if issue has no corresponding job.
if issue.rid() != self.rid:
if issue.replicate_p():
# "Issue '%s' should be replicated but is not."
self.log(872, id)
inconsistencies = inconsistencies + 1
continue
issue_id_to_job[id] = jobname
if not jobs.has_key(jobname):
# "Issue '%s' should be replicated to job '%s' but that job
# either does not exists or is not replicated."
self.log(873, (id, jobname))
inconsistencies = inconsistencies + 1
continue
# Get corresponding job.
job = jobs[jobname]
del jobs[jobname]
# Report if mapping is in error.
if job['P4DTI-issue-id'] != id:
# "Issue '%s' is replicated to job '%s' but that job is
# replicated to issue '%s'."
self.log(874, (id, jobname, job['P4DTI-issue-id']))
inconsistencies = inconsistencies + 1
# Report if job and issue don't match.
changes = self.translate_issue_dt_to_p4(issue, job)
if changes:
# "Job '%s' would need the following set of changes in order to
# match issue '%s': %s."
self.log(875, (jobname, id, str(changes)))
inconsistencies = inconsistencies + 1
# Report if the sets of filespecs differ.
p4_filespecs = self.job_filespecs(job)
dt_filespecs = issue.filespecs()
for p4_filespec, dt_filespec in self.filespecs_differences(dt_filespecs, p4_filespecs):
if p4_filespec and not dt_filespec:
# "Job '%s' has associated filespec '%s' but there is no
# corresponding filespec for issue '%s'."
self.log(876, (jobname, p4_filespec, id))
inconsistencies = inconsistencies + 1
elif not p4_filespec and dt_filespec:
# "Issue '%s' has associated filespec '%s' but there is no
# corresponding filespec for job '%s'."
self.log(877, (id, dt_filespec.name(), jobname))
inconsistencies = inconsistencies + 1
else:
# Corresponding filespecs can't differ (since their only
# attribute is their name).
assert 0
# Report if the sets of fixes differ.
p4_fixes = self.job_fixes(job)
dt_fixes = issue.fixes()
for p4_fix, dt_fix in self.fixes_differences(dt_fixes, p4_fixes):
if p4_fix and not dt_fix:
# "Change %s fixes job '%s' but there is no corresponding
# fix for issue '%s'."
self.log(878, (p4_fix['Change'], jobname, id))
inconsistencies = inconsistencies + 1
elif not p4_fix and dt_fix:
# "Change %d fixes issue '%s' but there is no corresponding
# fix for job '%s'."
self.log(879, (dt_fix.change(), id, jobname))
inconsistencies = inconsistencies + 1
else:
# "Change %s fixes job '%s' with status '%s', but change %d
# fixes issue '%s' with status '%s'."
self.log(880, (p4_fix['Change'], jobname, p4_fix['Status'], dt_fix.change(), id, dt_fix.status()))
inconsistencies = inconsistencies + 1
# There should be no remaining jobs, so any left are in error.
for job in jobs.values():
if issue_id_to_job.has_key(job['P4DTI-issue-id']):
# "Job '%s' is marked as being replicated to issue '%s' but
# that issue is being replicated to job '%s'."
self.log(881, (job['Job'], job['P4DTI-issue-id'], issue_id_to_job[job['P4DTI-issue-id']]))
inconsistencies = inconsistencies + 1
else:
# "Job '%s' is marked as being replicated to issue '%s' but
# that issue either doesn't exist or is not being replicated by
# this replicator."
self.log(882, (job['Job'], job['P4DTI-issue-id']))
inconsistencies = inconsistencies + 1
# Report on success/failure.
if len(issue_id_to_job) == 1:
# "Consistency check completed. 1 issue checked."
self.log(883)
else:
# "Consistency check completed. %d issues checked."
self.log(884, len(issue_id_to_job))
if inconsistencies == 0:
# "Looks all right to me."
self.log(885)
elif inconsistencies == 1:
# "1 inconsistency found."
self.log(886)
else:
# "%d inconsistencies found."
self.log(887, inconsistencies)
# fixes_differences(dt_fixes, p4_fixes). Each argument is a list of fixes
# for the same job/issue. Return list of pairs (p4_fix, dt_fix) of
# corresponding fixes which differ. Elements of pairs are None where there
# is no corresponding fix.
def fixes_differences(self, dt_fixes, p4_fixes):
assert isinstance(dt_fixes, types.ListType)
assert isinstance(p4_fixes, types.ListType)
# Make hash from change number to p4 fix.
p4_fix_by_change = {}
for p4_fix in p4_fixes:
assert isinstance(p4_fix, types.DictType)
p4_fix_by_change[int(p4_fix['Change'])] = p4_fix
# Make pairs (dt fix, corresponding p4 fix or None).
pairs = []
for dt_fix in dt_fixes:
assert isinstance(dt_fix, dt_interface.defect_tracker_fix)
if not p4_fix_by_change.has_key(dt_fix.change()):
pairs.append((None, dt_fix))
else:
p4_fix = p4_fix_by_change[dt_fix.change()]
del p4_fix_by_change[dt_fix.change()]
if dt_fix.status() != p4_fix['Status']:
pairs.append((p4_fix, dt_fix))
# Remaining p4 fixes are unpaired.
for p4_fix in p4_fix_by_change.values():
pairs.append((p4_fix, None))
return pairs
# filespecs_differences(dt_filespecs, p4_filespecs). Each argument is a
# list of filespecs for the same job/issue. Return list of pairs
# (p4_filespec, dt_filespec) of filespecs which differ. Elements of pairs
# are None where there is no corresponding filespec (this is always the
# case since there is no associated information with a filespec; the
# function is like this for consistency with fixes_differences, and so that
# it is easy to extend if there is ever a way to associate information with
# a filespec, for example the nature of the association -- see requirement
# 55).
def filespecs_differences(self, dt_filespecs, p4_filespecs):
assert isinstance(dt_filespecs, types.ListType)
assert isinstance(p4_filespecs, types.ListType)
# Make hash from name to p4 filespec.
p4_filespec_by_name = {}
for p4_filespec in p4_filespecs:
assert isinstance(p4_filespec, types.StringType)
p4_filespec_by_name[p4_filespec] = p4_filespec
# Make pairs (dt filespec, None).
pairs = []
for dt_filespec in dt_filespecs:
assert isinstance(dt_filespec, dt_interface.defect_tracker_filespec)
if not p4_filespec_by_name.has_key(dt_filespec.name()):
pairs.append((None, dt_filespec))
else:
del p4_filespec_by_name[dt_filespec.name()]
# Make pairs (None, p4 filespec).
for p4_filespec in p4_filespec_by_name.values():
pairs.append((p4_filespec, None))
return pairs
# check_jobspec(). Check that P4DTI-* fields are present in the
# jobspec. (I can't check the presence of the P4DTI-user field
# this way, since it isn't given a value for a non-existent job
# like the one we're asking for here. I can't check the
# P4DTI-filespecs field, since it's optional.)
def check_jobspec(self):
if not self.p4.jobspec_has_p4dti_fields(self.p4.get_jobspec()):
# "P4DTI fields not found in Perforce jobspec."
raise self.error, catalog.msg(836)
# job(jobname). Return the Perforce job with the given name if it exists,
# or an empty job specification (otherwise).
def job(self, jobname):
assert isinstance(jobname, types.StringType)
jobs = self.p4.run('job -o %s' % jobname)
if len(jobs) != 1 or not jobs[0].has_key('Job'):
# "Expected a job but found %s."
raise self.error, catalog.msg(837, str(jobs))
# Compare job names case-insensitively (see job000313).
elif string.lower(jobs[0]['Job']) != string.lower(jobname):
# "Asked for job '%s' but got job '%s'."
raise self.error, catalog.msg(838, (jobname, jobs[0]['Job']))
else:
return jobs[0]
# job_exists(job). Return true if the job exists, false if it is new. We
# do this by looking for the field 'P4DTI-user'. This is an 'always'
# field, so it does not appear in new jobs. This really is an effective
# way of determining if a job exists: see Chris Seiwald's e-mail,
# 2000-12-28 17:31:46 GMT.
def job_exists(self, job):
return job.has_key('P4DTI-user')
# job_filespecs(job). Return a list of filespecs for the given job. Each
# element of the list is a filespec, as a string.
def job_filespecs(self, job):
assert isinstance(job, types.DictType)
if job.has_key('P4DTI-filespecs'):
filespecs = string.split(job['P4DTI-filespecs'], '\n')
else:
filespecs = []
# Since Perforce text fields are terminated with a newline, the last
# item of the list should be empty. Remove it.
if filespecs:
if filespecs[-1] != '':
# "P4DTI-filespecs field has value '%s': this should end in a
# newline."
raise self.error, catalog.msg(839, job['P4DTI-filespecs'])
filespecs = filespecs[:-1]
return filespecs
# job_fixes(job). Return a list of fixes for the given job. Each element
# of the list is a dictionary with keys Change, Client, User, Job, and
# Status.
def job_fixes(self, job):
assert isinstance(job, types.DictType)
return self.p4.run('fixes -j %s' % job['Job'])
# job_format(job). Format a job so that people can read it. Also, indent
# the first line of the job so that it can be included in the body of a
# mail message without being wrapped; see mail().
def job_format(self, job):
def format_item(i):
key, value = i
if '\n' in value:
if value[-1] == '\n':
value = value[0:-1]
value = string.join(string.split(value,'\n'),'\n\t')
return "%s:\n\t%s" % (key, value)
else:
return "%s: %s" % (key, value)
items = job.items()
# Remove special Perforce system fields.
items = filter(lambda i: i[0] not in ['code','specdef'], items)
# Sort into lexical order.
items.sort()
return string.join(map(format_item, items), '\n')
# job_modifier(job). Return our best guess at who last modified the job.
#
# The reason we have this method is that (as of Perforce 2000.2) the
# "always" fields in a job don't get modified when a job is fixed (or a fix
# is deleted for a job). This means that the P4DTI-user field (which is
# always set to $user) may not be accurate, since there may have been fixes
# added later.
#
# So our strategy for finding the owner is as follows:
#
# 1. Is there a fix record, submitted more recently than the job has been
# modified, by someone other than the replicator? If so, take the person
# who submitted the most recent such fix as the modifier.
#
# 2. If not, does the P4DTI-user field contain a user other than the
# replicator? If so, take them as the modifier.
#
# 3. If not, take the job owner as the modifier.
#
# Note that this doesn't give an accurate answer (for example, if you fix a
# job and then delete the fix).
#
# See job000133 and job000270.
def job_modifier(self, job):
if job['P4DTI-user'] == self.config.p4_user:
modifier = job[self.config.job_owner_field]
else:
modifier = job['P4DTI-user']
# Dates in job fields look like 2000/12/31 23:59:59, but dates in fixes
# are seconds since 1970-01-01 00:00:00, so convert the job
# modification time to an integer for comparison.
match = re.match('^(\d{4})/(\d{2})/(\d{2}) (\d{2}):(\d{2}):(\d{2})$',
job[self.config.job_date_field])
if not match:
# "Job '%s' has a date field in the wrong format: %s."
raise self.error, catalog.msg(889, (job['Job'], job))
date = time.mktime(tuple(map(int, match.groups()) + [0,0,-1]))
fixes = self.job_fixes(job)
for f in fixes:
if int(f['Date']) > date and f['User'] != self.config.p4_user:
modifier = f['User']
date = int(f['Date'])
return modifier
# log(msg, args = ()). Write the message to the replicator's log.
def log(self, msg, args = ()):
if not isinstance(msg, message.message):
msg = catalog.msg(msg, args)
self.config.logger.log(msg)
# mail(recipients, subject, body). Send e-mail to the given recipients
# (pls the administrator) with the given subject and body. The recipients
# argument is a list of pairs (role, address). The body argument is a list
# of paragraphs. Paragraphs belonging to the message.message class will be
# wrapped to 80 columns. Ordinary strings will be left alone.
def mail(self, recipients, subject, body):
assert isinstance(recipients, types.ListType)
assert isinstance(subject, message.message)
assert isinstance(body, types.ListType)
# Don't send e-mail if administrator_address or smtp_server is None.
if (self.config.administrator_address == None
or self.config.smtp_server == None):
return
# Always e-mail the administrator
recipients.append(('P4DTI administrator',
self.config.administrator_address))
# Build the contents of the RFC822 To: header.
to = string.join(map(lambda r: "%s <%s>" % r, recipients), ', ')
# "Mailing '%s' re: '%s'."
self.log(800, (to, subject))
smtp = smtplib.SMTP(self.config.smtp_server)
message_paragraphs = [
("From: %s\n"
"To: %s\n"
"Subject: %s"
% (self.config.replicator_address, to, subject)),
# "This is an automatically generated e-mail from the
# Perforce Defect Tracking Integration replicator '%s'."
catalog.msg(865, self.rid),
] + body
def fmt(s, columns = self.columns):
if isinstance(s, message.message):
return s.wrap(columns)
else:
return str(s)
message_text = string.join(map(fmt, message_paragraphs), "\n\n")
smtp.sendmail(self.config.replicator_address,
map(lambda r: r[1], recipients),
message_text)
smtp.quit()
# mail(subject, body, job). Send e-mail to the people associated with the
# job (namely the job's owner and the last person to edit the job, unless
# either of these is the replicator).
def mail_concerning_job(self, subject, body, job):
recipients = []
# Owner of the job, if any.
owner = None
if (job.has_key(self.config.job_owner_field)):
owner = self.user_email_address(job[self.config.job_owner_field])
if owner:
recipients.append(('Job owner', owner))
# Last person to change the job, if neither replicator nor owner.
if (job.has_key('P4DTI-user')
and job['P4DTI-user'] != self.config.p4_user):
changer = self.user_email_address(job['P4DTI-user'])
if changer and changer != owner:
recipients.append(('Job changer', changer))
# Send it.
self.mail(recipients, subject, body)
def mail_startup_message(self):
# This message to the administrator is sent when the replicator starts
# to run. It exercises the SMTP server, which is the only way we can
# really test that part of the configuration. This is very important,
# because the replicator may often be run unattended, so we can't rely
# on log messages being read.
#
# Also this is a good time to tell the administrator about any
# unmatched user records, as he may wish to take action to fix them.
#
# NB 2001-01-23
(unmatched_dt_users, unmatched_p4_users, \
dt_user_string, p4_user_string) = \
self.config.user_translator.unmatched_users(self.dt, self.dt_p4)
# "The P4DTI replicator has started."
subject = catalog.msg(866)
body = [ subject ]
if unmatched_p4_users:
body = body + [
# "The following Perforce users do not correspond to defect
# tracker users. The correspondence is based on the e-mail
# addresses in the defect tracker and Perforce user records."
catalog.msg(867),
p4_user_string,
self.format_email_table(unmatched_p4_users),
]
if unmatched_dt_users:
body = body + [
# "The following defect tracker users do not correspond to
# Perforce users. The correspondence is based on the e-mail
# addresses in the defect tracker and Perforce user records."
catalog.msg(870),
dt_user_string,
self.format_email_table(unmatched_dt_users),
]
self.mail([], subject, body)
# format_email_table(self, user_dict). Format a table of users and e-mail
# addresses. The users argument is a dictoinary mapping userid to e-mail
# address. Return a string containing the table.
def format_email_table(self, user_dict):
# "User"
user_header = catalog.msg(868).text
# "E-mail address"
email_header = catalog.msg(869).text
longest_user = len(user_header)
longest_email = len(email_header)
users = user_dict.keys()
users.sort()
for u in users:
if len(u) > longest_user:
longest_user = len(u)
if len(user_dict[u]) > longest_email:
longest_email = len(user_dict[u])
spaces = longest_user + 2 - len(user_header)
table = [ " %s%s%s" % (user_header, ' ' * spaces, email_header),
" " + "-" * (longest_user + 2 + longest_email) ]
for u in users:
email = user_dict[u]
if email == '':
email = ''
spaces = longest_user + 2 - len(u)
table.append(" %s%s%s" % (u, ' ' * spaces, email))
return string.join(table, "\n")
# conflict_policy(issue, job). This method is called when both the issue
# and the corresponding job have changed since the last time they were
# consistent. Return 'p4' if the Perforce job is correct and should be
# replicated to the defect tracker. Return 'dt' if the defect tracking
# issue is correct and should be replicated to Perforce. Return anything
# else to indicate that the replicator should take no further action.
#
# The default policy is to return 'dt'. This is because we're treating the
# Perforce jobs database as a scratch copy of the real data in the defect
# tracker. So when there's a conflict the defect tracker is correct. See
# job000102 for details.
def conflict_policy(self, issue, job):
assert isinstance(issue, dt_interface.defect_tracker_issue)
assert isinstance(job, types.DictType)
return 'dt'
# poll(). Poll the DTS for changed issues. Poll Perforce for changed jobs
# and changelists. Replicate all of these entities.
def poll(self):
# Get the changed issues. (ignore changed changelists if any
# since we only replicate changelists from Perforce to the
# defect tracker).
changed_issues_cursor, _, dt_marker = self.dt.changed_entities()
if getattr(changed_issues_cursor, 'fetchone', None) == None:
changed_issues_cursor = list_cursor(changed_issues_cursor)
changed_jobs, changelists, p4_marker = self.changed_entities()
# Replicate the issues and the jobs.
self.replicate_many(changed_issues_cursor, changed_jobs)
# Replicate the affected changelists.
for c in changelists:
self.replicate_changelist_p4_to_dt(c)
# Tell the defect tracker and Perforce that we've finished
# replicating these changes.
self.dt.mark_changes_done(dt_marker)
self.mark_changes_done(p4_marker)
# refresh_perforce_jobs(). Delete all Perforce jobs then replicate all
# issues, filespecs and fixes from the defect tracker.
def refresh_perforce_jobs(self):
self.check_jobspec()
jobs = self.p4.run('jobs')
for job in jobs:
self.p4.run('job -d %s' % job['Job'])
self.replicate_all_dt_to_p4()
# replicate_all_dt_to_p4(). Go through all the issues in the defect
# tracker, set them up for replication if necessary, and replicate them to
# Perforce.
def replicate_all_dt_to_p4(self):
all_issues_cursor = self.dt.all_issues()
if getattr(all_issues_cursor, 'fetchone', None) == None:
all_issues_cursor = list_cursor(all_issues_cursor)
self.replicate_many(all_issues_cursor, {})
def replicate_changelist_p4_to_dt(self, changelist):
assert isinstance(changelist, types.DictType)
change = int(changelist['Change'])
client = changelist['Client']
date = self.config.date_translator.translate_1_to_0(changelist['Date'], self.dt, self.dt_p4)
description = self.config.text_translator.translate_1_to_0(changelist['Description'], self.dt, self.dt_p4)
status = changelist['Status']
user = self.config.user_translator.translate_1_to_0(changelist['User'], self.dt, self.dt_p4)
if self.dt.replicate_changelist(change, client, date, description, status, user):
# "Replicated changelist %d."
self.log(802, change)
# replicate_many(issues_cursor, jobs). Replicate the issues and
# jobs. The issues argument is a list of issues (which must belong
# to a subclass of defect_tracker_issue; the jobs list is a hash
# from jobname to job).
#
# The reason why the arguments have different conventions (list vs hash) is
# that the algorithm for getting the changed jobs from the p4 logger outpt
# involves constructing a hash from jobname to job, and it seems silly to
# turn this hash back into a list only to immediately turn it back into a
# hash again.
def replicate_many(self, issues_cursor, jobs):
assert getattr(issues_cursor, 'fetchone')
assert isinstance(jobs, types.DictType)
while 1:
issue = issues_cursor.fetchone()
if issue == None:
break
assert isinstance(issue, dt_interface.defect_tracker_issue)
# Is this the first time the issue is being replicated?
first_time = not issue.rid()
# Only replicate issues which pass replicate_p. (But if
# the issue is already set up for replication, don't ask
# again.)
if first_time and not issue.replicate_p():
continue
# Set the jobname. If we've replicated this issue before,
# then the defect tracker knows the jobname. Otherwise,
# if the administrator has specified
# 'use_perforce_jobnames', use 'new'.
if first_time and self.config.use_perforce_jobnames:
jobname = 'new'
else:
jobname = issue.corresponding_id()
if jobs.has_key(jobname):
job = jobs[jobname]
if job['P4DTI-rid'] != self.rid:
assert job['P4DTI-rid'] == 'None'
# "Issue '%s' is marked as being replicated to job
# '%s' but that job is marked as not being
# replicated (P4DTI-rid = None)."
self.log(900, (issue.id(), jobname))
self.replicate(issue, job, 'both')
del jobs[jobname]
else:
job = self.job(jobname)
self.replicate(issue, job, 'dt')
if first_time:
# If we started out with jobname == 'new', then by
# this time it must have been set to the new
# jobname by update_job().
assert job['Job'] != 'new'
issue.setup_for_replication(job['Job'])
# "Set up issue '%s' to replicate to job '%s'."
self.log(803, (issue.id(), job['Job']))
# Now go through the remaining changed jobs.
for job in jobs.values():
assert isinstance(job, types.DictType)
issue_id = job['P4DTI-issue-id']
if issue_id == 'None':
# Job is new in Perforce, so create new issue in the
# defect tracker.
self.replicate_new_issue_p4_to_dt(job)
else:
issue = self.dt.issue(issue_id)
if not issue:
# "Asked for issue '%s' but got an error instead."
raise self.error, catalog.msg(888, issue_id)
self.replicate(issue, job, 'p4')
# Replicate newly-created job over to defect tracker
def replicate_new_issue_p4_to_dt(self, job):
issue = self.create_issue(job)
# "Migrated job '%s' to issue '%s'."
self.log(892, (job['Job'], issue.readable_name()))
if issue.replicate_p():
try:
# The result of replicating back may be different from
# the original job.
job['P4DTI-rid'] = self.rid
job['P4DTI-issue-id'] = issue.id()
# "Post-migration replication of issue '%s' to job
# '%s'."
self.log(894, (issue.readable_name(), job['Job']))
changes = self.translate_issue_dt_to_p4(issue, job)
if changes:
# "-- Defect tracker made changes as a result of
# the update: %s."
self.log(826, changes)
self.update_job(job, changes)
# now go on to call this, which will apply the
# replication field map (rather than the migration
# field map) and also replicate fixes and filespecs.
self.replicate_issue_p4_to_dt(issue, job)
except:
issuename = issue.readable_name()
jobname = job['Job']
issue.delete()
self.update_job(job, { 'P4DTI-rid': 'None',
'P4DTI-issue-id': 'None' })
exc_type, exc_value, exc_traceback = sys.exc_info()
formatted_traceback = string.join(stacktrace.format_exception(exc_type, exc_value, exc_traceback), '')
# "Job '%s' could not be replicated to issue '%s'."
subject = catalog.msg(848, (jobname, issuename))
# "Job '%s' could not be replicated to issue '%s': %s:
# %s"
self.log(808, (jobname, issuename,
exc_type, exc_value))
if exc_type == 'TeamShare API error' and not exc_value:
body = [
# "The replicator failed to replicate Perforce
# job '%s' to defect tracker issue '%s'.
# There was no error message. See the Python
# traceback below for more details about the
# error."
catalog.msg(849, (jobname, issuename)),
# "The most likely reasons for this problem
# are: you don't have permission to update the
# issue; the job contained data that was
# invalid in TeamTrack; or the job was missing
# a field that is required in TeamTrack."
catalog.msg(850),
]
else:
body = [
# "The replicator failed to replicate Perforce
# job '%s' to defect tracker issue '%s',
# because of the following problem:"
catalog.msg(851, (jobname, issuename)),
]
if isinstance(exc_value, message.message):
body.append(exc_value)
else:
# "Defect tracker error (%s): %s"
body.append(catalog.msg(891, (exc_type,
exc_value)))
body = body + [
# "Here's a full Python traceback:"
catalog.msg(852),
formatted_traceback,
# "If you are having continued problems, please
# contact your P4DTI administrator <%s>."
catalog.msg(853, self.config.administrator_address),
]
for r in body:
self.log(r)
self.mail_concerning_job(subject, body, job)
return 1
# replicate(issue, job, changed). Replicate an issue to or from the
# corresponding job. The changed argument is 'dt' if the defect tracking
# issue has changed but not the Perforce job; 'p4' if vice versa; 'both' if
# both have changed.
#
# Basically this method is a series of conditions that end in one of the
# following cases:
#
# 1. Replicate the issue to the job or vice versa (the normal mode of
# operation).
#
# 2. Overwrite the job with the issue or vice versa (if they have both
# changed and the conflict policy says to overwrite). This is just like
# replication, except that the old version of the overwritten entity gets
# mailed to its owner as a record in case data was lost.
#
# 3. Do nothing (if both have changed and the conflict policy says to do
# nothing).
#
# 4. Revert the job from the issue (if we tried to replicate the job to the
# issue but it failed, probably due to lack of privileges or invalid data).
def replicate(self, issue, job, changed):
assert isinstance(issue, dt_interface.defect_tracker_issue)
assert isinstance(job, types.DictType)
assert changed in ['dt','p4','both']
issuename = issue.readable_name()
jobname = job['Job']
# Figure out what to do with this issue and job. Do nothing?
# Overwrite issue with job? Overwrite job with issue?
# Only the defect tracker issue has changed.
if changed == 'dt':
# "Replicating issue '%s' to job '%s'."
self.log(804, (issuename, jobname))
self.replicate_issue_dt_to_p4(issue, job)
# Only the Perforce job has changed.
elif changed == 'p4':
# "Replicating job '%s' to issue '%s'."
self.log(805, (jobname, issuename))
try:
self.replicate_issue_p4_to_dt(issue, job)
except:
self.revert_issue_dt_to_p4(issue, job)
# Both have changed. Apply the conflict resolution policy.
else:
assert changed == 'both'
# "Issue '%s' and job '%s' have both changed. Consulting
# conflict resolution policy."
self.log(806, (issuename, jobname))
decision = self.conflict_policy(issue, job)
if decision == 'dt':
# "Defect tracker issue '%s' and Perforce job '%s'
# have both changed since the last time the replicator
# polled the databases. The replicator's conflict
# resolution policy decided to overwrite the job with
# the issue."
reason = [ catalog.msg(841, (issuename, jobname)) ]
self.overwrite_issue_dt_to_p4(issue, job, reason)
elif decision == 'p4':
# "Defect tracker issue '%s' and Perforce job '%s'
# have both changed since the last time the replicator
# polled the databases. The replicator's conflict
# resolution policy decided to overwrite the issue
# with the job."
reason = [ catalog.msg(842, (issuename, jobname)) ]
self.overwrite_issue_dt_to_p4(issue, job, reason)
else:
# "Conflict resolution policy decided: no action."
self.log(807)
# report_failure_to_replicate_p4_to_dt(self, issue, job).
# Construct and send a message to the administrator reporting a
# failure to replicate an issue from Perforce to the defect
# tracker.
def report_failure_to_replicate_p4_to_dt(self, issue, job):
assert isinstance(issue, dt_interface.defect_tracker_issue)
assert isinstance(job, types.DictType)
exc_type, exc_value, exc_traceback = sys.exc_info()
formatted_traceback = string.join(stacktrace.format_exception(exc_type, exc_value, exc_traceback), '')
issuename = issue.readable_name()
jobname = job['Job']
# "Job '%s' could not be replicated to issue '%s'."
subject = catalog.msg(848, (jobname, issuename))
# "Job '%s' could not be replicated to issue '%s': %s: %s"
self.log(808, (jobname, issuename, exc_type, exc_value))
try:
# Get the issue again, since it might have been changed in
# memory in the course of the failed replication. Note
# new variable name so as not to overwrite the old issue.
# (Can we avoid all this nonsense by keeping better track
# of old and new issues?) GDR 2000-10-31.
issue_2 = self.dt.issue(issue.id())
if not issue_2:
# "Issue '%s' not found."
raise self.error, catalog.msg(840, issue.id())
if exc_type == 'TeamShare API error' and not exc_value:
reason = [
# "The replicator failed to replicate Perforce job
# '%s' to defect tracker issue '%s'. There was no
# error message. See the Python traceback below
# for more details about the error."
catalog.msg(849, (jobname, issuename)),
# "The most likely reasons for this problem are:
# you don't have permission to update the issue;
# the job contained data that was invalid in
# TeamTrack; or the job was missing a field that
# is required in TeamTrack."
catalog.msg(850),
]
else:
reason = [
# "The replicator failed to replicate Perforce job
# '%s' to defect tracker issue '%s', because of
# the following problem:"
catalog.msg(851, (jobname, issuename)),
]
if isinstance(exc_value, message.message):
reason.append(exc_value)
else:
# "Defect tracker error (%s): %s"
reason.append(catalog.msg(891,
(exc_type, exc_value)))
appendix = [
# "Here's a full Python traceback:"
catalog.msg(852),
formatted_traceback,
# "If you are having continued problems, please
# contact your P4DTI administrator <%s>."
catalog.msg(853, self.config.administrator_address),
]
self.overwrite_issue_dt_to_p4(issue_2, job,
reason, appendix)
except:
# Replicating back to Perforce failed. Report both errors
# to the administrator.
exc_type_2, exc_value_2, exc_traceback_2 = sys.exc_info()
formatted_traceback_2 = string.join(stacktrace.format_exception(exc_type_2, exc_value_2, exc_traceback_2), '')
# "Job '%s' could not be replicated to issue '%s': %s: %s"
self.log(808,
(jobname, issuename, exc_type_2, exc_value_2))
body = [
# "The replicator failed to replicate Perforce job
# '%s' to defect tracker issue '%s' because of this
# problem:"
catalog.msg(854, (jobname, issuename)),
exc_value,
# "Here's a full Python traceback:"
catalog.msg(852),
formatted_traceback,
# "The replicator attempted to restore the job to a
# copy of the issue, but this failed too, because of
# the following problem:"
catalog.msg(855),
exc_value_2,
# "Here's a full Python traceback:"
catalog.msg(852),
formatted_traceback_2,
# "The replicator has now given up."
catalog.msg(856),
]
self.mail([], subject, body)
return 1
# revert_issue_dt_to_p4(self, issue, job). This is called when an error
# has occurred in replicating from Perforce to the defect tracker. The
# most likely reason for this is a privilege failure (the user is not
# allowed to edit that issue in that way) or a failure to put valid values
# in the job fields. In this case, set the job back to a copy of the
# issue.
def revert_issue_dt_to_p4(self, issue, job):
assert isinstance(issue, dt_interface.defect_tracker_issue)
assert isinstance(job, types.DictType)
exc_type, exc_value, exc_traceback = sys.exc_info()
formatted_traceback = string.join(stacktrace.format_exception(exc_type, exc_value, exc_traceback), '')
issuename = issue.readable_name()
jobname = job['Job']
# "Job '%s' could not be replicated to issue '%s'."
subject = catalog.msg(848, (jobname, issuename))
# "Job '%s' could not be replicated to issue '%s': %s: %s"
self.log(808, (jobname, issuename, exc_type, exc_value))
try:
# Get the issue again, since it might have been changed in memory
# in the course of the failed replication. Note new variable name
# so as not to overwrite the old issue. (Can we avoid all this
# nonsense by keeping better track of old and new issues?) GDR
# 2000-10-31.
issue_2 = self.dt.issue(issue.id())
if not issue_2:
# "Issue '%s' not found."
raise self.error, catalog.msg(840, issue.id())
if exc_type == 'TeamShare API error' and not exc_value:
reason = [
# "The replicator failed to replicate Perforce job '%s' to
# defect tracker issue '%s'. There was no error message.
# See the Python traceback below for more details about the
# error."
catalog.msg(849, (jobname, issuename)),
# "The most likely reasons for this problem are: you don't
# have permission to update the issue; the job contained
# data that was invalid in TeamTrack; or the job was
# missing a field that is required in TeamTrack."
catalog.msg(850),
]
else:
reason = [
# "The replicator failed to replicate Perforce job '%s' to
# defect tracker issue '%s', because of the following
# problem:"
catalog.msg(851, (jobname, issuename)),
]
if isinstance(exc_value, message.message):
reason.append(exc_value)
else:
# "Defect tracker error (%s): %s"
msg = catalog.msg(891, (exc_type, exc_value))
reason.append(msg)
appendix = [
# "Here's a full Python traceback:"
catalog.msg(852),
formatted_traceback,
# "If you are having continued problems, please contact your
# P4DTI administrator <%s>."
catalog.msg(853, self.config.administrator_address),
]
self.overwrite_issue_dt_to_p4(issue_2, job, reason, appendix)
except:
# Replicating back to Perforce failed. Report both errors to the
# administrator.
exc_type_2, exc_value_2, exc_traceback_2 = sys.exc_info()
formatted_traceback_2 = string.join(stacktrace.format_exception(exc_type_2, exc_value_2, exc_traceback_2), '')
# "Job '%s' could not be replicated to issue '%s': %s: %s"
self.log(808, (jobname, issuename, exc_type_2, exc_value_2))
body = [
# "The replicator failed to replicate Perforce job '%s' to
# defect tracker issue '%s' because of this problem:"
catalog.msg(854, (jobname, issuename)),
exc_value,
# "Here's a full Python traceback:"
catalog.msg(852),
formatted_traceback,
# "The replicator attempted to restore the job to a copy of the
# issue, but this failed too, because of the following
# problem:"
catalog.msg(855),
exc_value_2,
# "Here's a full Python traceback:"
catalog.msg(852),
formatted_traceback_2,
# "The replicator has now given up."
catalog.msg(856),
]
self.mail([], subject, body)
return 1
# overwrite_issue_p4_to_dt(self, issue, job, reason, appendix=[]).
# As replicate_issue_p4_to_dt, but e-mails an old copy of the issue
# to the owner of the job and the administrator. The reason
# argument is a list of message objects giving the reason for the
# overwriting. The appendix argument (also a list of strings or
# message objects) gives extra information about the reason that
# should come at the end of the message. Return true if the
# replication was successful, otherwise throw an exception.
def overwrite_issue_p4_to_dt(self, issue, job, reason, appendix=[]):
assert isinstance(issue, dt_interface.defect_tracker_issue)
assert isinstance(job, types.DictType)
assert isinstance(reason, types.ListType)
assert isinstance(appendix, types.ListType)
issuename = issue.readable_name()
jobname = job['Job']
# "Overwrite issue '%s' with job '%s'."
self.log(810, (issuename, jobname))
for r in reason:
self.log(r)
# Build e-mail before overwriting so we get the old issue.
# "Issue '%s' overwritten by job '%s'."
subject = catalog.msg(857, (issuename, jobname))
body = reason + [
# "The replicator has therefore overwritten defect tracker issue
# '%s' with Perforce job '%s'."
catalog.msg(858, (issuename, jobname)),
# "The defect tracker issue looked like this before being
# overwritten:"
catalog.msg(859),
issue,
] + appendix
self.replicate_issue_p4_to_dt(issue, job)
self.mail_concerning_job(subject, body, job)
return 1
# overwrite_issue_dt_to_p4(self, issue, job, reason, appendix=[]).
# As replicate_issue_dt_to_p4, but e-mails an old copy of the
# issue to the owner of the job and the administrator. The reason
# argument is a list of strings or message objects given a reason
# for the overwriting. The appendix argument (also a list of
# strings or message objects) gives extra information about the
# reason that should come at the end of the message. Return true
# if the replication was successful, otherwise throw an exception.
def overwrite_issue_dt_to_p4(self, issue, job, reason, appendix=[]):
assert isinstance(issue, dt_interface.defect_tracker_issue)
assert isinstance(job, types.DictType)
assert isinstance(reason, types.ListType)
assert isinstance(appendix, types.ListType)
issuename = issue.readable_name()
jobname = job['Job']
# "Overwrite job '%s' with issue '%s'."
self.log(811, (jobname, issuename))
for r in reason:
self.log(r)
# Build e-mail before overwriting so we get the old job.
# "Job '%s' overwritten by issue '%s'."
subject = catalog.msg(860, (jobname, issuename))
body = reason + [
# "The replicator has therefore overwritten Perforce job '%s' with
# defect tracker issue '%s'. See section 2.2 of the P4DTI User
# Guide for more information."
catalog.msg(861, (jobname, issuename)),
# "The job looked like this before being overwritten:"
catalog.msg(862),
self.job_format(job),
] + appendix
self.replicate_issue_dt_to_p4(issue, job)
self.mail_concerning_job(subject, body, job)
return 1
# replicate_issue_dt_to_p4(issue, old_job). Replicate the given
# issue from the defect tracker to Perforce. Return true if the
# issue was replicated successfully. Otherwise throw an
# exception.
def replicate_issue_dt_to_p4(self, issue, job):
assert isinstance(issue, dt_interface.defect_tracker_issue)
assert isinstance(job, types.DictType)
# Transform the issue into a job. This has to be done first
# because the job might be new, and we won't be able to
# replicate fixes or filespecs until the job's been created
# (p4 fix won't accept non-existent jobnames). I suppose I
# could create a dummy job to act as a placeholder here, but
# that's not easy at all -- you have to know quite a lot about
# the jobspec to be able to create a job.
changes = self.translate_issue_dt_to_p4(issue, job)
if changes:
# "-- Changed fields: %s."
self.log(812, changes)
self.update_job(job, changes)
else:
# "-- No issue fields were replicated."
self.log(813)
# Replicate filespecs.
dt_filespecs = issue.filespecs()
p4_filespecs = self.job_filespecs(job)
if self.filespecs_differences(dt_filespecs, p4_filespecs):
names = map(lambda(f): f.name(), dt_filespecs)
self.update_job(job, { 'P4DTI-filespecs': string.join(names,'\n') })
# "-- Filespecs changed to '%s'."
self.log(814, string.join(names))
# Replicate fixes.
p4_fixes = self.job_fixes(job)
dt_fixes = issue.fixes()
job_status = job[self.config.job_status_field]
for p4_fix, dt_fix in self.fixes_differences(dt_fixes, p4_fixes):
if p4_fix and not dt_fix:
self.p4.run('fix -d -c %s %s'
% (p4_fix['Change'], p4_fix['Job']))
# "-- Deleted fix for change %s."
self.log(815, p4_fix['Change'])
elif not p4_fix and dt_fix:
try:
self.p4.run('fix -s %s -c %d %s'
% (dt_fix.status(), dt_fix.change(),
issue.corresponding_id()))
except p4.error, message:
# We get an error here if the changelist was somehow
# deleted. In this case there's not much we can do
# except log the error. See job000128.
self.log(message)
else:
job_status = dt_fix.status()
# "-- Added fix for change %d with status %s."
self.log(816, (dt_fix.change(), dt_fix.status()))
elif p4_fix['Status'] != dt_fix.status():
self.p4.run('fix -s %s -c %d %s'
% (dt_fix.status(), dt_fix.change(),
issue.corresponding_id()))
job_status = dt_fix.status()
# "-- Fix for change %d updated to status %s."
self.log(817, (dt_fix.change(), dt_fix.status()))
else:
# This should't happen, since fixes_differences returns only a
# list of pairs which differ.
assert 0
# It might be the case that the job status has been changed by
# replicating a fix from the defect tracker. But this changed status
# won't be right. So restore the correct status if necessary.
if job_status != job[self.config.job_status_field]:
self.update_job(job, { 'Status': job[self.config.job_status_field] })
return 1
# replicate_fixes_p4_to_dt(issue, job). Replicate fixes for the
# given job from Perforce to the defect tracker. Ensures that the
# changelists for the fixes are also replicated. Raise an
# exception on failure.
# If you change this function, you may have to change the regression
# test for job000385; see test_p4dti.py.
def replicate_fixes_p4_to_dt(self, issue, job, failed_before = 0):
p4_fixes = self.job_fixes(job)
dt_fixes = issue.fixes()
fix_diffs = self.fixes_differences(dt_fixes, p4_fixes)
for p4_fix, dt_fix in fix_diffs:
if dt_fix and not p4_fix:
dt_fix.delete()
# "-- Deleted fix for change %d."
self.log(818, dt_fix.change())
else: # p4 fix has changed
# "-- Considering Perforce fix %s."
self.log(819, p4_fix)
change, client, date, status, user = self.translate_fix_p4_to_dt(p4_fix)
# make sure changelist is replicated
try:
changelist = self.p4.run('change -o %s' %
change)[0]
except p4.error:
# The changelist might have been renumbered since we
# called job_fixes; see job000385. If it has, then
# try again. But don't get stuck in an infinite
# loop.
if failed_before:
raise
else:
self.replicate_fixes_p4_to_dt(issue, job,
failed_before = 1)
return
self.replicate_changelist_p4_to_dt(changelist)
if not dt_fix: # new fix; add to DT
issue.add_fix(change, client, date, status, user)
# "-- Added fix for change %s with status %s."
self.log(820, (p4_fix['Change'], p4_fix['Status']))
elif dt_fix.status() != p4_fix['Status']:
# status changed
dt_fix.update(change, client, date, status, user)
# "-- Fix for change %s updated to status %s."
self.log(821, (p4_fix['Change'], p4_fix['Status']))
else:
# This should't happen, since fixes_differences
# returns only a list of pairs which differ.
assert 0
# replicate_filespecs_p4_to_dt(issue, job). Replicate fixes for
# the given job from Perforce to the defect tracker. Raise an
# exception on failure.
def replicate_filespecs_p4_to_dt(self, issue, job):
p4_filespecs = self.job_filespecs(job)
dt_filespecs = issue.filespecs()
filespec_diffs = self.filespecs_differences(dt_filespecs, p4_filespecs)
for p4_filespec, dt_filespec in filespec_diffs:
if dt_filespec and not p4_filespec:
dt_filespec.delete()
# "-- Deleted filespec %s."
self.log(822, dt_filespec.name())
elif not dt_filespec:
issue.add_filespec(p4_filespec)
# "-- Added filespec %s."
self.log(823, p4_filespec)
else:
# This should't happen, since filespecs_differences returns
# only a list of pairs which differ.
assert 0
# replicate_issue_p4_to_dt(issue, job). Replicate the given job from
# Perforce to the defect tracker. Return true if the job was replicated
# successfully. Otherwise, throw an exception.
def replicate_issue_p4_to_dt(self, issue, job):
assert isinstance(issue, dt_interface.defect_tracker_issue)
assert isinstance(job, types.DictType)
# Replicate fixes.
self.replicate_fixes_p4_to_dt(issue, job)
# Replicate filespecs.
self.replicate_filespecs_p4_to_dt(issue, job)
# Transform the job into an issue and update the issue.
changes = self.translate_issue_p4_to_dt(issue, job)
if changes:
# "-- Changed fields: %s."
self.log(824, repr(changes))
p4_user = self.job_modifier(job)
dt_user = self.config.user_translator.translate_1_to_0(p4_user, self.dt, self.dt_p4)
issue.update(dt_user, changes)
else:
# "-- No job fields were replicated."
self.log(825)
# The issue may have changed as a consequence of updating it. For
# example, in TeamTrack the issue's owner changes when an issue goes
# through a transition. So we fetch the issue again, check for changes
# and replicate them back to the job if we find them. See job000053.
new_issue = self.dt.issue(issue.id())
if not new_issue:
# "Issue '%s' not found."
raise self.error, catalog.msg(840, issue.id())
new_changes = self.translate_issue_dt_to_p4(new_issue, job)
if new_changes:
# "-- Defect tracker made changes as a result of the update: %s."
self.log(826, new_changes)
self.update_job(job, new_changes)
return 1
def replicate_changelists(self):
# Replicate all the changelists.
# "Checking changelists to see if they need replicating..."
self.log(827)
changelists = self.p4.run('changes')
# "-- %d changelists to check."
self.log(828, len(changelists))
for c in changelists:
c2 = self.p4.run('change -o %s' % c['change'])[0]
self.replicate_changelist_p4_to_dt(c2)
def start_logger(self):
# Has the logger been started? (We must be careful not to set the
# logger counter to 0 more than once; this will confuse Perforce
# according to Chris Seiwald's e-mail .
logger_started = 0
logger_re = re.compile('logger = ([0-9]+)$')
counters = self.p4.run('counters')
for c in counters:
if (c.has_key('counter') and c['counter'] == 'logger'):
logger_started = 1
# If not, start it.
if not logger_started:
self.p4.run('counter logger 0')
# run() repeatedly polls the DTS.
def run(self):
self.check_jobspec()
self.start_logger()
poll_period = self.config.poll_period
self.mail_startup_message()
while 1:
try:
self.poll()
# Reset poll period when the poll was successful.
poll_period = self.config.poll_period
except AssertionError:
# Assertions indicate severe bugs in the replicator. It might
# cause serious data corruption if we continue. We also want
# these failures to be reported, and they might go unreported
# if the replicator carried on going.
raise
except KeyboardInterrupt:
# Allow people to stop the replicator with Control-C.
raise
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
formatted_traceback = string.join(stacktrace.format_exception(exc_type, exc_value, exc_traceback), '')
# "The replicator failed to poll successfully."
subject = catalog.msg(863)
# "The replicator failed to poll successfully: %s: %s"
self.log(829, (exc_type, exc_value))
body = [
# "The replicator failed to poll successfully, because of
# the following problem:"
catalog.msg(864),
exc_value,
# "Here's a full Python traceback:"
catalog.msg(852),
formatted_traceback,
]
self.mail([], subject, body)
# Clear some large strings so they don't show up in future
# tracebacks and bloat e-mails. See job000215.
formatted_traceback = ''
body = ''
# The poll failed; it's likely that it will fail again for the
# same reason the next time we poll. Back off exponentially so
# as not to mail bomb the admin. See job000215 and job000135.
poll_period = poll_period * 2
time.sleep(poll_period)
# migrate_users() ensures that there is a defect tracker user
# corresponding to each Perforce user.
def migrate_users(self):
p4_users = self.p4.run("users")
for user in p4_users:
self.dt.add_user(user['User'],
user['Email'],
user['FullName'])
# migrate() migrates all existing Perforce jobs to the DT.
def migrate(self):
if self.p4.jobspec_has_p4dti_fields(self.p4.get_jobspec()):
# "It looks as if migration has already been run (the P4
# jobspec has P4DTI fields). Please revert the Perforce
# and defect tracker databases before attempting to run
# migration again. A future P4DTI release will have a
# migration script which will handle this better."
raise self.error, catalog.msg(903)
try:
pairs=[]
jobs = self.all_jobs()
self.dt.new_issues_start()
for job in jobs:
if self.config.migrate_p(job):
issue = self.create_issue(job)
# "Migrated job '%s' to issue '%s'."
self.log(892, (job['Job'], issue.readable_name()))
# Migrate information about this issue
# only if we're going to replicate it.
if issue.replicate_p():
# Save for post-migration replication. Note
# that we save the issue ID, not the issue,
# for scalability reasons. (for some defect
# trackers, each issue object may be very
# large).
pairs.append((job, issue.id()))
# replicate fixes and changelists
self.replicate_fixes_p4_to_dt(issue, job)
# replicate filespecs
self.replicate_filespecs_p4_to_dt(issue, job)
self.dt.new_issues_end()
# consistency checks which could go in here: have all the
# jobs been migrated? (i.e. are they in the defect tracker
# with the correct P4DTI meta-data)
if self.config.migrated_jobspec_description:
self.p4.install_jobspec(self.config.migrated_jobspec_description)
# "Installed post-migration jobspec."
self.log(893)
for pair in pairs:
job, issue_id = pair
issue = self.dt.issue(issue_id)
job['P4DTI-rid'] = self.rid
job['P4DTI-issue-id'] = issue_id
# "Post-migration replication of issue '%s' to job
# '%s'."
self.log(894, (issue.readable_name(), job['Job']))
self.replicate_issue_dt_to_p4(issue, job)
# "Migration completed."
self.log(895)
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
formatted_traceback = string.join(stacktrace.format_exception(exc_type, exc_value, exc_traceback), '')
# "Migration failed."
self.log(901)
# Note that the traceback includes the exception value.
# "Here's a full Python traceback: \n%s"
self.log(902, formatted_traceback)
# translate_fix_p4_to_dt(p4_fix).
def translate_fix_p4_to_dt(self, p4_fix):
assert isinstance(p4_fix, types.DictType)
change = int(p4_fix['Change'])
client = p4_fix['Client']
date = self.config.date_translator.translate_1_to_0(p4_fix['Date'], self.dt, self.dt_p4)
status = p4_fix['Status']
user = self.config.user_translator.translate_1_to_0(p4_fix['User'], self.dt, self.dt_p4)
return (change, client, date, status, user)
# translate_issue_dt_to_p4(issue, job). Return changes as a dictionary but
# don't apply them yet.
def translate_issue_dt_to_p4(self, issue, job):
assert isinstance(issue, dt_interface.defect_tracker_issue)
assert isinstance(job, types.DictType)
changes = { }
# Do the P4DTI fields need to be changed? If so, record in changes.
for key, value in [('P4DTI-rid', self.rid),
('P4DTI-issue-id', issue.id())]:
if job[key] != value:
changes[key] = value
# What about the replicated fields?
for dt_field, p4_field, trans in self.config.field_map:
p4_value = trans.translate_0_to_1(issue[dt_field], self.dt, self.dt_p4, issue, job)
# A field not appearing in the job is equivalent to the field
# having a value of the empty string. See job000181.
if ((job.has_key(p4_field) and p4_value != job[p4_field])
or (not job.has_key(p4_field) and p4_value != '')):
changes[p4_field] = p4_value
return changes
# translate_issue_p4_to_dt(issue, job). Return changes as a dictionary but
# don't apply them yet.
def translate_issue_p4_to_dt(self, issue, job):
assert isinstance(issue, dt_interface.defect_tracker_issue)
assert isinstance(job, types.DictType)
changes = {}
for dt_field, p4_field, trans in self.config.field_map:
if job.has_key(p4_field):
p4_value = job[p4_field]
else:
# Missing fields indicate optional fields without a value --
# this happens when the empty string has been supplied for the
# value. So supply the empty string ourselves. See job000181.
p4_value = ''
dt_value = trans.translate_1_to_0(p4_value, self.dt, self.dt_p4, issue, job)
if dt_value != issue[dt_field]:
changes[dt_field] = dt_value
return changes
# create_issue(job). Makes a new issue corresponding to the
# job. Returns the new issue.
def create_issue(self, job):
assert isinstance(job, types.DictType)
dict = {}
self.config.pre_migrate_issue(self.config,
self.dt, self.dt_p4,
job)
for dt_field, p4_field, trans in self.config.field_map:
if job.has_key(p4_field):
p4_value = job[p4_field]
else:
# Missing fields indicate optional fields without a
# value -- this happens when the empty string has been
# supplied for the value. So supply the empty string
# ourselves. See job000181. When migrating, this
# will also happen for fields which we are about to
# add to the jobspec.
p4_value = ''
dt_value = trans.translate_1_to_0(p4_value,
self.dt, self.dt_p4,
None, job)
dict[dt_field] = dt_value
self.config.migrate_issue(self.config,
self.dt, self.dt_p4,
dict, job)
return self.dt.new_issue(dict, job['Job'])
# update_job(job, changes).
update_job_re = re.compile('^Job ([^ ]+)')
def update_job(self, job, changes = {}):
assert isinstance(job, types.DictType)
assert isinstance(changes, types.DictType)
for key, value in changes.items():
job[key] = value
results = self.p4.run('job -i', job)
# Check that the results of the 'job -i' command are as
# expected: Perforce should say something like 'Job job012345
# saved.' or 'Job job012345 not changed.' If the jobname was
# 'new', then record the jobname that Perforce gave the new
# job so that we can call setup_for_replication() in
# replicate_many().
if (len(results) == 1 and results[0].has_key('data')):
match = self.update_job_re.match(results[0]['data'])
if not match or match.group(1) == 'new':
# "Expected Perforce output of 'job -i' to say 'Job
# jobname ...', but found '%s'."
raise self.error, catalog.msg(897, results[0]['data'])
elif job['Job'] == 'new':
job['Job'] = match.group(1)
elif job['Job'] != match.group(1):
# "Tried to update job '%s', but Perforce replied '%s'."
raise self.error, catalog.msg(899, (job['Job'], results[0]['data']))
else:
# "Unexpected output from Perforce command 'job -i': %s."
raise self.error, catalog.msg(898, results)
# Return the e-mail address of a Perforce user, or None if the address
# can't be found.
def user_email_address(self, user):
assert isinstance(user, types.StringType)
# Even though "p4 user -o foo" doesn't actually create a user, it does
# fail if the user foo doesn't exist and there are no spare licences.
# So trap that case. See job000204.
try:
u = self.p4.run('user -o %s' % user)
except:
return None
# Existing users have Access and Update fields in the returned
# structure; non-existing users don't.
if (len(u) == 1 and u[0].has_key('Access') and u[0].has_key('Update')
and u[0].has_key('Email')):
return u[0]['Email']
else:
return None
# B. Document History
#
# 2000-12-05 NB addess -> address
#
# 2000-12-05 GDR Imported p4 module so replicator can catch p4.error. Added
# replicator method mail_concerning_job() for e-mailing people about a job.
# There were several places where the owner of a job was been fetched and
# e-mailed, some of which were buggy. This method replaces all those
# instances, hopefully correctly.
#
# 2000-12-06 GDR Fixed the replicator's user_email_address method so that it
# really returns None when there is no such user.
#
# 2000-12-06 GDR Updated supported Perforce changelevel to 18974 (this is the
# changelevel we document against).
#
# 2000-12-06 GDR Fixing job000133 (replicator gets wrong user when a job is
# fixed): When the last person who changed the job is the replicator user,
# update the issue on behalf of the job owner instead.
#
# 2000-12-06 GDR If the owner of a job and the person who last changed it are
# the same (a common occurrence), include them only once in any e-mail sent by
# the replicator about that job.
#
# 2000-12-06 GDR E-mail messages from the replicator concerning overwritten
# jobs are much improved.
#
# 2000-12-06 GDR The overwriting methods now send e-mail with the new
# issue/job in them, not the old issue/job.
#
# 2000-12-07 GDR When there's no error message (typically in the case of
# assertion failure), say so. Format the job properly in all messages
# (including the one sent by the conflict method). Use "Perforce job" and
# "defect tracker issue" for clarity. (Even better would be to have a
# defect_tracker.name so it could say "TeamTrack issue".)
#
# 2000-12-07 GDR Created new class dt_perforce; a placeholder for an eventual
# full implementation of a defect_tracker subclass that interfaces to Perforce.
#
# 2001-01-19 NB Better stack traces.
#
# 2001-01-19 GDR Handle empty fields. Fix comments.
#
# 2001-01-23 NB SMTP server test (unmatched users).
#
# 2001-02-04 GDR Updated definition of defect_tracker.all_issues() method.
#
# 2001-02-12 GDR Fixed bug in check_consisteney.
#
# 2001-02-13 GDR Don't send e-mail if administrator_address or smtp_server is
# None.
#
# 2001-02-14 GDR user_email_address returns None if the user doesn't exist,
# even when there are no spare licences (see job000204).
#
# 2001-02-16 NB Added replicate-p configuration parameter.
#
# 2001-02-21 GDR The replicator backs off exponentially if it fails to poll
# successfully, so as not to mailbomb the administrator.
#
# 2001-02-22 GDR replicate_changelist_p4_to_dt applies a text translator to the
# change description, since a change description can have several lines of
# text.
#
# 2001-02-23 GDR Added a corresponding_id method to the defect_tracker_issue
# class.
#
# 2001-03-02 RB Transferred copyright to Perforce under their license.
#
# 2001-03-11 GDR Use messages for errors, logging and e-mail.
#
# 2001-03-13 GDR Removed the recording of conflicts. Conflict resolution is
# always immediate. Moved translator class to translator module. Moved defect
# tracker interface classes to dt_interface.py.
#
# 2001-03-14 GDR Use messages when consistency checking.
#
# 2001-03-15 GDR Get configuration from config module.
#
# 2001-03-16 GDR Added refresh_perforce_jobs() method.
#
# 2001-03-21 GDR The setup_for_replication() method takes a jobname argument.
#
# 2001-03-23 GDR New method job_modifier returns a best guess at who last
# modified the job, to fix job000270.
#
# 2001-03-24 GDR Check supported Perforce server changelevel in p4.py, not
# replicator.__init__ (so that we find out if p4 -G jobspec -i will work before
# actually trying it in init.py).
#
# 2001-03-25 RB Moved message 889 to catalog due to merge from version
# 1.0 sources.
#
# 2001-05-17 GDR Defect tracker methods 'add_issue' and
# 'changed_entities' may return cursors as well as lists.
#
# 2001-05-19 GDR Added progress report for consistency checking.
#
# 2001-05-22 GDR Compare job names case-insensitively when fetching a
# job to work around job000313.
#
# 2001-06-14 GDR The reason argument to overwrite_issue_dt_to_p4 must
# consist only of messages, since they are logged as well as mailed.
# Each call to the defect_tracker.issue() method now has error checking.
#
# 2001-06-15 NB Moved functionality out of init because it's not all
# required by all the scripts. Creation of client and calling
# defect_tracker.init moved to __init__ method. Checking of jobspec
# moved to new method check_jobspec and called from check, refresh and
# run.
#
# 2001-06-25 NB post-migration replication now works!
#
# 2001-06-25 NB Now support the 'use_perforce_jobnames' configuration
# parameter by specifying 'new' for the jobname in replicate_many()
# and then recording the jobname when we find out what it is in
# update_job().
#
# 2001-06-26 NB Now support the creation of new jobs in Perforce.
# Also moved the replication of changelists, and changed the interface
# to changed_entities (so that changelists are replicated iff the
# matching fixes are replicated).
#
# 2001-06-27 NB Moved code from new_issue out to the defect tracker
# (changed new_issue interface).
#
# 2001-06-29 NB Produce full traceback if migration fails.
#
# 2001-06-30 GDR The replicator doesn't stop if it can't replicate a fix
# because the changelist has been deleted (see job000128).
#
# 2001-07-04 NB Changed issue creation system so we use the regular
# field map.
#
# 2001-08-06 GDR Specify -1 for DST argument to mktime().
#
# 2001-10-03 GDR Handle renumbered changelist race condition during
# replication of fixes from Perforce to the defect tracker; see
# job000385.