# Perforce Defect Tracking Integration Project
#
#
# DT_TEAMTRACK.PY -- DEFECT TRACKING INTERFACE (TEAMTRACK)
#
# Gareth Rees, Ravenbrook Limited, 2000-09-07
#
#
# 1. INTRODUCTION
#
# This Python module implements an interface between the P4DTI
# replicator and the TeamTrack 4 defect tracker [Requirements, 6], by
# defining the classes listed in [GDR 2000-10-16, 7]. In particular, it
# defines the following classes:
#
# [3] teamtrack_case(dt_interface.defect_tracker_issue) [GDR 2000-10-16,
# 7.2]
#
# [4] teamtrack_fix(dt_interface.defect_tracker_fix) [GDR 2000-10-16,
# 7.3]
#
# [5] teamtrack_filespec(dt_interface.defect_tracker_filespec) [GDR
# 2000-10-16, 7.4].
#
# [6] dt_teamtrack(dt_interface.defect_tracker) [GDR 2000-10-16, 7.1]
#
# [7] Translators [GDR 2000-10-16, 7.5] for dates [GDR 2000-10-16,
# 7.5.1], elapsed times, foreign keys, single select fields, states [GDR
# 2000-10-16, 7.5.2], multi-line text fields [GDR 2000-10-16, 7.5.3] and
# users [GDR 2000-10-16, 7.5.4].
#
# This module accesses the TeamTrack database using the Python interface
# to TeamTrack [GDR 2000-08-08] and it accesses and stores data
# according to the TeamTrack database schema [TeamShare 2000-01-20] and
# the TeamTrack schema extensions [GDR 2000-09-04].
#
# The intended readership of this document is project developers.
#
# This document is not confidential.
import catalog
import dt_interface
import message
import replicator
import re
import string
import teamtrack
import time
import translator
import types
# 2. DATA AND UTILITIES
# 2.1. Error object
#
# All exceptions raised by this module use 'error' as the exception
# object.
error = 'P4DTI TeamTrack interface error'
# 2.2. Field types in the VCACTIONS table
#
# The TeamTrack integration stores all its relations in the TS_VCACTIONS
# table, distinguishing them by the value in the TS_TYPE field [GDR
# 2000-09-04, 2.1]. these variables are the types for filespecs, fixes,
# changelist descriptions, and replicator configuration records
# respectively.
vcactions_type_filespec = 1
vcactions_type_fix = 2
vcactions_type_changelist = 3
vcactions_type_config = 4
# 2.3. Escape quotes in a string for SQL
#
# sql_escape(string). Return the input string, escaped for inclusion in
# a SQL query by doubling single quotes. This works for Microsoft
# Access, but I'm not sure that it's correct ANSI SQL. See job000031.
def sql_escape(s):
return string.replace(s, "'", "''")
# 2.4. String representation (workaround for job000223)
#
# string_repr_workaround(string) returns a representation of its
# argument, just like the build in Python function repr(), but it always
# surround the argument with single quotes, and precedes both single
# quotes and double quotes with backslashes (despite the fact that in
# Python double quotes need not be backslashed in a string surrounded by
# single quotes).
#
# The reason for using this function rather than repr() is because of a
# bug in TeamTrack 4.5 (see job000223). The TeamTrack schema extensions
# store multiple values in a single field by storing the Python
# representation of a dictionary in the field; see [GDR 2000-09-04,
# 2.2]. TeamTrack 4.5 doesn't implement Python's string syntax
# correctly: it assumes that strings are surrounded with single quotes,
# and it assumes that an unescaped double quote terminates a string.
#
# So this function, and dict_repr_workaround(), work around that bug.
# If TeamShare later fixes the bug then this workaround can be dropped,
# and repr() used instead.
#
# The code is based directly on Python's string_repr() function.
def string_repr_workaround(s):
assert isinstance(s, types.StringType)
result = "'"
for c in s:
if c in ['"', "'", '\\']:
result = result + '\\' + c
elif c < ' ' or c >= '\177':
result = result + ('\\%03o' % (ord(c) & 0377))
else:
result = result + c
return result + "'"
# 2.5. Dictionary representation (workaround for job000223)
#
# dict_repr_workaround(dict) returns a string representation of its
# argument for storage in a field in the TeamTrack database [GDR
# 2000-09-04, 2.2]. Its result is just like that of the built in Python
# function repr(), but it uses string_repr_workaround() for string keys
# and values [2.4]. It also checks that all keys are strings and values
# are strings or integers, as promised in [GDR 2001-09-04, 2.2].
def dict_repr_workaround(d):
assert isinstance(d, types.DictType)
result = '{'
for k, v in d.items():
assert isinstance(k, types.StringType)
result = result + string_repr_workaround(k) + ': '
if isinstance(v, types.IntType):
result = result + repr(v)
elif isinstance(v, types.StringType):
result = result + string_repr_workaround(v)
else:
assert 0
result = result + ', '
return result[0:-2] + '}'
# 2.6. Cursors
#
# Instances of these cursor classes behave like database cursors: you
# can repeatedly call the fetchone() method to get one record from the
# result set; finally the query returns None.
#
# The TeamShare API provides no real cursor interface so this emulates
# cursors by making repeated queries that each return a single result
# (the next result from the original query). Because these are not real
# cursors and the TeamShare API provides no locking or transactions, the
# results you get may be inconsistent if the database changes in the
# course of getting the results.
#
# These classes helps to work around job000277 and job000278.
#
# See [GDR 2000-10-16, 7.6] for the specification of a cursor.
# 2.6.1. Generic cursor
#
# This cursor can handle any query to any table. See [GDR 2001-05-16]
# for the analysis that led to this implementation.
#
# To create an instance of this cursor, pass the following arguments to
# the constructor:
#
# dt: The defect tracker instance.
# table_name: The table you're querying, for example 'CASES'.
# query: The SQL query you want, for example 'TS_TYPE=5'. If
# query is None, then the cursor will fetch no records.
# wrapper: A function or class with which to wrap the returned
# results, for example teamtrack_case. None means don't
# wrap. Defaults to None.
# wrapper_args: Additional arguments to pass to the wrapper function.
class cursor:
chunk_size = 20
def __init__(self, dt, table_name, query, wrapper = None,
wrapper_args = ()):
assert isinstance(dt, dt_teamtrack)
assert isinstance(table_name, types.StringType)
assert query == None or isinstance(query, types.StringType)
self.cache = []
self.dt = dt
self.finished = (query == None)
self.last_id = -1
self.query = query
self.table_id = teamtrack.table[table_name]
if table_name == 'CASES':
self.table_name = dt.server.case_table_name()
else:
self.table_name = 'TS_' + table_name
self.wrapper = wrapper
self.wrapper_args = wrapper_args
if query == '':
self.where_1 = ''
self.where_2 = ''
else:
self.where_1 = '(%s) AND ' % self.query
self.where_2 = ' AND (%s)' % self.query
if query:
# "TeamTrack query: SELECT * FROM %s WHERE %s."
dt.log(631, (self.table_name, query))
elif query == '':
# "TeamTrack query: SELECT * FROM %s."
dt.log(632, self.table_name)
def fetchone(self):
if self.finished:
return None
if not self.cache:
query = ("%sTS_ID BETWEEN (SELECT MIN(TS_ID) FROM %s "
"WHERE TS_ID > %d%s) AND (SELECT MIN(TS_ID)+%d "
"FROM %s WHERE TS_ID > %d%s) ORDER BY TS_ID"
% (self.where_1, self.table_name, self.last_id,
self.where_2, self.chunk_size - 1,
self.table_name, self.last_id, self.where_2))
self.cache = self.dt.server.query(self.table_id, query)
if self.cache:
self.last_id = max(self.cache[-1]['ID'],
self.last_id + self.chunk_size)
else:
self.finished = 1
return None
result = self.cache[0]
self.cache = self.cache[1:]
if self.wrapper:
return apply(self.wrapper, (result,) + self.wrapper_args)
else:
return result
# 2.6.2. Cases cursor
#
# This is a cursor that returns cases from a list of cases.
#
# To create an instance of this cursor, pass the following argumntes to
# the constructor:
#
# dt: The defect tracker instance.
# cases: A list of case ids to query.
# query: Query that the returned cases must match.
class cases_cursor:
chunk_size = 20
def __init__(self, dt, cases, query):
assert isinstance(dt, dt_teamtrack)
assert isinstance(cases, types.ListType)
self.cache = []
self.cases = cases
self.dt = dt
self.finished = 0
self.query = query
self.table_id = dt.server.case_table_id()
if query:
self.where = ' AND (%s)' % query
else:
self.where = ''
if cases:
q = ('TS_ID IN (%s)%s' % (str(self.cases)[1:-1],
self.where))
# "TeamTrack query: SELECT * FROM %s WHERE %s."
self.dt.log(631, ('TS_CASES', q))
def fetchone(self):
if not self.cache:
if self.cases:
n = min(self.chunk_size, len(self.cases))
cases = self.cases[0:n]
self.cases = self.cases[n:]
query = ("TS_ID IN (%s)%s"
% (str(cases)[1:-1], self.where))
self.cache = self.dt.server.query(self.table_id, query)
if self.cache:
result = self.cache[0]
self.cache = self.cache[1:]
return teamtrack_case(result, self.dt)
else:
return None
# 3. TEAMTRACK CASE INTERFACE
#
# This class implements the replicator's interface to the cases in
# TeamTrack [GDR 2000-10-16, 7.2].
class teamtrack_case(dt_interface.defect_tracker_issue):
dt = None # The defect tracker this case belongs to.
case = None # The teamtrack_record object representing the case.
def __init__(self, case, dt):
self.case = case
self.dt = dt
def __getitem__(self, key):
assert isinstance(key, types.StringType)
return self.case[key]
def __repr__(self):
return repr(self.case)
def __setitem__(self, key, value):
assert isinstance(key, types.StringType)
self.case[key] = value
def add_filespec(self, filespec):
filespec_record = self.dt.server.new_record(
teamtrack.table['VCACTIONS'])
filespec_record['TYPE'] = vcactions_type_filespec
filespec_record['CHAR2'] = self.dt.sid
filespec_record['CHAR1'] = self.rid()
filespec_record['TABLEID'] = self.dt.server.case_table_id()
filespec_record['RECID'] = self.case['ID']
filespec_record['TIME1'] = 0
filespec_record['TIME2'] = 0
filespec_record['FILENAME'] = dict_repr_workaround({
'filespec': filespec,
})
filespec_record.add()
def add_fix(self, change, client, date, status, user):
fix_record = self.dt.server.new_record(
teamtrack.table['VCACTIONS'])
fix_record['TYPE'] = vcactions_type_fix
fix_record['CHAR2'] = self.dt.sid
fix_record['CHAR1'] = self.rid()
fix_record['TABLEID'] = self.dt.server.case_table_id()
fix_record['RECID'] = self.case['ID']
fix_record['TIME2'] = 0
fix = teamtrack_fix(fix_record, self)
fix.transform_from_p4(change, client, date, status, user)
fix.add()
def corresponding_id(self):
if self['P4DTI_JOBNAME']:
return self['P4DTI_JOBNAME']
else:
return self.readable_name()
def delete(self):
self.dt.server.delete_record(self.case.table(), self['ID'])
def filespecs(self):
query = ("TS_TYPE=%d AND TS_RECID=%s"
% (vcactions_type_filespec, self.id()))
return map(lambda f, s=self: teamtrack_filespec(f, s),
self.dt.query('VCACTIONS', query))
# find_transition(old_state, new_state). Given an issue and the old
# and new states, find a transition that corresponds to this state
# change, or return None if there is no such transition.
def find_transition(self, old_state, new_state):
assert isinstance(old_state, types.IntType)
assert isinstance(new_state, types.IntType)
project = self['PROJECTID']
psst = self.dt.project_to_states_to_transition
ss = (old_state, new_state)
if not (psst.has_key(project) and psst[project].has_key(ss)):
# The transitions may have changed since we last looked in
# the database, so refresh our cache.
self.dt.read_transitions()
psst = self.dt.project_to_states_to_transition
if psst.has_key(project) and psst[project].has_key(ss):
return psst[project][ss]
else:
# No appropriate transition found.
return None
def fixes(self):
query = ("TS_TYPE=%d AND TS_RECID=%s"
% (vcactions_type_fix, self.id()))
return map(lambda f, s=self: teamtrack_fix(f, s),
self.dt.query('VCACTIONS', query))
def id(self):
return str(self['ID'])
def readable_name(self):
if self.dt.type_id_to_prefix.has_key(self['ISSUETYPE']):
return ('%s%s'
% (self.dt.type_id_to_prefix[self['ISSUETYPE']],
self['ISSUEID']))
else:
return '%s' % (self['ISSUEID'],)
def rid(self):
return self['P4DTI_RID']
def setup_for_replication(self, jobname):
self['P4DTI_RID'] = self.dt.rid
self['P4DTI_SID'] = self.dt.sid
self['P4DTI_JOBNAME'] = jobname
self.case.update()
def update(self, user, changes = {}):
assert isinstance(user, types.IntType)
assert isinstance(changes, types.DictType)
# Work out a transition based on the old case state and the new
# case state, if the state changed. Otherwise, update the case
# by using transition 0 (this is a secret feature of the
# transition function - see John McGinley's e-mail).
transition = 0
if (changes.has_key('STATE')
and changes['STATE'] != self['STATE']):
transition = self.find_transition(self['STATE'],
changes['STATE'])
if not transition:
old = self.dt.state_id_to_name[self['STATE']]
new = self.dt.state_id_to_name[changes['STATE']]
# "No transition from state '%s' to state '%s'."
raise error, catalog.msg(614, (old, new))
for key, value in changes.items():
self[key] = value
user = self.dt.user_id_to_name[user]
if transition:
# "-- Transition: %d; User: %s."
self.dt.log(600, (transition, user))
self.case.transition(user, transition)
# 4. TEAMTRACK FIX INTERFACE
#
# This class implements the replicator's interface to a fix record in
# TeamTrack [GDR 2000-10-16, 7.3].
class teamtrack_fix(dt_interface.defect_tracker_fix):
# The TeamTrack case to which the fix refers.
case = None
# The teamtrack_record object representing the fix record.
fix = None
# The data that go in the TS_FILENAME field.
data = { 'status': '', 'client': '' }
def __init__(self, fix, case):
assert isinstance(case, teamtrack_case)
self.fix = fix
self.case = case
if fix['FILENAME']:
self.data = eval(fix['FILENAME'])
def __getitem__(self, key):
return self.fix[key]
def __repr__(self):
return repr(self.fix)
def __setitem__(self, key, value):
assert isinstance(key, types.StringType)
self.fix[key] = value
if key == 'FILENAME':
self.data = eval(value)
def add(self):
self.fix.add()
def change(self):
return self['INFO1']
def delete(self):
self.case.dt.server.delete_record(teamtrack.table['VCACTIONS'],
self.fix['ID'])
def status(self):
return self.data['status']
def transform_from_p4(self, change, client, date, status, user):
assert isinstance(change, types.IntType)
assert isinstance(client, types.StringType)
assert isinstance(date, types.IntType)
assert isinstance(status, types.StringType)
assert isinstance(user, types.IntType)
self['INFO1'] = change
self['TIME1'] = date
self['AUTHOR1'] = user
self['FILENAME'] = dict_repr_workaround({ 'status': status,
'client': client })
def update(self, change, client, date, status, user):
self.transform_from_p4(change, client, date, status, user)
self.fix.update()
# 5. TEAMTRACK FILESPEC INTERFACE
#
# This class implements the replicator's interface to a filespec record
# in TeamTrack [GDR 2000-10-16, 7.4].
class teamtrack_filespec(dt_interface.defect_tracker_filespec):
# The TeamTrack case to which the filespec refers.
case = None
# The teamtrack_record object representing the filespec.
filespec = None
# The data that go in the TS_FILENAME field.
data = { 'filespec': '' }
def __init__(self, filespec, case):
self.filespec = filespec
self.case = case
if self['FILENAME']:
self.data = eval(self['FILENAME'])
def __getitem__(self, key):
assert isinstance(key, types.StringType)
return self.filespec[key]
def __setitem__(self, key, value):
assert isinstance(key, types.StringType)
self.filespec[key] = value
if key == 'FILENAME':
self.data = eval(value)
def delete(self):
self.case.dt.server.delete_record(teamtrack.table['VCACTIONS'],
self.filespec['ID'])
def name(self):
return self.data['filespec']
# 6. TEAMTRACK INTERFACE
#
# This class implements the replicator's interface to TeamTrack [GDR
# 2000-10-16, 7.1].
class dt_teamtrack(dt_interface.defect_tracker):
# 6.1. Instance variables
# TeamTrack server connection.
server = None
# Userid on the TeamTrack server.
userid = None
# Replicator identifier.
rid = None
# Perforce server identifier.
sid = None
# A map from auxiliary table ID to the ID of the entities in that
# table to the name for that entity. For example, if entity 7 in
# the PROJECTS table (table 8) has the name "Image Builder", then
# table_to_id_to_name[8][7] == "Image Builder".
table_to_id_to_name = None
# A map from auxiliary table ID to the ID of the entities in that
# table to the name for that entity. For example, if entity 7 in
# the PROJECTS table (table 8) has the name "Image Builder", then
# table_to_name_to_id[8]["Image Builder"] == 7.
table_to_name_to_id = None
# A map from field id to the record from the FIELDS table.
field = None
# A map from field name to selection name to the id for that
# selection. For example, if selection called "Foo" in field
# "PROJECTS" has id 7, then
# field_to_selection_to_id['PROJECTS']['Foo'] == 7.
field_to_selection_to_id = None
# A map from project id to the record from the PROJECTS table.
project = None
# A map from TeamTrack project to a map from lowercased state name
# to state id. So for example, if project 7 has a state called
# "Open" with id 25, then project_to_name_to_state[7]['open'] == 25.
# The reason why the state names are lowercase here is because they
# are lowercase in Perforce for usability, to avoid having two
# states which differ only in case.
project_to_name_to_state = None
# A map from project and a pair of states to a transition that is
# available to cases in that project and which transitions between
# the states. For example, if transition 27 is available in project
# 5 and transitions between state 6 and state 7, then
# project_to_states_to_transition[5][(6,7)] == 27.
project_to_states_to_transition = None
# A map from selection id to name.
selection_id_to_name = None
# A map from TeamTrack state id to state name.
state_id_to_name = None
# A map from TeamTrack state name to state id.
state_name_to_id = None
# A map from the type of an issue to the prefix for that type (e.g.,
# "BUG", "ENH").
type_id_to_prefix = None
# A map from TeamTrack user id (the ID field in the USERS table) to
# their user name.
user_id_to_name = None
# A map from TeamTrack user name to their user id.
user_name_to_id = None
# A map from TeamTrack user id to their e-mail address.
user_id_to_email = None
# A map from TeamTrack user's e-mail address to their user id.
user_email_to_id = None
# These members are flags that indicate whether data has been read
# from TeamTrack and cached. The flags are cleared at the start of
# each poll (in changed_entities). The idea is to ensure that the
# caches are filled no more than once per poll (see job000148).
cached_auxiliary_table = None
cached_fields = 0
cached_selections = 0
cached_states = 0
cached_transitions = 0
cached_types = 0
cached_users = 0
# 6.2. Initialization
def __init__(self, config):
# Initialize the caches (see the rule code/python/instance).
self.cached_auxiliary_table = {}
self.field = {}
self.field_to_selection_to_id = {}
self.project = {}
self.project_to_name_to_state = {}
self.project_to_states_to_transition = {}
self.selection_id_to_name = {}
self.state_id_to_name = {}
self.state_name_to_id = {}
self.table_to_id_to_name = {}
self.table_to_name_to_id = {}
self.type_id_to_prefix = {}
self.user_email_to_id = {}
self.user_id_to_email = {}
self.user_id_to_name = {}
self.user_name_to_id = {}
# The 'migrate_issues' and 'new_issues' features are supported
# only if the Python interface to TeamTrack supports the
# 'submit' feature. See [GDR 2000-08-08].
submit_supported = teamtrack.feature.get('submit', 0)
self.feature['migrate_issues'] = submit_supported
self.feature['new_issues'] = submit_supported
# Store configuration, connect and initialize.
self.config = config
self.rid = config.rid
self.sid = config.sid
if not self.config.teamtrack_user:
self.config.teamtrack_user = 'P4DTI-%s' % self.rid
self.server = teamtrack.connect(self.config.teamtrack_user,
self.config.teamtrack_password,
self.config.teamtrack_server)
self.read_types()
self.read_users()
# Get the userid corresponding to the replicator's userid. This
# will be used in queries to ignore records changed most
# recently by the replicator.
user = self.query('USERS', "TS_LOGINID = '%s'"
% sql_escape(self.config.teamtrack_user))
if len(user) != 1:
# "No login id in TeamTrack's USERS table corresponding to
# replicator's login id '%s'."
raise error, catalog.msg(615, self.config.teamtrack_user)
self.userid = user[0]['ID']
def query(self, table_name, query):
if query:
# "TeamTrack query: SELECT * FROM %s WHERE %s."
self.log(631, (table_name, query))
elif query == '':
# "TeamTrack query: SELECT * FROM %s."
self.log(632, table_name)
return self.server.query(teamtrack.table[table_name], query)
def log(self, msg, args = ()):
if not isinstance(msg, message.message):
msg = catalog.msg(msg, args)
self.config.logger.log(msg)
# 6.3. Defect tracker interface
#
# See [GDR 2000-10-16, 7.1].
def all_issues(self):
query = ("TS_P4DTI_RID='%s' OR ((TS_P4DTI_RID='' OR "
"TS_P4DTI_RID IS NULL) AND TS_LASTMODIFIEDDATE>=%d)"
% (self.rid, self.config.start_date))
return cursor(self, 'CASES', query, teamtrack_case, (self,))
def changed_entities(self):
# Reset the cache flags at the start of this poll, so that
# caches may be re-read. The idea is to make sure that tables
# only get read once per poll even if there are many misses.
# See job000148.
self.cached_auxiliary_table = {}
self.cached_fields = 0
self.cached_selections = 0
self.cached_states = 0
self.cached_transitions = 0
self.cached_types = 0
self.cached_users = 0
# Get the last change record that was dealt with.
query = ("TS_TYPE=%d AND TS_CHAR1='%s' "
"AND TS_CHAR2='LAST_CHANGE'"
% (vcactions_type_config, self.rid))
last_change = self.query('VCACTIONS', query)
if not last_change:
# "No LAST_CHANGE record for this replicator."
raise error, catalog.msg(616)
# Get the list of changes to cases that haven't been dealt with
# yet. Ignore changes made by the replicator. See job000033
# and job000233.
last_change_id = last_change[0]['INFO1']
query = ("TS_TABLEID = %d AND TS_ID > %d "
"AND TS_REALUSERID <> %d AND TS_USERID <> %d"
% (self.server.case_table_id(), last_change_id,
self.userid, self.userid))
changes_cursor = cursor(self, 'CHANGES', query)
# Work out the set of changed cases (since a changed case may
# appear several times in the CHANGES table but we don't want to
# replicate it more than once) and the last change id.
case_id_found = {}
while 1:
c = changes_cursor.fetchone()
if c == None:
break
case_id_found[c['CASEID']] = 1
if c['ID'] > last_change_id:
last_change_id = c['ID']
# Get a cursor that will fetch the changed cases. The IS NULL
# condition is there because the TeamShare API doesn't reliably
# set a NULL field to the empty string when you assign the empty
# string to the field and update the record. See e-mail to
# Larry Fish, 2000-09-19.
query = ("TS_P4DTI_RID='%s' OR TS_P4DTI_RID='' OR "
"TS_P4DTI_RID IS NULL" % (self.rid))
case_ids = case_id_found.keys()
case_ids.sort()
changed_cases_cursor = cases_cursor(self, case_ids, query)
# Make a marker that can be passed to mark_changes_done to
# record that these changes have been replicated.
if last_change[0]['INFO1'] != last_change_id:
last_change[0]['INFO1'] = last_change_id
marker = last_change[0]
else:
marker = None
# Note that there are no changed changelists.
return changed_cases_cursor, [], marker
def mark_changes_done(self, last_change):
if last_change:
last_change.update()
def init(self):
# Check that the TeamTrack database version is supported.
supported_dbver = 27
system_info = self.server.read_record(
teamtrack.table['SYSTEMINFO'], 1)
teamtrack_dbver = system_info['DBVER']
if teamtrack_dbver < supported_dbver:
# "TeamTrack database version %d is not supported by the
# P4DTI. The minimum supported version is %d."
raise error, catalog.msg(617, (teamtrack_dbver,
supported_dbver))
# Fields to add to the TS_CASES table.
new_fields = [
{ 'name': 'P4DTI_RID',
'type': teamtrack.field_type['TEXT'],
'length': 32,
'attributes': 1, # Fixed-width text.
'description': "P4DTI replicator identifier",
'value': '' },
{ 'name': 'P4DTI_SID',
'type': teamtrack.field_type['TEXT'],
'length': 32,
'attributes': 1, # Fixed-width text.
'description': "P4DTI Perforce server identifier",
'value': '' },
{ 'name': 'P4DTI_JOBNAME',
'type': teamtrack.field_type['TEXT'],
'length': 0, # Arbitrarily long.
'attributes': 0, # "Memo" = variable-width.
'description': "P4DTI Perforce jobname",
'value': '' },
]
# Make a TS_CASES record so we can see if the new fields are
# already present.
case = self.server.new_record(self.server.case_table_id())
# Add each new field if not present.
added_fields = []
for new_field in new_fields:
if not case.has_key(new_field['name']):
# "Installing field '%s' in the TS_CASES table."
self.log(601, new_field['name'])
f = self.server.new_record(teamtrack.table['FIELDS'])
f['TABLEID'] = self.server.case_table_id()
f['NAME'] = new_field['description']
f['DBNAME'] = new_field['name']
f['FLDTYPE'] = new_field['type']
f['LEN'] = new_field['length']
f['ATTRIBUTES'] = new_field['attributes']
f['STATUS'] = 0 # Active, not deleted.
f['PROPERTY'] = 1 # Not editable.
f['DESCRIPTION'] = new_field['description']
f['DEFAULTCHAR'] = new_field['value']
f.add_field()
added_fields.append(new_field)
if added_fields:
if len(added_fields) != len(new_fields):
# "Partially installed the new fields in the TS_CASES
# table. Previous installation was not up to date."
self.log(602)
else:
# "Installed all new fields in the TS_CASES table."
self.log(603)
# Really we shouldn't be doing all this nonsense up front,
# because there's usually no need to change the LAST_CHANGE
# record.
# Find the first change made on or after the start date. If
# there is one, then use the previous change record as the "last
# change". See job000189. It's important to make sure not to
# fetch large numbers of change records from TeamTrack here
# since there may be very many, and TeamTrack may be very slow;
# hence the roundabout nature of this query.
query = ("TS_ID IN (SELECT MIN(TS_ID) FROM TS_CHANGES WHERE "
"TS_TIME>=%d)" % self.config.start_date)
first_change = self.query('CHANGES', query)
if first_change:
last_change = first_change[0]['ID'] - 1
else:
# Find the highesy ID used in the CHANGES table. We'll use
# this for the initial value of the LAST_CHANGE parameter.
# See job000047 and the TeamTrack schema documentation.
query = 'TS_ID=(SELECT MAX(TS_ID) FROM TS_CHANGES)'
last_change = self.query('CHANGES', query)[0]['ID']
# Build a string of status values separated by / for the
# STATUS_VALUES keyword.
status_values = string.join(map(lambda p: p[1],
self.config.state_pairs),'/')
# config_params gives the values that should appear in the
# Replicator's configuration parameters table. Each entry in
# the config_params list is a 4-tuple ( parameter name, field
# name, field value, force update?) See the TeamTrack schema
# extensions document for the meaning of these configuration
# parameters.
config_params = [
( 'LAST_CHANGE', 'INFO1', last_change, 0 ),
( 'SERVER', 'FILENAME',
dict_repr_workaround({'sid': self.sid, 'description':
self.config.p4_server_description}),
1 ),
( 'STATUS_VALUES', 'FILENAME',
dict_repr_workaround({ 'sid': self.sid, 'description':
status_values }),
1 ),
]
if self.config.changelist_url:
config_params.append(
( 'CHANGELIST_URL', 'FILENAME',
dict_repr_workaround({ 'sid': self.sid, 'description':
self.config.changelist_url}),
1 ))
if self.config.job_url:
config_params.append(
( 'JOB_URL', 'FILENAME',
dict_repr_workaround({ 'sid': self.sid, 'description':
self.config.job_url}),
1 ))
# Get all the configuration parameters for this replicator; make
# a hash by parameter name.
query = ("TS_TYPE=%d AND TS_CHAR1='%s'"
% (vcactions_type_config, self.rid))
params = {}
for p in self.query('VCACTIONS', query):
params[p['CHAR2']] = p
# Now add or update.
for name, field, value, force_p in config_params:
if not params.has_key(name):
r = self.server.new_record(teamtrack.table['VCACTIONS'])
r['TYPE'] = vcactions_type_config
r['CHAR1'] = self.rid
r['CHAR2'] = name
r[field] = value
r.add()
# "Put '%s' parameter in replicator configuration with
# value '%s'."
self.log(604, (name, repr(value)))
elif force_p and params[name][field] != value:
params[name][field] = value
params[name].update()
# "Updated '%s' parameter in replicator configuration to
# have value '%s'."
self.log(605, (name, repr(value)))
# Delete CHANGELIST_URL parameter if changelist-url is None.
# See job000169.
if (params.has_key('CHANGELIST_URL')
and not self.config.changelist_url):
self.server.delete_record(teamtrack.table['VCACTIONS'],
params['CHANGELIST_URL']['ID'])
# Delete JOB_URL parameter if job_url is None. See
# job000169.
if params.has_key('JOB_URL') and not self.config.job_url:
self.server.delete_record(teamtrack.table['VCACTIONS'],
params['JOB_URL']['ID'])
# Supported features; see [GDR 2000-10-16, 3.5].
feature = {
'filespecs': 1,
'fixes': 1,
'migrate_issues': 1,
'new_issues': 1,
'new_users': 0,
}
def supports(self, feature):
return self.feature.get(feature, 0)
def issue(self, case_id):
assert isinstance(case_id, types.StringType)
try:
case = self.server.read_record(self.server.case_table_id(),
int(case_id))
return teamtrack_case(case, self)
except ValueError:
# case_id was not a number and int() failed.
return None
except teamtrack.tsapi_error:
# No such issue.
return None
# 6.4. Migration interface
#
# See [GDR 2000-10-16, 7.1]. Note that we don't know how to add new
# users from the API (see job000412) so there's no add_user method.
def new_issue(self, dict, jobname):
assert isinstance(dict, types.DictType)
assert isinstance(jobname, types.StringType)
case = self.server.new_record(self.server.case_table_id())
for k, v in dict.items():
case[k] = v
self.read_users()
if (case['SUBMITTER'] == 0
or not self.user_id_to_name.has_key(case['SUBMITTER'])):
# "Can't submit new issue to TeamTrack: SUBMITTER %d is
# unknown."
raise error, catalog.msg(640, case['SUBMITTER'])
issue_id = case.submit(self.user_id_to_name[case['SUBMITTER']])
case = self.server.read_record(self.server.case_table_id(),
issue_id)
case = teamtrack_case(case, self)
case.setup_for_replication(jobname)
return case
def new_issues_start(self):
pass
def new_issues_end(self):
pass
# 6.5. Read and cache TeamTrack database
# read_auxiliary_table(table). Record mappings between id and name
# for a given auxiliary table (the tables PROJECTS, COMPANIES,
# PRODUCTS, SERVICEAGREEMENTS are suitable for this method).
def read_auxiliary_table(self, table_name):
assert isinstance(table_name, types.StringType)
table_id = teamtrack.table[table_name]
if self.cached_auxiliary_table.get(table_id, 0):
return
else:
self.cached_auxiliary_table[table_id] = 1
aux_cursor = cursor(self, table_name, '')
self.table_to_id_to_name[table_id] = {}
self.table_to_name_to_id[table_id] = {}
while 1:
r = aux_cursor.fetchone()
if r == None:
break
self.table_to_id_to_name[table_id][r['ID']] = r['NAME']
if self.table_to_name_to_id[table_id].has_key(r['NAME']):
# "Warning: table '%s' has two entries called '%s'."
self.log(607, (table_name, r['NAME']))
self.table_to_name_to_id[table_id][r['NAME']] = r['ID']
# read_fields(). Read and cache the TS_FIELDS table.
def read_fields(self):
if self.cached_fields:
return
else:
self.cached_fields = 1
fields_cursor = cursor(self, 'FIELDS',
'TS_TABLEID = %d AND TS_STATUS = 0'
% self.server.case_table_id())
while 1:
f = fields_cursor.fetchone()
if f == None:
break
# Normalise the case of the database field name so that we
# can rely on it being uppercase throughout the code. See
# [GDR 2000-11-01, item 14].
f['DBNAME'] = string.upper(f['DBNAME'])
self.field[f['ID']] = f
# read_selections(). Record mappings between selection name and id,
# so that we can transform single-select fields.
def read_selections(self):
if self.cached_selections:
return
else:
self.cached_selections = 1
self.read_fields()
# sn_map is a map from selection id to selection name, used to
# map from TeamTrack to Perforce.
sn_map = {}
# fns_map is a map from selection name to field name to
# selection id. This is used to map from Perforce to TeamTrack.
fns_map = {}
selections_cursor = cursor(self, 'SELECTIONS', '')
while 1:
s = selections_cursor.fetchone()
if s == None:
break
sn_map[s['ID']] = s['NAME']
if self.field.has_key(s['FLDID']):
field_name = self.field[s['FLDID']]['DBNAME']
if not fns_map.has_key(field_name):
fns_map[field_name] = {}
fns_map[field_name][s['NAME']] = s['ID']
# Store these maps for later use.
self.field_to_selection_to_id = fns_map
self.selection_id_to_name = sn_map
# Determine a mapping from project id and transition name to the
# transition id. Determine a mapping from project id and state name
# to state id.
def read_states(self):
if self.cached_states:
return
else:
self.cached_states = 1
# sn_map and ns_map are maps from state id to state name
sn_map = {}
ns_map = {}
states_cursor = cursor(self, 'STATES', '')
while 1:
s = states_cursor.fetchone()
if s == None:
break
# Normalize the case of the state name. See "Case of state
# names" design decisions [RB 2000-11-28].
sn_map[s['ID']] = string.lower(s['NAME'])
ns_map[string.lower(s['NAME'])] = s['ID']
# pns_map is a map from project id and lower-cased state name to
# the state id corresponding to that state in that project.
pns_map = {}
self.project = {}
projects_cursor = cursor(self, 'PROJECTS', '')
while 1:
p = projects_cursor.fetchone()
if p == None:
break
pid = p['ID']
self.project[pid] = p
if not pns_map.has_key(pid):
pns_map[pid] = {}
for s in self.server.read_state_list(pid, 1):
# Normalize the case of the state name. See "Case of
# state names" design decisions [RB 2000-11-28].
pns_map[pid][string.lower(s['NAME'])] = s['ID']
# Remember these maps for use later.
self.project_to_name_to_state = pns_map
self.state_id_to_name = sn_map
self.state_name_to_id = ns_map
# Determine a mapping from project id and transition name to the
# transition id. Determine a mapping from project id and state name
# to state id.
def read_transitions(self):
if self.cached_transitions:
return
else:
self.cached_transitions = 1
# psst_map is a map from project id and a pair of state ids to
# the transition id in that project that takes a case from one
# state to the other.
psst_map = {}
projects_cursor = cursor(self, 'PROJECTS', '')
while 1:
p = projects_cursor.fetchone()
if p == None:
break
pid = p['ID']
if not psst_map.has_key(pid):
psst_map[pid] = {}
for t in self.server.read_transition_list(pid):
old = t['OLDSTATEID']
new = t['NEWSTATEID']
psst_map[pid][(old, new)] = t['ID']
# Remember this map for use when choosing transitions.
self.project_to_states_to_transition = psst_map
# read_types(). Record the mapping between issue type and the
# prefix for that type.
def read_types(self):
if self.cached_types:
return
else:
self.cached_types = 1
self.type_id_to_prefix = {}
selections_cursor = cursor(self, 'SELECTIONS', '')
while 1:
t = selections_cursor.fetchone()
if t == None:
break
self.type_id_to_prefix[t['ID']] = t['PREFIX']
# read_users(). Record the mapping between userid and username; and
# between userid and e-mail address (we'll use the latter to map
# Perforce users to TeamTrack users under the assumption that they
# have the same e-mail address in both systems).
def read_users(self):
if self.cached_users:
return
else:
self.cached_users = 1
self.user_name_to_id = {}
self.user_id_to_name = {}
self.user_email_to_id = {}
self.user_id_to_email = {}
users_cursor = cursor(self, 'USERS', '')
while 1:
u = users_cursor.fetchone()
if u == None:
break
self.user_name_to_id[u['LOGINID']] = u['ID']
self.user_id_to_name[u['ID']] = u['LOGINID']
self.user_email_to_id[string.lower(u['EMAIL'])] = u['ID']
self.user_id_to_email[u['ID']] = string.lower(u['EMAIL'])
def replicate_changelist(self, change, client, date, description,
status, user):
query = ("TS_TYPE=%d AND TS_CHAR1='%s' AND TS_INFO1=%d"
% (vcactions_type_changelist, self.rid, change))
changelists = self.query('VCACTIONS', query)
if len(changelists) == 0:
changelist = self.server.new_record(
teamtrack.table['VCACTIONS'])
self.translate_changelist(changelist, change, client, date,
description, status, user)
changelist.add()
return 1
elif self.translate_changelist(changelists[0], change, client,
date, description, status, user):
changelists[0].update()
return 1
else:
return 0
# translate_changelist(tt_changelist, change, client, date,
# description, status, user). Return the changes that were made to
# tt_changelist.
def translate_changelist(self, tt_changelist, change, client, date,
description, status, user):
assert isinstance(change, types.IntType)
assert isinstance(client, types.StringType)
assert isinstance(date, types.IntType)
assert isinstance(description, types.StringType)
assert isinstance(status, types.StringType)
assert isinstance(user, types.IntType)
changes = {}
changes['TYPE'] = vcactions_type_changelist
changes['CHAR1'] = self.rid
changes['CHAR2'] = self.sid
changes['INFO1'] = change
changes['AUTHOR1'] = user
changes['INFO2'] = (status == 'submitted')
changes['TIME1'] = date
changes['FILENAME'] = dict_repr_workaround({
'description': description,
'client': client,
})
for key, value in changes.items():
if tt_changelist[key] != value:
tt_changelist[key] = value
else:
del changes[key]
return changes
# 7. TRANSLATORS
#
# These classes translate values of particular types between TeamTrack
# and Perforce [GDR 2000-10-16, 7.5].
# 7.1. Date translator
#
# This translator class translates dates [GDR 2000-10-16, 7.5.1]
#
# Dates in changelists and jobs are represented as strings in the format
# "2000/01/01 00:00:00". Dates in fixes are represnted as strings
# giving the number of seconds since 1970-01-01 00:00:00.
class date_translator(translator.translator):
readable_date_re = re.compile(
"^([0-9][0-9][0-9][0-9])/([0-9][0-9])/([0-9][0-9]) "
"([0-9][0-9]):([0-9][0-9]):([0-9][0-9])$")
seconds_date_re = re.compile("^[0-9]+$")
def translate_0_to_1(self, tt_date, tt, p4, case = None,
job = None):
assert isinstance(tt_date, types.IntType)
assert isinstance(tt, dt_teamtrack)
assert isinstance(p4, dt_interface.defect_tracker)
assert case == None or isinstance(case, teamtrack_case)
# Empty date fields in TeamTrack appear to be represented by -2.
# See job000146.
if tt_date < 0:
return ''
else:
# TeamTrack stores dates in local time (see job000379).
return time.strftime("%Y/%m/%d %H:%M:%S",
time.localtime(tt_date))
def translate_1_to_0(self, p4_date, tt, p4, case = None,
job = None):
assert isinstance(p4_date, types.StringType)
assert isinstance(tt, dt_teamtrack)
assert isinstance(p4, dt_interface.defect_tracker)
assert case == None or isinstance(case, teamtrack_case)
# Empty date fields in TeamTrack appear to be represented by -2.
# See job000146.
if p4_date == '':
return -2
match = self.readable_date_re.match(p4_date)
if match:
# Note that months are 1-12 in Python, unlike in C. Specify
# -1 for the DST flag -- see job000381.
return int(time.mktime((int(match.group(1)),
int(match.group(2)),
int(match.group(3)),
int(match.group(4)),
int(match.group(5)),
int(match.group(6)),
0, 0, -1)))
elif self.seconds_date_re.match(p4_date):
return int(p4_date)
else:
# "Incorrect date in Perforce: '%s' is not in the format
# 'YYYY/mm/dd HH:MM:SS'."
raise error, catalog.msg(618, p4_date)
# 7.2. Elapsed time translator
#
# This translator class translates elapsed times (for example, the
# ESTTIMETOFIX field in the default TeamTrack workflow). Elapsed times
# are represented in TeamTrack as a number of seconds; there's no
# corresponding field type in Perforce so we represent them as strings
# in the format "Hours:Mins:Secs".
class elapsed_time_translator(translator.translator):
elapsed_time_re = re.compile("^([0-9]+):([0-9][0-9]):([0-9][0-9])$")
def translate_0_to_1(self, tt_time, tt, p4, case = None,
job = None):
assert isinstance(tt_time, types.IntType)
assert isinstance(tt, dt_teamtrack)
assert isinstance(p4, dt_interface.defect_tracker)
assert case == None or isinstance(case, teamtrack_case)
# Empty time fields in TeamTrack appear to be represented by -2.
# See job000146.
if tt_time < 0:
return ''
else:
return '%d:%02d:%02d' % (tt_time / 3600,
(tt_time / 60) % 60,
tt_time % 60)
def translate_1_to_0(self, p4_time, tt, p4, case = None,
job = None):
assert isinstance(p4_time, types.StringType)
assert isinstance(tt, dt_teamtrack)
assert isinstance(p4, dt_interface.defect_tracker)
assert case == None or isinstance(case, teamtrack_case)
# Empty time fields in TeamTrack appear to be represented by -2.
# See job000146.
if p4_time == '':
return -2
match = self.elapsed_time_re.match(p4_time)
if match:
return (int(match.group(1)) * 3600
+ int(match.group(2)) * 60
+ int(match.group(3)))
else:
# "Incorrect time in Perforce: '%s' is not in the format
# 'H:MM:SS'."
raise error, catalog.msg(619, p4_time)
# 7.3. Foreign key translator
#
# This class translates foreign key fields; that is, fields in TeamTrack
# that reference the TS_ID field in an auxiliary table (for example, the
# TS_PROJECTID field in the default TeamTrack database refers to the
# TS_ID field in the TS_PROJECTS table).
#
# The name of the auxiliary table must be supplied to the constructor.
class auxiliary_translator(translator.translator):
table_name = None
table = None
def __init__(self, table_name):
if not teamtrack.table.has_key(table_name):
# "No such table: %s."
raise error, catalog.msg(620, table_name)
self.table_name = table_name
self.table = teamtrack.table[table_name]
def translate_0_to_1(self, tt_value, tt, p4, case = None,
job = None):
assert isinstance(tt_value, types.IntType)
assert isinstance(tt, dt_teamtrack)
assert isinstance(p4, dt_interface.defect_tracker)
assert case == None or isinstance(case, teamtrack_case)
if tt_value == 0:
return '(None)'
if not (tt.table_to_id_to_name.has_key(self.table)
and tt.table_to_id_to_name[self.table].
has_key(tt_value)):
# The entity might have been added since we last looked in
# the database, so refresh our cache.
tt.read_auxiliary_table(self.table_name)
if (tt.table_to_id_to_name.has_key(self.table)
and tt.table_to_id_to_name[self.table].has_key(tt_value)):
return tt.table_to_id_to_name[self.table][tt_value]
else:
# "No TeamTrack entity in table '%s' with id %d."
raise error, catalog.msg(621, (self.table_name, tt_value))
def translate_1_to_0(self, p4_value, tt, p4, case = None,
job = None):
assert isinstance(p4_value, types.StringType)
assert isinstance(tt, dt_teamtrack)
assert isinstance(p4, dt_interface.defect_tracker)
assert case == None or isinstance(case, teamtrack_case)
if p4_value == '(None)' or p4_value == '':
return 0
if not (tt.table_to_name_to_id.has_key(self.table)
and tt.table_to_name_to_id[self.table].
has_key(p4_value)):
# The entity might have been added since we last looked in
# the database, so refresh our cache.
tt.read_auxiliary_table(self.table_name)
if (tt.table_to_name_to_id.has_key(self.table)
and tt.table_to_name_to_id[self.table].has_key(p4_value)):
return tt.table_to_name_to_id[self.table][p4_value]
else:
# "No TeamTrack entity in table '%s' with name '%s'."
raise error, catalog.msg(622, (self.table_name, p4_value))
# 7.4. Single select translator
#
# This class translates values in single select fields. In TeamTrack
# the value of a single select field is a reference to the TS_ID field
# in the TS_SELECTIONS table [TeamShare 2000-01-20]. In Perforce we
# represent the value as the name of the selection. Because single
# selection fields in TeamTrack are mapped to select fields in Perforce,
# we have to translate the name using the keyword translator [GDR
# 2000-10-16, 7.5.2] so that it is valid in Perforce.
class single_select_translator(translator.translator):
# The field that this translator translates.
field = None
# The Perforce keyword translator
keyword_translator = None
def __init__(self, field, keyword_translator):
self.field = field
self.keyword_translator = keyword_translator
def translate_0_to_1(self, tt_selection, tt, p4, case = None,
job = None):
assert isinstance(tt_selection, types.IntType)
assert isinstance(tt, dt_teamtrack)
assert isinstance(p4, dt_interface.defect_tracker)
assert case == None or isinstance(case, teamtrack_case)
if not (tt.selection_id_to_name.has_key(tt_selection)):
# The selection might have been added since we last looked
# in the database, so refresh our cache.
tt.read_selections()
if tt.selection_id_to_name.has_key(tt_selection):
# Selections are 'word's in Perforce and arbitrary strings
# in TeamTrack, so translate it.
return self.keyword_translator.translate_0_to_1(
tt.selection_id_to_name[tt_selection])
else:
# "No TeamTrack selection name for selection id '%d'."
raise error, catalog.msg(623, tt_selection)
def translate_1_to_0(self, p4_selection, tt, p4, case = None,
job = None):
assert isinstance(p4_selection, types.StringType)
assert isinstance(tt, dt_teamtrack)
assert isinstance(p4, dt_interface.defect_tracker)
assert case == None or isinstance(case, teamtrack_case)
# Selections are 'word's in Perforce and arbitrary strings in
# TeamTrack, so translate it.
tt_selection = self.keyword_translator.translate_1_to_0(
p4_selection)
if p4_selection == '(None)' or p4_selection == '':
return 0
if not (tt.field_to_selection_to_id.has_key(self.field)
and tt.field_to_selection_to_id[self.field].
has_key(tt_selection)):
# The selection might have been added since we last looked
# in the database, so refresh our cache.
tt.read_selections()
if (tt.field_to_selection_to_id.has_key(self.field)
and tt.field_to_selection_to_id[self.field].
has_key(tt_selection)):
return tt.field_to_selection_to_id[self.field][tt_selection]
else:
# "No TeamTrack selection for field '%s' corresponding to
# Perforce selection '%s'."
raise error, catalog.msg(624, (self.field, p4_selection))
# 7.5. State translator
#
# This class translates case states [GDR 2000-10-16, 7.5.2]. In
# TeamTrack a state is a reference to the TS_ID field in the TS_STATES
# table. In Perforce we store the name of the state. Because the state
# field is a select field in Perforce, we have to translate the states
# name using the keyword translator so that it is valid in Perforce.
#
# Note that when translating from Perforce to TeamTrack we use the
# project of the case in case we need to disambiguate between states
# that have the same name.
class state_translator(translator.translator):
# A map from TeamTrack state name to Perforce state name.
state_tt_to_p4 = {}
# A map from Perforce state name to TeamTrack state name (the
# reverse of the above map).
state_p4_to_tt = {}
# The states argument is a list of pairs (TeamTrack state name,
# Perforce state name).
def __init__(self, states):
# Compute the maps.
for tt_state, p4_state in states:
assert isinstance(tt_state, types.StringType)
assert isinstance(p4_state, types.StringType)
self.state_tt_to_p4[tt_state] = p4_state
self.state_p4_to_tt[p4_state] = tt_state
def translate_0_to_1(self, tt_state, tt, p4, case = None,
job = None):
assert isinstance(tt_state, types.IntType)
assert isinstance(tt, dt_teamtrack)
assert isinstance(p4, dt_interface.defect_tracker)
assert case == None or isinstance(case, teamtrack_case)
if not tt.state_id_to_name.has_key(tt_state):
# The workflows may have changed since we last looked in the
# database, so refresh our cache.
tt.read_states()
if tt.state_id_to_name.has_key(tt_state):
tt_name = tt.state_id_to_name[tt_state]
if self.state_tt_to_p4.has_key(tt_name):
return self.state_tt_to_p4[tt_name]
else:
# "No Perforce state corresponding to TeamTrack state
# '%s'."
raise error, catalog.msg(625, tt_name)
else:
# "No state name for TeamTrack state %d."
raise error, catalog.msg(626, tt_state)
def translate_1_to_0(self, p4_state, tt, p4, case, job = None):
assert isinstance(p4_state, types.StringType)
assert isinstance(tt, dt_teamtrack)
assert isinstance(p4, dt_interface.defect_tracker)
if not self.state_p4_to_tt.has_key(p4_state):
# "Perforce state '%s' is unknown."
raise error, catalog.msg(627, p4_state)
tt_state = self.state_p4_to_tt[p4_state]
# If we know the case's project, we'll find a state belonging to
# that project.
if case:
project = case['PROJECTID']
if not (tt.project_to_name_to_state.has_key(project)
and tt.project_to_name_to_state[project].
has_key(tt_state)):
# The state might have been added or the workflows
# changed since we last looked in the database, so
# refresh our cache.
tt.read_states()
if (tt.project_to_name_to_state.has_key(project)
and tt.project_to_name_to_state[project].
has_key(tt_state)):
return tt.project_to_name_to_state[project][tt_state]
else:
# "No TeamTrack state in project '%s' corresponding to
# Perforce state '%s'."
raise error, catalog.msg(628, (project, p4_state))
# Otherwise just do the best we can by finding a state with the
# specified name. The idea is to do our best to support
# migration (when we don't know which project the newly-created
# issue will belong to).
else:
if not tt.state_name_to_id.has_key(tt_state):
# The state might have been added or the workflows
# changed since we last looked in the database, so
# refresh our cache.
tt.read_states()
if tt.state_name_to_id.has_key(tt_state):
return tt.state_name_to_id[tt_state]
else:
# "No TeamTrack state corresponding to Perforce state
# '%s'."
raise error, catalog.msg(642, p4_state)
# 7.6. Text translator
#
# This class translates multi-line text fields [GDR 2000-10-16, 7.5.3].
class text_translator(translator.translator):
# Transform TeamTrack memo field contents to Perforce text field
# contents by converting line endings. See job000008 and job000009.
def translate_0_to_1(self, tt_string, tt, p4, case = None,
job = None):
assert isinstance(tt_string, types.StringType)
assert isinstance(tt, dt_teamtrack)
assert isinstance(p4, dt_interface.defect_tracker)
assert case == None or isinstance(case, teamtrack_case)
# Replace \r\n with \n.
tt_string = string.replace(tt_string, '\r\n', '\n')
# Add final newline, unless the string is empty.
if tt_string:
tt_string = tt_string + '\n'
return tt_string
# Transform Perforce text field contents to TeamTrack memo field
# contents by converting line endings. See job000008 and job000009.
def translate_1_to_0(self, p4_string, tt, p4, case = None,
job = None):
assert isinstance(p4_string, types.StringType)
assert isinstance(tt, dt_teamtrack)
assert isinstance(p4, dt_interface.defect_tracker)
assert case == None or isinstance(case, teamtrack_case)
# Remove final newlines (if any).
while p4_string and p4_string[-1] == '\n':
p4_string = p4_string[:-1]
# Replace \n with \r\n.
p4_string = string.replace(p4_string, '\n', '\r\n')
return p4_string
# 7.7. User translator
#
# This class translates users [GDR 2000-10-16, 7.5.3].
#
# The user_translator needs to cope with three special cases.
#
# First, user fields in TeamTrack can be empty (that is, userid is 0).
# We replicate these to and from the dummy user "(None)" in Perforce,
# since that's how non-existent users show up in TeamTrack.
#
# Second, we don't insist that all users in Perforce have licences in
# TeamTrack [GDR 2000-10-16, 4.8]. For example, a user who made a
# changelist long ago and has left the company. These users are mapped
# to the TeamTrack userid 0. (But note that people shouldn't be able to
# *change* issues in TeamTrack unless they have a licence -- this was
# agreed with TeamShare [GDR 2000-08-18, 3.3.1]. It isn't the
# responsibility of the user_translator to worry about this. That's up
# to the teamtrack_case.update method). See job000087.
#
# Third, we don't insist that all users in TeamTrack have licences in
# Perforce. For these users we simply put their TeamTrack user name in
# Perforce -- this works because Perforce doesn't check user names.
class user_translator(translator.user_translator):
# Build match by e-mail address?
matched_users = 0
# A map from Perforce user's e-mail address to Perforce user id.
p4_email_to_user = None
# A map from Perforce user id to Perforce full name.
p4_user_to_fullname = None
# A map from Perforce user name to e-mail address for users with
# duplicate e-mail addresses in Perforce.
user_p4_duplicate = None
# A map from Perforce user name to e-mail address for Perforce
# users that can't be matched with users in TeamTrack.
unmatched_p4_users = None
# A map from TeamTrack user name to e-mail address for TeamTrack
# users that can't be matched with users in Perforce.
unmatched_tt_users = None
# A map from TeamTrack user name to e-mail address for users with
# duplicate e-mail addresses in TeamTrack.
user_tt_duplicate = None
# A map from TeamTrack user id to Perforce username (for users where
# we can work out a correspondence by e-mail address).
user_tt_to_p4 = None
# A map from Perforce username to TeamTrack user id (for users where
# we can work out a correspondence by e-mail address).
user_p4_to_tt = None
def __init__(self):
self.p4_email_to_user = {}
self.p4_user_to_fullname = {}
self.user_tt_to_p4 = {}
self.user_p4_to_tt = {}
# Build the user_tt_to_p4 and user_p4_to_tt maps.
def match_users(self, tt, p4):
assert isinstance(tt, dt_teamtrack)
assert isinstance(p4, replicator.dt_perforce)
# Don't do this more than once per poll for performance reasons
# (see job000148).
if self.matched_users and tt.cached_users:
return
# Clear the maps.
self.p4_email_to_user = {}
self.p4_user_to_fullname = {}
self.unmatched_p4_users = {}
self.unmatched_tt_users = {}
self.user_p4_duplicate = {}
self.user_p4_to_tt = {}
self.user_tt_duplicate = {}
self.user_tt_to_p4 = {}
# Read TeamTrack users and e-mail addresses.
tt.read_users()
# Read Perforce users and e-mail addresses.
p4_users = p4.p4.run("users")
for u in p4_users:
email = string.lower(u['Email'])
if self.p4_email_to_user.has_key(email):
self.user_p4_duplicate[u['User']] = email
user = self.p4_email_to_user[email]
self.user_p4_duplicate[user] = email
else:
self.p4_email_to_user[email] = u['User']
self.p4_user_to_fullname[u['User']] = u['FullName']
# Pair up users by e-mail address, or if that fails, then by
# name. Record duplicate e-mail addresses and unmatched
# TeamTrack users.
for user, email in tt.user_id_to_email.items():
# Ignore special TeamTrack user 0 (None).
if user == 0:
continue
name = tt.user_id_to_name[user]
if self.p4_email_to_user.has_key(email):
p4_user = self.p4_email_to_user[email]
if self.user_p4_to_tt.has_key(p4_user):
tt_userid = self.user_p4_to_tt[p4_user]
duplicate = tt.user_id_to_name[tt_userid]
# "Two TeamTrack users ('%s' and '%s') have the same
# e-mail address '%s'."
tt.log(635, (name, duplicate, email))
self.user_tt_duplicate[duplicate] = email
self.user_tt_duplicate[name] = email
self.unmatched_tt_users[name] = email
else:
# "Matched TeamTrack user '%s' with Perforce user
# '%s' by e-mail address '%s'."
tt.log(613, (name, p4_user, email))
self.user_tt_to_p4[user] = p4_user
self.user_p4_to_tt[p4_user] = user
elif self.p4_user_to_fullname.has_key(name):
# "Matched TeamTrack user '%s' with Perforce user '%s'
# by userid."
tt.log(634, (name, name))
self.user_tt_to_p4[user] = name
self.user_p4_to_tt[name] = user
else:
self.unmatched_tt_users[name] = email
# Record unmatched Perforce users.
for email, user in self.p4_email_to_user.items():
if not self.user_p4_to_tt.has_key(user):
self.unmatched_p4_users[user] = email
self.matched_users = 1
def unmatched_users(self, tt, p4):
assert isinstance(tt, dt_teamtrack)
assert isinstance(p4, replicator.dt_perforce)
self.match_users(tt, p4)
# "These TeamTrack users will appear as themselves in Perforce
# even though there is no such Perforce user."
tt_user_msg = catalog.msg(629)
# "These Perforce users will appear in TeamTrack as the user
# (None). It will not be possible to assign issues to these
# users."
p4_user_msg = catalog.msg(630)
# "These TeamTrack users have duplicate e-mail addresses. They
# may have been matched with the wrong Perforce user."
tt_duplicate_msg = catalog.msg(636)
# "These Perforce users have duplicate e-mail addresses. They
# may have been matched with the wrong TeamTrack user."
p4_duplicate_msg = catalog.msg(637)
return (self.unmatched_tt_users, self.unmatched_p4_users,
tt_user_msg, p4_user_msg, self.user_tt_duplicate,
self.user_p4_duplicate, tt_duplicate_msg,
p4_duplicate_msg)
keyword = translator.keyword_translator()
def translate_0_to_1(self, tt_user, tt, p4, case = None,
job = None):
assert isinstance(tt_user, types.IntType)
assert isinstance(tt, dt_teamtrack)
assert isinstance(p4, replicator.dt_perforce)
assert case == None or isinstance(case, teamtrack_case)
if tt_user == 0:
return '(None)'
if not self.user_tt_to_p4.has_key(tt_user):
# User is not in our map by e-mail address, so rebuild it
# (see job000162).
self.match_users(tt, p4)
if self.user_tt_to_p4.has_key(tt_user):
return self.user_tt_to_p4[tt_user]
else:
# User is still not in our map by e-mail address, so use the
# TeamTrack username as the Perforce username, if we know
# it, otherwise use '(None)'
tt_name = tt.user_id_to_name.get(tt_user, '(None)')
return self.keyword.translate_0_to_1(tt_name)
def translate_1_to_0(self, p4_user, tt, p4, case = None,
job = None):
assert isinstance(p4_user, types.StringType)
assert isinstance(tt, dt_teamtrack)
assert isinstance(p4, replicator.dt_perforce)
assert case == None or isinstance(case, teamtrack_case)
if p4_user == '(None)' or p4_user == '':
return 0
if not self.user_p4_to_tt.has_key(p4_user):
# User is not in our map by e-mail address, so rebuild it
# (see job000162).
self.match_users(tt, p4)
if self.user_p4_to_tt.has_key(p4_user):
return self.user_p4_to_tt[p4_user]
else:
# User is still not in our map by e-mail address, so use the
# Perforce username as the TeamTrack username, if possible,
# otherwise use user 0 '(None)'.
tt_name = self.keyword.translate_1_to_0(p4_user)
return tt.user_name_to_id.get(tt_name, 0)
# 7.8. Journal translator
#
# This translator translates multi-line journal fields [GDR 2001-09-26].
class journal_translator(text_translator):
separator = '------- Append additional comments below -------'
separator_re = re.compile('^' + re.escape(separator) + '$', re.M)
whitespace_re = re.compile('^\s*$')
tt_fieldname = None
user_translator = None
append_only = 0
def __init__(self, append_only, tt_fieldname, user_translator):
self.append_only = append_only
self.tt_fieldname = tt_fieldname
self.user_translator = user_translator
# Apply multi-line text translation, add separator line.
def translate_0_to_1(self, tt_string, tt, p4, case = None,
job = None):
return (text_translator.translate_0_to_1(self, tt_string, tt,
p4, case, job)
+ self.separator + '\n')
# Apply multi-line text translation, remove separator line, add
# header line if necessary.
def translate_1_to_0(self, p4_string, tt, p4, case = None,
job = None):
if self.append_only and case:
old_value = self.translate_0_to_1(case[self.tt_fieldname],
tt, p4, case, job)
if old_value != p4_string[:len(old_value)]:
# "The TeamTrack field %s is append-only: you're not
# allowed to edit previous comments."
raise error, catalog.msg(633, self.tt_fieldname)
match = self.separator_re.search(p4_string)
if match:
s = p4_string[:match.start(0)]
new_entry = p4_string[match.end(0):]
if not self.whitespace_re.match(new_entry):
if s:
s = s + '\n'
u = job['P4DTI-user']
name = self.user_translator.p4_user_to_fullname.get(u,u)
# TeamTrack formats dates in these journal entries like
# '9/1/2001 2:05PM', regardless of the locale of the
# server machine, so for consistency reproduce this.
# Note the lack of leading zeroes in month, day and
# hour, and that hour is in the range [1,12].
t = time.localtime(time.time())
date = ('%d/%d/%d %d:%02d%s'
% (t[1], t[2], t[0], (t[3] + 11) % 12 + 1, t[4],
time.strftime("%p", t)))
s = (s + date + ' - ' + name + new_entry)
else:
s = p4_string
return text_translator.translate_1_to_0(self, s, tt, p4, case,
job)
# 7.9. Numeric translators
#
# Perforce has no numeric fields, so we use fixed-width text fields.
# Non-numeric content in Perforce fields gets translated to zero.
class numeric_translator(translator.translator):
def translate_0_to_1(self, tt_value, tt, p4, case = None,
job = None):
return str(tt_value)
class int_translator(numeric_translator):
def translate_1_to_0(self, p4_value, tt, p4, case = None,
job = None):
try:
return int(p4_value)
except ValueError:
return 0
class float_translator(numeric_translator):
def translate_1_to_0(self, p4_value, tt, p4, case = None,
job = None):
try:
return float(p4_value)
except ValueError:
return 0.0
class fixed_point_translator(float_translator):
# Number of digits after the decimal point.
digits = 0
def __init__(self, digits):
self.digits = min(max(digits, 0), 100)
def translate_0_to_1(self, tt_value, tt, p4, case = None,
job = None):
return "%%0.%df" % self.digits % tt_value
# 7.10. Binary translator
class binary_translator(translator.translator):
def __init__(self, labels):
self.labels = labels
def translate_0_to_1(self, tt_value, tt, p4, case = None,
job = None):
return self.labels[tt_value != 0]
def translate_1_to_0(self, p4_value, tt, p4, case = None,
job = None):
return p4_value != self.labels[0]
# A. REFERENCES
#
# [GDR 2000-08-08] "Python interface to TeamTrack: design"; Gareth Rees;
# Ravenbrook Limited; 2000-08-08;
# .
#
# [GDR 2000-08-18] "TeamShare design meetings, 2000-08-14/2000-08-16";
# Gareth Rees; Ravenbrook Limited; 2000-08-18;
# .
#
# [GDR 2000-09-04] "TeamTrack database schema extensions for integration
# with Perforce"; Gareth Rees; Ravenbrook Limited; 2000-09-04;
# .
#
# [GDR 2000-10-16] "Perforce Defect Tracking Integration Integrator's
# Guide"; Gareth Rees; Ravenbrook Limited; 2000-10-16;
# .
#
# [GDR 2000-11-01] "Alpha test report for Quokka Sports, 2000-11-01";
# Gareth Rees; Ravenbrook Limited; 2000-11-01;
# .
#
# [GDR 2001-05-16] "Performance analysis of TeamShare API workarounds";
# Gareth Rees; Ravenbrook Limited; 2001-05-16;
# .
#
# [GDR 2001-09-26] "TeamTrack journal fields"; Gareth Rees; Ravenbrook
# Limited; 2001-09-26;
#
#
# [Requirements] "Perforce Defect Tracking Integration Project
# Requirements"; Gareth Rees; Ravenbrook Limited; 2000-05-24;
# .
#
# [TeamShare 2000-01-20] "TeamTrack Database Schema (Database Version:
# 21)"; TeamShare; 2000-01-20;
# .
#
# [TeamShare 2001-04-30] "TeamTrack Database Schema (Database Version:
# 514)"; TeamShare; 2001-04-30;
# .
#
#
# B. DOCUMENT HISTORY
#
# 2000-12-05 GDR Starts replicating TeamTrack data from installation and
# not from the beginning of time.
#
# 2000-12-06 GDR Report both the user and the transition when a
# transition is generated, to make job000133 easier to spot and debug if
# it happens again. Improved message sent when a transition can't be
# found, by using the state names rather than the state ID numbers.
#
# 2000-12-07 GDR Updated user_translator so that it matches users by
# e-mail address if it can; otherwise it defaults to the original
# algorithm: assume that they are identical.
#
# 2000-12-08 GDR The date_translator copes with empty date fields.
#
# 2001-01-19 GDR Handle empty optional fields.
#
# 2001-01-23 GDR Added unmatched_users() method to user_translator
# class.
#
# 2001-02-04 GDR Added start date; changed the initialization of the
# LAST_CHANGE parameter.
#
# 2001-02-12 GDR Fixed queries involving start_date (now seconds since
# epoch).
#
# 2001-02-13 GDR Don't needlessly read/write the whole CASES table when
# starting up.
#
# 2001-02-15 GDR Added elapsed_time_translator class. Delete
# CHANGELIST_URL configuration parameter if changelist_url is None.
#
# 2001-02-19 NB Moved keyword translation to p4.py.
#
# 2001-02-21 GDR Added string_repr_workaround() and
# dict_repr_workaround() functions that do the job of repr() but work
# around a bug in TeamTrack 4.5's string parsing.
#
# 2001-02-23 GDR Added corresponding_id() method to teamtrack_case
# class.
#
# 2001-02-27 GDR Replicator ignores changes with TS_USERID= as
# well as changes with TS_REALUSERID=, to fix job000233.
#
# 2001-03-02 RB Transferred copyright to Perforce under their license.
#
# 2001-03-12 GDR Use messages for errors, logging and e-mail. Removed
# the P4DTI_ACTION field since conflict resolution is now always
# immediate. Get translator class from translator, not replicator. Get
# defect tracker classes from dt_interface, not replicator.
#
# 2001-03-15 GDR Get configuration from the config module.
#
# 2001-03-21 GDR The setup_for_replication() method takes a jobname
# argument.
#
# 2001-04-29 GDR Formatted as a document. Added many references to
# requirements and design.
#
# 2001-05-15 GDR Added cursor class. Changed calls to query() that may
# return many results so that they use cursors instead.
#
# 2001-05-18 GDR Log all calls to query() in the TeamTrack interface, so
# don't bother to log explicitly in read_*() methods. Improved
# debugging output of read_auxiliary_table().
#
# 2001-06-21 NB Treat email addresses case-insensitively. job000337.
#
# 2001-06-26 NB changed interface to changed_entities.
#
# 2001-06-27 NB Make all_issues return all issues replicated by this
# replicator regardless of modification date.
#
# 2001-06-29 GDR Made portable between TeamTrack 4.5 and TeamTrack 5.0
# by using case_table_id() and case_table_name().
#
# 2001-06-30 GDR Added flags which indicate whether TeamTrack data has
# been read and cached this poll, so that we don't read these tables
# more than once per poll, to fix job000148.
#
# 2001-07-02 GDR Fixed bug in cursor implementation. The query was
# being formed as "TS_ID > n AND query", which meant that if the query
# had an OR operator at top level then the meaning would be changed.
# Added extra parentheses.
#
# 2001-07-03 NB Restored middle result of changed_entities.
#
# 2001-07-09 NB Add job_url by analogy with changelist_url.
#
# 2001-07-19 GDR Fixed bug in cursor implementation; added cases_cursor
# for efficient implementation of fetching set of cases.
#
# 2001-07-24 GDR Specify default value for fields added to cases table
# in TeamTrack.
#
# 2001-08-02 GDR Use time.localtime when converting dates, since
# TeamTrack stores dates in local time.
#
# 2001-08-06 GDR Specify -1 for DST argument to mktime().
#
# 2001-08-07 GDR Rebuild user mapping if we don't find a user in it (but
# no more than once per poll).
#
# 2001-10-01 GDR Support TeamTrack journal fields; see job000371.
#
# 2001-10-02 GDR Unmatched user report covers both halves of matching
# algorithm (name as well as e-mail): see job000359.
#
# 2001-10-04 GDR Added numeric and binary translators.
#
# 2001-10-18 GDR Improved error messages for bad dates and times.
#
# 2001-10-25 GDR Support migration from Perforce.
#
# 2001-11-06 GDR Get hold of submitted issue even if TeamTrack changes
# the issue type during submission.
#
# 2001-11-22 GDR Added fixed_point_translator.
#
# 2001-11-26 GDR The state translator tries to translate from Perforce
# to TeamTrack even if it doesn't know the project.
#
# 2001-12-04 GDR New method supports.
#
# 2002-01-07 GDR Find the highest ID in the CHANGES table in a way
# that's portable between TeamTrack database versions.
#
# 2002-01-10 GDR Use submit method that returns the record id. Support
# migrate_issues and new_issues features only if submit method is
# supported by the Python interface to TeamTRack.
#
# 2002-01-29 GDR User translator applies the keyword translator for
# unknown users, in case they have spaces or other forbidden characters.
#
#
# C. COPYRIGHT AND LICENCE
#
# This file is copyright (c) 2001 Perforce Software, Inc. All rights
# reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# HOLDERS AND CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
# OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
# TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
# DAMAGE.
#
#
# $Id: //info.ravenbrook.com/project/p4dti/version/1.5/code/replicator/dt_teamtrack.py#3 $