# Perforce Defect Tracking Integration Project
#
#
# CONFIGURE_TEAMTRACK.PY -- BUILD P4DTI CONFIGURATION FOR TEAMTRACK
#
# Gareth Rees, Ravenbrook Limited, 2000-11-17
#
#
# 1. INTRODUCTION
#
# This module defines a configuration generator for the TeamTrack integration.
# Configuration generators are documented in detail in [IG, 8].
#
# The intended readership of this document is project developers.
#
# This document is not confidential.
import catalog
import check_config
import dt_teamtrack
import logger
import message
import re
import socket
import string
import teamtrack
import time
import translator
import types
error = "TeamTrack configuration error"
# 2. BUILD THE MAPPING BETWEEN STATES IN TEAMTRACK AND PERFORCE
#
# The make_state_pairs function takes a TeamTrack connection and the "closed
# state", and returns a list of pairs of state names (TeamTrack state, Perforce
# state). This list will be used to translate between states, and also to
# generate the possible values for the State field in Perforce.
#
# The closed_state argument is the TeamTrack state which maps to the special
# state 'closed' in Perforce, or None if there is no such state. See
# requirement 45. See the decision decision [RB 2000-11-28b].
#
# The case of state names in these pairs is normalized for usability in
# Perforce: see the design decision [RB 2000-11-28a].
keyword_translator = translator.keyword_translator()
def make_state_pairs(teamtrack_connection, closed_state):
# 2.1. Get TeamTrack states
#
# Select states belong to CASES workflows (ignore states belonging to
# INCIDENTS workflows). This was motivated by job000194. See the
# definition of the TS_STATES table in the TeamTrack database schema
# [TeamShare 2000-01-20].
query = ('TS_PROJECTID IN (SELECT TS_ID FROM TS_WORKFLOWS WHERE '
'TS_TABLEID = %d)' % teamtrack_connection.case_table_id())
states = teamtrack_connection.query(teamtrack.table['STATES'], query)
state_pairs = []
state_p4_to_tt = {}
found_closed_state = 0
# 2.2. Make Perforce versions of the state names
#
# We convert the TeamTrack state name to lowercase (so that it's easy to
# type in Perforce), apply the keyword translator (so that the state is
# legal in Perforce), and then apply two special cases:
#
# 1. The "closed_state" configuration parameter.
#
# 2. Perforce jobs can't have state "new" (this indicates a fresh job and
# Perforce changes the state to "open"). Nor can they have state "ignore",
# because that is used in the submit form to indicate that a job shouldn't
# be fixed by the change.
#
# Unfortunately, "new" and "ignore" are common names for states in the
# defect tracker (the former is in the default workflow in TeamTrack), so
# we don't disallow them, but translate them to "_new" and "_ignore"; we
# disallow those instead (or rather, we quit if two TeamTrack states map to
# the same state in Perforce.) See job000141.
for s in states:
tt_state = string.lower(s['NAME'])
if closed_state != None and tt_state == string.lower(closed_state):
p4_state = 'closed'
found_closed_state = 1
else:
p4_state = keyword_translator.translate_0_to_1(tt_state)
if p4_state in ['new', 'ignore']:
p4_state = '_' + p4_state
if (state_p4_to_tt.has_key(p4_state)
and state_p4_to_tt[p4_state] != tt_state):
# "Two TeamTrack states '%s' and '%s' map to the same Perforce
# state '%s'."
raise error, catalog.msg(400, (tt_state, state_p4_to_tt[p4_state], p4_state))
state_p4_to_tt[p4_state] = tt_state
pair = (tt_state, p4_state)
if pair not in state_pairs:
state_pairs.append(pair)
if closed_state != None and not found_closed_state:
# "You specified the closed_state '%s', but there's no such TeamTrack
# state."
raise error, catalog.msg(401, closed_state)
return state_pairs
# 3. CONVERT DATE/TIME TO SECONDS
#
# This function converts a date/time in standard format, like '2001-02-12
# 19:19:24' [ISO 8601] into seconds since the epoch.
#
# We use this to convert the start_date configuration parameter. It is
# specified as an date/time for ease of entry, but TeamTrack represents
# date/times as seconds since the epoch. (Note that we specify -1 for
# the DST flag -- see job000381).
def convert_isodate_to_secs(isodate):
assert isinstance(isodate, types.StringType)
date_re = "^(\d\d\d\d)-(\d\d)-(\d\d) (\d\d):(\d\d):(\d\d)$"
match = re.match(date_re, isodate)
assert match
return time.mktime(tuple(map(int, match.groups()) + [0,0,-1]))
# 4. BUILD P4DTI CONFIGURATION FOR TEAMTRACK
def configuration(config):
# 4.1. Check TeamTrack-specific configuration parameters
check_config.check_list_of(config.replicated_fields, 'replicated_fields',
types.StringType, 'strings')
check_config.check_string(config.teamtrack_password, 'teamtrack_password')
check_config.check_string(config.teamtrack_server, 'teamtrack_server')
check_config.check_string(config.teamtrack_user, 'teamtrack_user')
# 4.2. Open a connection to the TeamTrack server
connection = teamtrack.connect(config.teamtrack_user,
config.teamtrack_password,
config.teamtrack_server)
# 4.3. Translators
#
# We make only one user translator so that the TeamTrack USER table only
# needs to be read once when the replicator starts and once when there's a
# cache miss. See job000148.
date_translator = dt_teamtrack.date_translator()
elapsed_time_translator = dt_teamtrack.elapsed_time_translator()
null_translator = translator.translator()
text_translator = dt_teamtrack.text_translator()
user_translator = dt_teamtrack.user_translator()
state_pairs = make_state_pairs(connection, config.closed_state)
state_translator = dt_teamtrack.state_translator(state_pairs)
# 4.4. TeamTrack field types
#
# The table 'type_table' defines which TeamTrack field types are supported
# by the P4DTI, and defines how values in these fields are translated
# between TeamTrack and Perforce.
#
# 'type_table' is a map from TeamTrack field type to a tuple (TeamTrack
# field type name, Perforce field type, Perforce field length, translator).
#
# The TeamTrack field type is the value in the TS_FLDTYPE column in the
# TS_FIELDS table; see the TeamTrack schema for details [TeamShare
# 2000-01-20].
#
# The field type name is for documentation only.
#
# The Perforce field length is None when it should to be the same as the
# corresponding field length in TeamTrack (this applies only to
# fixed-length text fields).
#
# Commented out fields are ones we don't support. If new translators are
# written for those field types, we can uncomment them.
#
# This table isn't complete. There are a number of special cases that will
# be applied when each field is considered:
#
# 1. A TEXT field (type 101) with TS_ATTRIBUTES = 0 is actually a MEMO
# field.
#
# 2. A DATETIME field (type 103) with TS_ATTRIBUTES = 2 or 3 is actually
# an elapsed time field. We have the dummy type 'time' for elapsed time
# fields.
#
# 3. Each SELECTION field (type 104) needs its own translator, and the
# translator needs to know the name of the field in the TS_CASES table. So
# we make a translator object for each replicated field.
aux = dt_teamtrack.auxiliary_translator
type_table = {
100: ( 'NUMERIC', 'word', 32, null_translator ),
101: ( 'TEXT', 'line', None, null_translator ),
102: ( 'MEMO', 'text', 0, text_translator ),
103: ( 'DATETIME', 'date', 20, date_translator ),
'time': ( 'DATETIME', 'line', 32, elapsed_time_translator ),
104: ( 'SELECTION', 'select', 32, None ),
#105: ( 'BINARY', 'word', 3, ? ),
106: ( 'STATE', 'select', 32, state_translator ),
107: ( 'USER', 'word', 32, user_translator ),
108: ( 'PROJECTS', 'line', 80, aux('PROJECTS') ),
#109: ( 'SUMMATION', '?', ?, ? ),
#110: ( 'MULTIPLE_SELECTION', '?', ?, ? ),
#111: ( 'CONTACT', 'word', 32, ? ),
112: ( 'COMPANIES', 'line', 80, aux('COMPANIES') ),
#113: ( 'INCIDENT', 'word', 32, ? ),
114: ( 'PRODUCTS', 'line', 80, aux('PRODUCTS') ),
115: ( 'SERVICEAGREEMENTS', 'line', 80, aux('SERVICEAGREEMENTS') ),
#116: ( 'FOLDER', 'word', 32, ? ),
#117: ( 'KEYWORDLIST', '?', ?, ? ),
#118: ( 'PRODUCTLIST', '?', ?, ? ),
#119: ( 'PROBLEM', 'word', 32, ? ),
#120: ( 'RESOLUTION', 'word', 32, ? ),
#121: ( 'MERCHANDISE', 'word', 32, ? ),
}
# 4.5. Read TeamTrack fields
#
# Read the TS_FIELDS table and turn it into a map from the field's database
# name (DBNAME field) to the field definition.
#
# We only want to consider fields from the TS_CASES (if there's a field in
# the TS_INCIDENTS table with the same DBNAME as a field in the TS_CASES
# table we don't want to use the former by mistake). Hence the selection
# on the TS_TABLEID field.
#
# If the TS_STATUS of a field is not 0, then it's "inactive", meaning
# deleted. We don't want to consider those either.
tt_fields = {}
for f in connection.query(teamtrack.table['FIELDS'],
'TS_TABLEID = %d AND TS_STATUS = 0'
% connection.case_table_id()):
tt_fields[f['DBNAME']] = f
try:
state_description = tt_fields['STATE']['DESCRIPTION']
owner_description = tt_fields['OWNER']['DESCRIPTION']
title_description = tt_fields['TITLE']['DESCRIPTION']
title_length = tt_fields['TITLE']['LEN']
except KeyError:
# "Couldn't get descriptions for TeamTrack system fields STATE, OWNER,
# and TITLE."
raise error, catalog.msg(402)
# 4.6. Make values for the State field in Perforce
#
# Work out the legal values of the State field in Perforce. Note that
# "closed" must be a legal state because "p4 fix -c CHANGE JOBNAME" always
# sets the State to "closed" even if "closed" is not a legal value. See
# job000225.
legal_states = map((lambda x: x[1]), state_pairs)
if 'closed' not in legal_states:
legal_states.append('closed')
state_values = string.join(legal_states, '/')
# 4.7. Fields that always appear in the Perforce jobspec
#
# The 'p4_fields' table maps TeamTrack field name to a definition of the
# corresponding field in Perforce. The table also has entries for field
# not replicated from TeamTrack: these appear under dummy TeamTrack field
# names in parentheses.
#
# Perforce field definitions have nine elements:
#
# 1. Field number;
#
# 2. Field name;
#
# 3. Field type (word, line, select, date, text);
#
# 4. Field length;
#
# 5. Field disposition (always, required, optional, default);
#
# 6. The default value for the field, or None if there isn't one (field
# Preset);
#
# 7. Legal values for the field (if it's a "select" field) or None
# otherwise (field Values);
#
# 8. Help text for the field;
#
# 9. Translator for the field (if the field is replicated from TeamTrack),
# or None (if the field is not replicated).
#
# The five fields 101 to 105 are predefined because they are required by
# Perforce. The fields Job and Date are special: they are required by
# Perforce but are not replicated from TeamTrack. Note that their help
# text is given (the other help texts will be fetched from TeamTrack).
#
# We extend this table with fields from the "replicated_fields"
# configuration parameter (section 4.8). Next we use the table to buid the
# Perforce jobspec (section 4.9). Finally, we use the table to build the
# "field_map" configuration parameter which the replicator module uses to
# replicate the field (section 4.10).
p4_fields = {
'(JOB)': ( 101, 'Job', 'word', 32, 'required',
None, None,
"The job name.",
None ),
'STATE': ( 102, 'State', 'select', 32, 'required',
state_pairs[0][1], state_values,
state_description,
state_translator ),
'OWNER': ( 103, 'Owner', 'word', 32, 'required',
'$user', None,
owner_description,
user_translator ),
'(DATE)': ( 104, 'Date', 'date', 20, 'always',
'$now', None,
"The date this job was last modified.",
None ),
'TITLE': ( 105, 'Title', 'line', title_length, 'required',
'$blank', None,
title_description,
null_translator ),
'(FILESPECS)': ( 191, 'P4DTI-filespecs', 'text', 0, 'optional',
None, None,
"Associated filespecs.",
None ),
'(RID)': ( 192, 'P4DTI-rid', 'word', 32, 'required',
'None', None,
"P4DTI replicator identifier. Do not edit!",
None ),
'(ISSUE)': ( 193, 'P4DTI-issue-id', 'word', 32, 'required',
'None', None,
"TeamTrack issue database identifier. Do not edit!",
None ),
'(USER)': ( 194, 'P4DTI-user', 'word', 32, 'always',
'$user', None,
"Last user to edit this job. You can't edit this!",
None ),
}
# 4.8. Add replicated fields to table
#
# Go through the "replicated_fields" configuration parameter. For each
# replicated field in TeamTrack, build a structure of none elements
# describing the corresponding field in Perforce (see section 4.7 for the
# definition of this structure) and add it to the p4_fields table.
#
# Replicated fields will start with the Perforce field number 110. This
# gives some opportunity for people using advanced configuration to add
# extra fields near the start of the Perforce jobspec. Since the P4DTI
# fields start at 191, this allows 81 replicated fields, which should be
# plenty.
p4_field_id = 110
for tt_field_name in config.replicated_fields:
# Convert the field name to uppercase. This allows people to specify
# field names case-insensively in the "replicated_fields" configuration
# parameter.
tt_field_name = string.upper(tt_field_name)
# The field must be a valid TeamTrack field.
if not tt_fields.has_key(tt_field_name):
# "Field '%s' specified in 'replicated_fields' list not in
# TeamTrack FIELDS table."
raise error, catalog.msg(403, tt_field_name)
# The field must not be already in the p4_fields table.
if p4_fields.has_key(tt_field_name):
if tt_field_name in ['STATE', 'OWNER', 'TITLE']:
# "Field '%s' specified in 'replicated_fields' list is a system
# field: leave it out!"
raise error, catalog.msg(404, tt_field_name)
else:
# "Field '%s' appears twice in 'replicated_fields'."
raise error, catalog.msg(405, tt_field_name)
f = tt_fields[tt_field_name]
tt_field_type = f['FLDTYPE']
# TeamTrack variable length text fields are not given as MEMO
# fields (type 102) but as TEXT fields (type 101) with
# ATTRIBUTES set to 0 (ordinary variable-length), 2 (journal) or
# 3 (read-only journal). Handle these special cases here. See
# job000370 for the problem with journal fields.
if tt_field_type == 101 and f['ATTRIBUTES'] != 1:
tt_field_type = 102
# TeamTrack date/time fields have four types: 0 (date only), 1 (date
# and time), 2 (time of day), 3 (elapsed time). The former two should
# be represented as Perforce dates and translated using
# date_translator; the latter two should be represented as text fields
# and translated using elapsed_time_translator. See job000182.
elif tt_field_type == 103 and f['ATTRIBUTES'] in [2,3]:
tt_field_type = 'time'
# Look up the field type in type_table to find out how to convert it to
# a Perforce field. See section ???.
if not type_table.has_key(tt_field_type):
# "Field '%s' has type %d: this is not supported by P4DTI."
raise error, catalog.msg(406, (tt_field_name, tt_field_type))
tt_table_name, p4_field_type, p4_field_length, trans = \
type_table[tt_field_type]
p4_field_values = None
p4_field_preset = None
p4_field_name = keyword_translator.translate_0_to_1(f['NAME'])
# "p4 -G" uses the field "code" to indicate whether the Perforce
# command succeeded or failed. See job000003.
if p4_field_name == 'code':
# "You can't have a field called 'code' in the Perforce jobspec."
raise error, catalog.msg(407)
# Fixed-length text fields get the length from TeamTrack.
if p4_field_length == None:
p4_field_length = f['LEN']
# For single-select fields, work out the values for that field in
# Perforce, and build a translator.
if tt_field_type == 104:
selections = connection.query(teamtrack.table['SELECTIONS'],
"TS_FLDID = %d" % f['ID'])
p4_field_values = string.join(map(lambda(s): keyword_translator.translate_0_to_1(s['NAME']), selections) + ['(None)'], '/')
trans = dt_teamtrack.single_select_translator(f['DBNAME'], keyword_translator)
# Build a structure describing the Perforce field; add it to the table.
p4_fields[tt_field_name] = \
( p4_field_id,
p4_field_name,
p4_field_type,
p4_field_length,
'optional',
p4_field_preset,
p4_field_values,
f['DESCRIPTION'],
trans
)
p4_field_id = p4_field_id + 1
if p4_field_id >= 191:
# "Too many fields to replicate: Perforce jobs can contain only 99
# fields."
raise error, catalog.msg(408)
# 4.9. Build a Perforce jobspec
#
# The jobspec should be suitable for passing to "p4 -G jobspec -i". This
# means that it will look like this:
#
# { 'Comments': '# Form comments...',
# 'Fields0': '101 Job word 32 required',
# 'Fields1': '102 State select 32 required',
# 'Values1': '_new/assigned/closed/verified/deferred',
# 'Presets1': '_new',
# ...
# }
jobspec = {}
# Get a list of Perforce fields, sorted by field number. Note that the
# call to the sort methdo works in Python 1.5.2, but it doesn't follow the
# manual, which says that the comparison function must return -1, 0, or 1.
p4_fields_list = p4_fields.values()
p4_fields_list.sort(lambda x,y: x[0] - y[0])
i = 0
for field in p4_fields_list:
jobspec['Fields%d' % i] = "%s %s %s %s %s" % field[0:5]
i = i + 1
i = 0
for field in p4_fields_list:
if field[6] != None:
jobspec['Values%d' % i] = "%s %s" % (field[1], field[6])
i = i + 1
i = 0
for field in p4_fields_list:
if field[5] != None:
jobspec['Presets%d' % i] = "%s %s" % (field[1], field[5])
i = i + 1
def comment(field):
return "# %s: %s\n" % (field[1], field[7])
jobspec['Comments'] = ("# DO NOT CREATE NEW JOBS IN PERFORCE. "
"USE TEAMTRACK.\n#\n" +
string.join(map(comment, p4_fields.values()), ""))
# 4.10. Generate configuration parameters
# Make a logger object that logs to standard output and to the file named
# by the log_file configuration parameter (if not None).
loggers = [logger.file_logger(priority = config.log_level)] # to stdout
if config.log_file != None:
# to log_file
loggers.append(logger.file_logger(open(config.log_file, "a"),
priority = config.log_level))
config.logger = logger.multi_logger(loggers)
# Set configuration parameters needed by dt_teamtrack.
config.start_date = convert_isodate_to_secs(config.start_date)
config.state_pairs = state_pairs
# Set configuration parameters needed by the replicator.
config.date_translator = date_translator
config.job_owner_field = 'Owner'
config.job_status_field = 'State'
config.job_date_field = 'Date'
config.text_translator = text_translator
config.user_translator = user_translator
# The field_map parameter is a list of triples (TeamTrack database field
# name, Perforce field name, translator) required by the replicator.
#
# This is generated from the p4_field table by filtering out fields that
# aren't replicated (these have no translator) and selecting only the three
# elements of interest.
config.field_map = \
map(lambda item: (item[0], item[1][1], item[1][8]),
filter(lambda item: item[1][8] != None, p4_fields.items()))
return (jobspec, config)
# A. REFERENCES
#
# [GDR 2000-10-16] "Perforce Defect Tracking Integration Integrator's Guide";
# Richard Brooksby; Ravenbrook Limited; 2000-10-16.
#
# [IG] "Perforce Defect Tracking Integration Integrator's Guide" (living
# document); Gareth Rees; Ravenbrook Limited; 2000-10-16.
#
# [ISO 8601] "Representation of dates and times"; ISO; 1988-06-15;
# .
#
# [RB 2000-11-28a] "Case of state names" (e-mail message); Richard Brooksby;
# Ravenbrook; 2000-11-28;
# .
#
# [RB 2000-11-28b] "Distinguished state to map to 'closed'" (e-mail message);
# Richard Brooksby; Ravenbrook; 2000-11-28;
# .
#
# [TeamShare 2000-01-20] "TeamTrack Database Schema (Database Version: 21)";
# TeamShare; 2000-01-20;
# .
#
#
# B. DOCUMENT HISTORY
#
# 2000-12-07 RB Updated "resolver" to "administrator" in some error messages.
# Fixed the field length on "P4DTI-issue-id".
#
# 2000-12-08 GDR Translate state name "ignore" to "_ignore".
#
# 2000-12-15 NB Added verbosity control.
#
# 2001-01-19 NB Validate config items. log_file may be None.
#
# 2001-02-04 GDR Added start_date parameter.
#
# 2001-02-12 GDR Convert start_date to seconds since epoch before passing to
# dt_teamtrack.
#
# 2001-02-13 GDR Allow administrator_address and smtp_server to be None.
#
# 2001-02-15 GDR Time fields map to text fields, use elapsed_time_translator
#
# 2001-02-16 NB Added replicate_p configuration parameter.
#
# 2001-02-19 NB Moved keyword translation to p4.py.
#
# 2001-02-22 GDR Moved keyword translation to keyword.py. Made sure that
# 'closed' is a legal job state in Perforce. Added text translator to
# replicator config.
#
# 2001-02-26 GDR Changed error name to "TeamTrack configuration error" for
# consistency with "Bugzilla configuration error" in configure_bugzilla.py.
# Refer to "TeamTrack" explicitly in messages, not just "defect tracker".
#
# 2001-03-02 RB Transferred copyright to Perforce under their license.
#
# 2001-03-12 GDR Using messages for errors.
#
# 2001-03-13 GDR Remove verbose parameter; added log_level. Removed
# P4DTI-action field. Made P4DTI-filespec field optional. Get
# keyword_translator from translator, not keyword.
#
# 2001-03-15 GDR Get configuration from the config module.
#
# 2001-03-17 GDR Formatted as document. Added links to design. Make only a
# single user translator to fix job000148.
#
# 2001-03-23 GDR Added job-date-field to replicator configuration.
#
# 2001-06-29 GDR Made portable between TeamTrack 4.5 and TeamTrack 5.0
# by using case_table_id() and case_table_name().
#
# 2001-07-24 GDR Recognize journal fields as being multi-line text
# fields.
#
# 2001-08-06 GDR Specify -1 for DST argument to mktime().
#
#
# C. COPYRIGHT AND LICENCE
#
# This file is copyright (c) 2001 Perforce Software, Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS AND CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
#
# $Id: //info.ravenbrook.com/project/p4dti/version/1.1/code/replicator/configure_teamtrack.py#5 $