# dt_bugzilla.py -- defect tracking interface (Bugzilla).
# Nicholas Barnes, Ravenbrook Limited, 2000-11-21.
# $Id: //info.ravenbrook.com/project/p4dti/version/0.4/code/replicator/dt_bugzilla.py#2 $
#
# Copyright 2000 Ravenbrook Limited.  This document is provided "as is",
# without any express or implied warranty. In no event will the authors
# be held liable for any damages arising from the use of this document.
# You may make and distribute copies and derivative works of this
# document provided that (1) you do not charge a fee for this document or
# for its distribution, and (2) you retain as they appear all copyright
# and licence notices and document history entries, and (3) you append
# descriptions of your modifications to the document history.

import replicator
import re
import socket
import string
import bugzilla
import types
import time

error = 'P4DTI Bugzilla interface error'

class bugzilla_bug(replicator.defect_tracker_issue):
    dt = None # The defect tracker this bug belongs to.
    bug = None # The dictionary representing the bugzilla bug.
    p4dti_bug = None # The dictionary representing the p4dti_bugs record.

    def __init__(self, bug, dt):
        self.dt = dt
        self.bug = bug
        self.p4dti_bug = self.dt.bugzilla.bug_p4dti_bug(bug)

    def __getitem__(self, key):
        if self.bug.has_key(key) :
            return self.bug[key]
        else :
            return self.p4dti_bug[key]

    def action(self):
        return self.p4dti_bug['action']

    def add_filespec(self, filespec):
        filespec_record = {}
        filespec_record['filespec'] = filespec
        filespec_record['bug_id'] = self.bug['bug_id']
        filespec = bugzilla_filespec(self, filespec_record)
        filespec.add()

    def add_fix(self, change, client, date, status, user):
        fix_record = {}
        fix_record['bug_id'] = self.bug['bug_id']
        fix_record['changelist'] = change
        fix_record['client'] = client
        fix_record['p4date'] = date
        fix_record['status'] = status
        fix_record['user'] = user
        fix = bugzilla_fix(self, fix_record)
        fix.add()

    def id(self):
        return str(self.bug['bug_id'])

    def filespecs(self):
        filespecs = []
        for filespec in self.dt.bugzilla.filespecs_from_bug_id(
            self.bug['bug_id']):
            filespecs.append(bugzilla_filespec(self, filespec))
        return filespecs

    def fixes(self):
        fixes = []
        for fix in self.dt.bugzilla.fixes_from_bug_id(self.bug['bug_id']) :
            fixes.append(bugzilla_fix(self, fix))
        return fixes

    def readable_name(self):
        if (self.p4dti_bug != None) and (self.p4dti_bug.has_key('jobname')):
            return self.p4dti_bug['jobname']
        else:
            return self.dt.config['jobname-function'](self.bug)

    def rid(self):
        if self.p4dti_bug == None : # not yet replicated
            return ""
        else:
            return self.p4dti_bug['rid']

    def setup_for_replication(self):
        assert self.p4dti_bug == None
        self.p4dti_bug = {}
        self.p4dti_bug['bug_id'] = self.bug['bug_id']
        self.p4dti_bug['jobname'] = self.readable_name()
        self.p4dti_bug['action'] = 'replicate'
        self.dt.bugzilla.add_p4dti_bug(self.p4dti_bug)

    # Check Bugzilla permissions.
    # In Bugzilla 2.10, permissions are checked in CheckCanChangeField()
    # in process_bug.cgi.  This is the test:
    #
    # 1. anyone can make a null change;
    # 2. anyone can make a change which just adds or removes
    #    whitespace at the beginning of a value;
    # 3. anyone can add a description record;
    # 4. anyone in the "editbugs" group can make any change;
    # 5. anyone in the "canconfirm" group can change the status to an
    #    opened status.
    # 6. anyone can change the status to an opened status if the bug has
    #    'everconfirmed' set.
    # 7. The reporter, or assigned_to, or qa_contact of a bug can make any
    #    change to the bug other than a change to an opened status.
    # 8. Nobody else can make a change.
    #
    # An opened status is NEW, REOPENED, or ASSIGNED.
    #
    # Note that there is not a check made of whether the user is in
    # the bug group of the bug.  There is an implicit check of this in
    # buglist.pl and bug_form.pl; if the user is not in the bug group,
    # the bug is not displayed.

    def opened_status(self, status):
        return ((status == 'NEW') or
                (status == 'REOPENED') or
                (status == 'ASSIGNED'))

    def can_change_field(self, user, canconfirm, key, old, new):
        # 1. null changes.
        if old == new:
            return 1
        # 2. whitespace changes.
        if (isinstance(old, types.StringType) and
            isinstance(new, types.StringType) and
            string.strip(old) == string.strip(new)):
            return 1
        # 3. we don't have description records.
        # 4. editbugs handled by check_permissions().
        if ((key == 'bug_status') and
            (self.opened_status(new))):
            # 5. canconfirm
            if canconfirm:
                return 1
            # 6 everconfirmed
            if self.bug['everconfirmed'] == 1:
                return 1
        else:
            # 7. reporter/assigned_to/qa_contact
            if ((user == self.bug['reporter']) or
                (user == self.bug['assigned_to']) or
                (user == self.bug['qa_contact'])):
                return 1
        # 8. nobody else
        return 0

    def check_permissions(self, user, changes):
        user_groupset = self.dt.user_groupset(user)
        bug_groupset = self.bug['groupset']

        assert ((bug_groupset == 0) or
                (self.dt.singleton_groupset(bug_groupset)))

        # 4. user in editbugs can make any change.
        if self.dt.groupset_has_named_group(user_groupset, 'editbugs'):
            return
        # Are we in the bug's groupset?
        # Bugzilla doesn't check this, but there is an implicit
        # check because Bugzilla won't show this bug to this user.
        if ((bug_groupset != 0) and
            (user_groupset & bug_groupset) != bug_groupset):
            raise error, ("user %d can't change"
                          " field '%s' of bug %d to %s: not in bug group."
                          % (user, key, self.bug['bug_id'], repr(newvalue)))

        canconfirm = self.dt.groupset_has_named_group(user_groupset,
                                                      'canconfirm')
        for (key, newvalue) in changes.items() :
            if not self.can_change_field(user, canconfirm,
                                         key, self.bug[key], newvalue):
                raise error, ("user %d can't change"
                              " field '%s' of bug %d to %s."
                              % (user, key, self.bug['bug_id'],
                                 repr(newvalue)))
                
                
    # Enforce Bugzilla's transition invariants, whereby bugs in
    # 'RESOLVED', 'VERIFIED', and 'CLOSED' states must have a valid
    # 'resolution' field, whereas bugs in other states must have an
    # empty 'resolution' field.

    def status_needs_resolution(self, status):
        return (status == 'RESOLVED' or
                status == 'VERIFIED' or
                status == 'CLOSED')

    def enforce_invariants(self, changes):

        assert self.bug.has_key('resolution')
        assert self.bug.has_key('bug_status')

        if (changes.has_key('resolution') and
            changes['resolution'] == 'DUPLICATE'):
            raise error, ("P4DTI does not support"
                          " marking bugs as DUPLICATE from Perforce.")

        if changes.has_key('bug_status') :

            # We are making a transition.

            if (self.status_needs_resolution(changes['bug_status']) and
                not self.status_needs_resolution(self.bug['bug_status'])) :
                # We are transitioning to a status which requires a
                # resolution from one which does not.

                if (changes.has_key('resolution') and
                    changes['resolution'] == '') :

                    # We are also clearing the resolution.  This may
                    # happen due to a timing problem; if one p4 user
                    # correctly transitions a bug to REOPENED and
                    # clears the resolution field, and then another p4
                    # user transitions the bug to RESOLVED without
                    # setting the resolution, without an intervening
                    # replication, we may end up here.

                    changes['resolution'] = 'FIXED'
                    
                if (self.bug['resolution'] == '' and
                    not changes.has_key('resolution')) :

                    # We are not setting the resolution field.  We
                    # can't force Perforce users to set the resolution
                    # field, and even if procedures require it we can
                    # still get here due to a race problem.  If it
                    # does happen, we set the resolution to FIXED.

                    changes['resolution'] = 'FIXED'

            if not self.status_needs_resolution(changes['bug_status']) :
                # We are transitioning to a status which requires
                # an empty resolution.  If we don't have an empty
                # resolution, put one in.
                if changes.has_key('resolution'):
                    if changes['resolution'] != '':
                        changes['resolution'] = ''
                else:
                    if self.bug['resolution'] != '':
                        changes['resolution'] = ''

    # after making a change to a bugs record, we have to record
    # the change in the bugs_activity table.

    def update_bugs_activity(self, user, changes):
        activity_record = {}
        activity_record['bug_id'] = self.bug['bug_id']
        activity_record['who'] = user
        activity_record['bug_when'] = self.dt.bugzilla.now()
        for (key, newvalue) in changes.items():
            oldvalue = self.bug[key]
            activity_record['fieldid'] = self.dt.fieldid(key)
            activity_record['oldvalue'] = str(oldvalue)
            activity_record['newvalue'] = str(newvalue)
            self.dt.bugzilla.add_activity(activity_record)
            
    def update(self, user, changes) :
        # should check permissions for this user to make
        # these changes here.
        changes_bug = {}
        changes_p4dti_bug = {}

        for key, value in changes.items() :
            if self.bug.has_key(key) :
                changes_bug[key] = value
            elif self.p4dti_bug.has_key(key):
                changes_p4dti_bug[key] = value
            else:
                raise error, ("updating non-existent bug field '%s'" %
                              key)

        self.enforce_invariants(changes_bug)
        self.check_permissions(user, changes_bug)
            
        self.dt.bugzilla.update_bug(changes_bug, self.bug['bug_id'])

        self.update_bugs_activity(user, changes_bug)
        # Now the bug is updated in the database, update our copy.
        for key, value in changes_bug.items() :
            self.bug[key] = value

        self.dt.bugzilla.update_p4dti_bug(changes_p4dti_bug,
                                          self.bug['bug_id'])
        # Now the p4dti_bug is updated in the database, update our copy.
        for key, value in changes_p4dti_bug.items() :
            self.p4dti_bug[key] = value

    def update_action(self, action):
        if self.p4dti_bug['action'] != action:
            self.p4dti_bug['action'] = action
            self.dt.bugzilla.update_p4dti_bug({'action' : action},
                                              self.bug['bug_id'])

class bugzilla_fix(replicator.defect_tracker_fix):
    bug = None # The Bugzilla bug to which the fix refers.
    fix = None # The dictionary representing the bugzilla fix record.

    def __init__(self, bug, dict):
        self.bug = bug
        self.fix = dict

    def __getitem__(self, key):
        return self.fix[key]

    def __repr__(self):
        return repr(self.fix)

    def __setitem__(self, key, value):
        self.fix[key] = value

    def add(self):
        self.bug.dt.bugzilla.add_fix(self.fix)

    def change(self):
        return self.fix['changelist']

    def delete(self):
        self.bug.dt.bugzilla.delete_fix(self.fix)

    def status(self):
        return self.fix['status']

    def update(self, p4_fix):
        self.transform_from_p4(p4_fix)
        self.bug.dt.bugzilla.update_fix(self.fix,
                                        self.fix['bug_id'],
                                        self.fix['changelist'])



class bugzilla_filespec(replicator.defect_tracker_filespec):
    bug = None # The Bugzilla bug to which the filespec refers.
    filespec = None # The dictionary representing the filespec record.

    def __init__(self, bug, dict):
        self.bug = bug
        self.filespec = dict

    def __getitem__(self, key):
        return self.filespec[key]

    def __repr__(self):
        return repr(self.filespec)

    def __setitem__(self, key, value):
        self.filespec[key] = value

    def add(self):
        self.bug.dt.bugzilla.add_filespec(self.filespec)

    def delete(self):
        self.bug.dt.bugzilla.delete_filespec(self.filespec)

    def name(self):
        return self.filespec['filespec']


# The dt_bugzilla class implements a generic interface between the
# replicator and the Bugzilla defect tracker.  Some configuration can
# be done by passing a configuration hash to the constructor; for more
# advanced configuration you should subclass this and replace some of
# the methods.

class dt_bugzilla(replicator.defect_tracker):
    config = { }
    
    rid = None
    sid = None
    bugzilla = None

    def __init__(self, rid, sid, config = {}):
        replicator.defect_tracker.__init__(self, rid, sid, config)
	self.bugzilla = bugzilla.bugzilla(config['db'],
	                                  config['dbms-host'],
	                                  config['dbms-port'],
	                                  config['dbms-database'],
	                                  config['dbms-user'],
	                                  config['dbms-password'],
	                                  rid,
	                                  sid)
        self.bugzilla.create_p4dti_tables()

    def all_issues(self):
        bugs = self.bugzilla.all_bugs()
        return map(lambda bug,dt=self: bugzilla_bug(bug,dt), bugs)

    def changed_entities(self):
        self.bugzilla.lock_tables()
        replication = self.bugzilla.new_replication()
        last = self.bugzilla.latest_complete_replication()
        bugs = self.bugzilla.all_bugs_since(last)
        return (
            map(lambda bug,dt=self: bugzilla_bug(bug,dt), bugs),
             { }, # changed changelists
            replication
            )

    def mark_changes_done(self, replication):
        self.bugzilla.end_replication()
        self.bugzilla.unlock_tables()

    def init(self):
        pass

    def issue(self, bug_id):
        bug = self.bugzilla.bug_from_bug_id(int(bug_id))
        return bugzilla_bug(bug, self)

    def replicate_changelist(self, change, client, date, description,
                             status, user):
        dt_changelists = self.bugzilla.changelists(change)
        if len(dt_changelists) == 0:
            # no existing changelist; make a new one
            dt_changelist={}
            self.transform_changelist(dt_changelist, change, client, date,
                                      description, status, user)
            self.bugzilla.add_changelist(dt_changelist)
            return 1
        else: # determine the changes
            changes = self.transform_changelist(dt_changelists[0],
                                                change, client, date,
                                                description, status, user)
            if changes :
                self.bugzilla.update_changelist(changes, change)
                return 1
            else:
                return 0

    def transform_changelist(self, dt_changelist,
                             change, client, date, description,
                             status, user):
        changes = {}
        changes['changelist'] = change
        changes['client'] = client
        changes['p4date'] = date
        changes['description'] = description
        changes['flags'] = (status == 'submitted')
        changes['user'] = user
        for key, value in changes.items():
            if (not dt_changelist.has_key(key) or
                dt_changelist[key] != value):
                dt_changelist[key] = value
            else:
                del changes[key]
        return changes

    # groups.
    group_by_name = {}
    group_by_bit = {}

    def init_group_tables(self):
        groups = self.bugzilla.groups()
        for group in groups:
            self.group_by_name[group['name']] = group
            self.group_by_bit[group['bit']] = group

    def groupset_has_named_group(self, groupset, group):
        if not self.group_by_name.has_key(group):
            self.init_group_tables()
        if not self.group_by_name.has_key(group):
            raise error, ("No group '%s'." % group)
        groupbit = self.group_by_name[group]['bit']
        return (groupset & groupbit) == groupbit

    def user_has_named_group(self, user, group):
        groupset = self.bugzilla.groupset_from_userid(user)
        return self.groupset_has_named_group(groupset, group)

    def user_groupset(self, user):
        return self.bugzilla.groupset_from_userid(user)

    def singleton_groupset(self, groupset):
        return ((groupset != 0) and
                (groupset & (groupset - 1)) == 0)

    # fields
    field_by_name = {}
    def init_field_tables(self):
        fielddefs = self.bugzilla.fielddefs()
        for fielddef in fielddefs:
            self.field_by_name[fielddef['name']] = fielddef

    def fieldid(self, name):
        if not self.field_by_name.has_key(name):
            self.init_field_tables()
        if not self.field_by_name.has_key(name):
            raise error, ("No field '%s'." % name)
        return self.field_by_name[name]['fieldid']
    

class status_translator(replicator.translator):
    # A map from Bugzilla status name to Perforce status name.
    status_bz_to_p4 = { }

    # A map from Perforce status name to Bugzilla status name (the reverse of
    # the above map).
    status_p4_to_bz = { }

    def __init__(self, statuses):
        # Compute the maps.
        for bz_status, p4_status in statuses:
            assert isinstance(bz_status, types.StringType)
            assert isinstance(p4_status, types.StringType)
            self.status_bz_to_p4[bz_status] = p4_status
            self.status_p4_to_bz[p4_status] = bz_status

    def translate_0_to_1(self, bz_status, bz, p4, issue = None, job = None):
        assert isinstance(bz_status, types.StringType)
        if self.status_bz_to_p4.has_key(bz_status):
            return self.status_bz_to_p4[bz_status]
        else:
            raise error, ("No Perforce status corresponding to "
                          "Bugzilla status '%s'" % bz_status)

    def translate_1_to_0(self, p4_status, bz, p4, issue = None, job = None):
        assert isinstance(p4_status, types.StringType)
        if self.status_p4_to_bz.has_key(p4_status):
            return self.status_p4_to_bz[p4_status]
        else:
            raise error, ("No Bugzilla status corresponding to "
                          "Perforce status '%s'." % p4_status)

class resolution_translator(replicator.translator):
    def translate_0_to_1(self, bz_status, bz, p4, issue = None, job = None):
        assert isinstance(bz_status, types.StringType)
        if (bz_status == ''):
            return 'NONE'
        else:
            return bz_status

    def translate_1_to_0(self, p4_status, bz, p4, issue = None, job = None):
        if (p4_status == 'NONE'):
            return ''
        else:
            return p4_status

# The date_translator class translates date fields between defect trackers
# Bugzilla (0) and Perforce (1).

# Some Perforce dates are reported in the form 2000/01/01 00:00:00 (e.g., dates
# in changeslists) and others are reported as seconds since 1970-01-01 00:00:00
# (e.g., dates in fixes).  I don't know why this is, but I have to deal with it
# by checking for both formats.

# MySQL datetime values are in the form YYYY-MM-DD hh:mm:ss

class date_translator(replicator.translator):
    p4_date_regexps = [re.compile("^([0-9][0-9][0-9][0-9])/([0-9][0-9])/([0-9][0-9]) ([0-9][0-9]):([0-9][0-9]):([0-9][0-9])$"),
                       re.compile("^[0-9]+$")]

    bz_date_regexp = re.compile("^([0-9][0-9][0-9][0-9])-([0-9][0-9])-([0-9][0-9]) ([0-9][0-9]):([0-9][0-9]):([0-9][0-9])$")

    def translate_0_to_1(self, bz_date, bz, p4, issue = None, job = None):
        assert isinstance(bz_date, types.StringType)
        assert isinstance(bz, dt_bugzilla)
        assert isinstance(p4, replicator.defect_tracker)
        assert issue == None or isinstance(issue, bugzilla_bug)
        match = self.bz_date_regexp.match(bz_date)
        if match:
            return ('%s/%s/%s %s:%s:%s' %
                    (match.group(1), match.group(2), match.group(3),
                     match.group(4), match.group(5), match.group(6)))
        
    def translate_1_to_0(self, p4_date, bz, p4, issue = None, job = None):
        assert isinstance(p4_date, types.StringType)
        assert isinstance(bz, dt_bugzilla)
        assert isinstance(p4, replicator.defect_tracker)
        assert issue == None or isinstance(issue, bugzilla_bug)
        match = self.p4_date_regexps[0].match(p4_date)
        if match:
            return ('%s-%s-%s %s:%s:%s' %
                    (match.group(1), match.group(2), match.group(3),
                     match.group(4), match.group(5), match.group(6)))
                     
        elif self.p4_date_regexps[1].match(p4_date):
            return time.strftime("%Y-%m-%d %H:%M:%S",
                                 time.gmtime(int(p4_date)))

# The text_translator class translates multi-line text fields between defect
# trackers Bugzilla (0) and Perforce (1).

class text_translator(replicator.translator):    
    # Transform Bugzilla text field contents to Perforce text field contents
    # by converting line endings.
    
    def translate_0_to_1(self, bz_string, bz, p4, issue = None, job = None):
        assert isinstance(bz_string, types.StringType)
        # Replace \r\n with \n.
        string = re.sub('\r\n', '\n', bz_string)
        # Add final newline, unless the string is empty.
        if bz_string:
            bz_string = bz_string + '\n'
        return bz_string

    # Transform Perforce text field contents to Bugzilla text field contents
    # by converting line endings.
    
    def translate_1_to_0(self, p4_string, bz, p4, issue = None, job = None):
        assert isinstance(p4_string, types.StringType)
        # Remove final newline (if any).
        if p4_string and p4_string[-1] == '\n':
            p4_string = p4_string[:-1]
        # Replace \n with \r\n.
        p4_string = re.sub('\n', '\r\n', p4_string)
        return p4_string


class user_translator(replicator.translator):

    user_bz_to_p4 = { }
    user_p4_to_bz = { }
    bz_id_to_email = { }
    bz_email_to_id = { }
    bugzilla_user = None
    p4_user = None

    def __init__(self, bugzilla_user, p4_user):
        # Get data from the database.
        self.bugzilla_user = bugzilla_user
        self.p4_user = p4_user

    # Obtain a dictionary email -> Perforce id.  (This method copied from
    # dt_teamtrack.user_translator; it probably belongs in the dt_perforce
    # class so it can be shared between user translators for various
    # defect tracking systems.)
    def p4_user_dict(self, p4) :
        p4_users = p4.p4.run("users")
        dict={}
        for user in p4_users :
            dict[user['Email']] = user['User']
        return dict

    # Deduce and record the mapping between Bugzilla userid and
    # Perforce username.
    def init_users(self, bz, p4):
        bugzilla_users = bz.bugzilla.user_id_and_email_list()
        p4_users = self.p4_user_dict(p4)
        self.user_bz_to_p4={}
        self.user_p4_to_bz={}
        self.bz_email_to_id={}
        self.bz_id_to_email={}
        for id, email in bugzilla_users :
            self.bz_email_to_id[email] = id
            self.bz_id_to_email[id] = email
            if p4_users.has_key(email) :
                p4_user = p4_users[email]
                self.user_bz_to_p4[id] = p4_user
                self.user_p4_to_bz[p4_user] = id
        # if the bugzilla P4DTI user is in the table,
        # make sure it corresponds to the P4 P4DTI user.
        if self.bz_email_to_id.has_key(self.bugzilla_user) :
            bugzilla_id = self.bz_email_to_id[self.bugzilla_user]
            # special Bugzilla user is in Bugzilla
            if self.user_bz_to_p4.has_key(bugzilla_id) :
                # special Bugzilla user has P4 counterpart
                if (self.user_bz_to_p4[bugzilla_id] !=
                    self.p4_user) :
                    raise error, ("Bugzilla P4DTI user '%s' has email address "
                                  "matching Perforce user '%s', not Perforce "
                                  "P4DTI user '%s'." %
                                  (self.bugzilla_user,
                                   self.user_bz_to_p4[bugzilla_id],
                                   self.p4_user))
            else :
                # Perforce user table doesn't have the counterpart.
                self.user_bz_to_p4[bugzilla_id] = self.p4_user
                self.user_p4_to_bz[self.p4_user] = bugzilla_id
        else :
            # special Bugzilla user not in Bugzilla
            raise error, ("Bugzilla P4DTI user '%s' "
                          "is not a known Bugzilla user." % self.bugzilla_user)
        

    def translate_1_to_0(self, p4_user, bz, p4, issue = None, job = None):
        if not self.user_p4_to_bz.has_key(p4_user):
            self.init_users(bz, p4)
        if self.user_p4_to_bz.has_key(p4_user):
            return self.user_p4_to_bz[p4_user]
        elif self.bz_email_to_id.has_key(p4_user):
            return self.bz_email_to_id[p4_user]
        else :
            return self.bz_email_to_id[self.bugzilla_user]

    def translate_0_to_1(self, bz_user, bz, p4, issue = None, job = None):
        if not self.user_bz_to_p4.has_key(bz_user):
            self.init_users(bz, p4)
        if self.user_bz_to_p4.has_key(bz_user):
            return self.user_bz_to_p4[bz_user]
        else:
            return self.bz_id_to_email[bz_user]

# B. Document History
# 
# 2000-12-05 NB Fixes for job job000089 and job000118.  We update bugs_activity
# and have a new table p4dti_bugs_activity which duplicates bugs_activity rows
# added by this replicator.  A complicated select then identifies bugs which
# have been changed other than by the replicator.  Locking added.  Fixes,
# filespecs, and changelists now work.
# 
# 2000-12-07 RB Changed call to create bugzilla object to pass explicit
# parameters (see corresponding change in bugzilla.py there).