# perforce_sample_data.py -- generate random activity in Perforce. # Gareth Rees, Ravenbrook Limited, 2000-09-25 # $Id: //info.ravenbrook.com/project/p4dti/branch/2000-11-27/auto-config/code/replicator/perforce_sample_data.py#1 $ # # Copyright 2000 Ravenbrook Limited. This document is provided "as is", # without any express or implied warranty. In no event will the authors # be held liable for any damages arising from the use of this document. # You may make and distribute copies and derivative works of this # document provided that (1) you do not charge a fee for this document or # for its distribution, and (2) you retain as they appear all copyright # and licence notices and document history entries, and (3) you append # descriptions of your modifications to the document history. import os import p4 import random import re import socket import string import time # Configuration. rid is the replicator identifier for the replicator that's # going to replicate the sample repository. p4_user is the userid for the # replicator. p4_client is the client that the replicator uses. # p4_client_root is the root directory of the sample repository. rid = 'case' p4_user = 'P4DTI-%s' % rid p4_client = 'p4dti-%s' % socket.gethostname() p4_client_root = 'd:\\p4dti-test' # p4_run(...). This is just a wrapper for p4.run that provides deaults for the # input, user and client arguments. See the p4 module for details. def p4_run(arguments, input=None, user=p4_user, client=p4_client): return p4.run(arguments, input=input, user=user, client=client) # The structure of the sample repository, represented as a list whose # first element is a directory name, and the remaining elements are the # entries in that directory (which may be other directories). repository = [ '//depot/', [ 'project/', [ 'compiler/', [ 'doc/', 'goals', 'requirements', 'plan', 'design' ], [ 'src/', 'assembler.c', 'assembler.h', 'compiler.h', 'lexer.c', 'lexer.h', 'main.c', 'parser.c', 'parser.h', 'register.c', 'register.h' ], [ 'build/', 'cc' ] ], [ 'editor/', [ 'doc/', 'goals', 'requirements', 'plan', 'design' ], [ 'src/', 'input.c', 'input.h', 'output.c', 'output.h', 'buffer.c', 'buffer.h', 'commands.c', 'commands.h', 'main.c' ], [ 'build/', 'vi' ] ] ] ] # Transform the repository structure so that all filespecs include the full # path. def expand_filespecs(parent, self): self[0] = parent + self[0] for i in range(len(self) - 1): if (type(self[i+1]) == type([])): expand_filespecs(self[0], self[i+1]) else: self[i+1] = self[0] + self[i+1] expand_filespecs('', repository) # choose_k(choices, k). Choose k of the choices randomly and return them as a # list. def choose_k(choices, k): n = len(choices) i = 0 results = [] while i < n and k > 0: if (random.uniform(0,1) < float(k)/float(n-i)): results.append(choices[i]) k = k - 1 i = i + 1 return results # random_filespecs(min_level). Returns a random list of filespecs from the # repository. The distribution is skewed so that generally sensible results # are returned. def random_filespecs(min_level = 0): # The deepest directory in the repository (0 is the depot). max_level = 3 # Distribution of lengths of paths in returned filespecs (less 1). levels = [0,1,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,3] # Distribution of number of filespecs to return. n_filespecs = [0,0,0,0,1,1,1,1,2,2,3,4,5] # How many filespecs to return? k = random.choice(n_filespecs) # Pick a level at which to choose a directory. if (k > 2): level = max_level else: level = random.choice(levels) if level < min_level: level = min_level # Pick a directory from which to choose the filespecs. dir = repository for i in range(level): dir = random.choice(dir[1:]) # Now choose k out of n filespecs from the selected directory. filespecs = choose_k(dir[1:], k) if level < max_level: filespecs = map(lambda (f): f[0] + '...', filespecs) return filespecs # add_file(filespec). If the file doesn't exist, create it and add it to the # Perforce repository (but don't submit the addition). Return the number of # files added to Perforce. def add_file(filespec): name = string.split(p4_run('where ' + filespec)[0]['data'])[2] if not os.access(name, 0555): stream = open(name, 'w') stream.write('Created.\n') stream.close() p4_run('add ' + name) print 'Added', name return 1 else: return 0 # add_directory(dir). Add files and directories to Perforce. The dir argument # should be a repository structure (see above). If the directory specified by # the first argument doesn't exist, create it. Then recurse to add the # subdirectories or files. Don't submit the additions. Return the number of # files added to Perforce. def add_directory(dir): if dir[0] == '//depot/': name = p4_client_root else: # The [:-1] is necessary because p4 where doesn't understand directory # names ending in slashes. name = string.split(p4_run('where ' + dir[0][:-1])[0]['data'])[2] if not os.access(name, 0777): os.mkdir(name) print 'Created directory', name n_added = 0 for i in dir[1:]: if type(i) == type(''): n_added = n_added + add_file(i) else: n_added = n_added + add_directory(i) return n_added # init_repository(). Adds all the files in the 'repository' structure to the # sample Perforce repository. Submit the additions. def init_repository(): if add_directory(repository): change = p4_run('change -o')[0] change['Jobs'] = '' change['Description'] = 'Files automatically added to sample repository' p4_run('submit -i', [change]) # users. A list of structures describing the sample TeamTrack users (from the # TeamShare demo). The 'name' key is their user name in both TeamTrack and # Perforce. The 'id' key is their userid in TeamTrack. The 'client' key is # the Perforce client they use to edit files in the sample repository. users = [ { 'name': 'chad', 'id': 3, 'client': 'chad-larch' }, { 'name': 'chris', 'id': 10, 'client': 'chris-elm' }, { 'name': 'jill', 'id': 14, 'client': 'jill-pine' }, { 'name': 'joe', 'id': 2, 'client': 'joe-beech' }, { 'name': 'newton', 'id': 11, 'client': 'newton-lime' }, { 'name': 'pat', 'id': 13, 'client': 'pat-holly' }, { 'name': 'sarah', 'id': 12, 'client': 'sarah-willow' }, ] # make_client(user, clientname). Make a Perforce client (unless it already # exists) belonging to the given user and with the given name. Its root # directory is given by p4_client_root, plus the user name. def make_client(user, clientname): # Get the list of clients. clients = p4_run('clients') for c in clients: if c['client'] == clientname: # It already exists. Don't change it. return client = p4_run('client -o', user=user, client=clientname)[0] client['Root'] = '%s/%s' % (p4_client_root, user) client['View0'] = '//depot/... //%s/...' % clientname p4_run('client -i', [client], user=user, client=clientname) print 'Added client', clientname, 'for user', user # init_clients(). Create Perforce clients for the replicator (p4_user, # p4_client) and the sample users specified in the users data structure (if the # clients don't already exist). def init_clients(): make_client(p4_user, p4_client) for u in users: make_client(u['name'], u['client']) # Status values for Perforce jobs in the sample repository. statuses = [ 'new', 'assigned', 'deferred', 'resolved', 'closed' ] # make_change(jobs). Make a change in the sample repository by picking some # files, editing them, submitting the edits, and making fix records associating # the change with some jobs. The jobs argument should be a list of jobs for # which fix records can be added. The Status field in the fixed jobs is # changed to reflect the new status (but the other fields are not changed and # may be out of date). def make_change(all_jobs): # User who will make the change. user = random.choice(users) # Sync their workspace. Note that sync returns an error if everything is # up to date. try: p4_run('sync', user=user['name'], client=user['client']) except p4.error: 0 # no action # Files affected by the change. filespecs = [] while not filespecs: filespecs = random_filespecs(min_level = 3) # Jobs fixed by the change. k = random.choice([0,1,1,1,1,1,1,2,2,3]) jobs = choose_k(all_jobs, k) # Edit the files. for f in filespecs: p4_run('edit ' + f, user=user['name'], client=user['client']) where = p4_run('where ' + f, user=user['name'], client=user['client']) filename = string.split(where[0]['data'])[2] assert(os.access(filename, 0777)) stream = open(filename, 'a') now = time.asctime(time.localtime(time.time())) stream.write('Edited at %s by %s\n' % (now, user['name'])) stream.close() print filename, 'edited at', now, 'by', user['name'] # Submit the change (don't do the fixes at this stage since the Perforce # job interface doesn't allow you to specify the status). change = p4_run('change -o', user=user['name'], client=user['client'])[0] change['Jobs'] = '' change['Description'] = ('Automatically generated change comment.\n' 'Edited files: ' + str(filespecs)) results = p4_run('submit -i', [change], user=user['name'], client=user['client']) # Work out the change number from the message returned by Perforce, which # should be the last element in the results array. match = re.compile('Change ([0-9]+) submitted\\.$').match(results[-1]['data']) assert(match) change_number = int(match.group(1)) # Now make the fix records. for job in jobs: # What's the current status of the job? status = statuses.index(job['Status']) # Change the status keyword by moving it forward in the statuses array # with probability 90%. if (status == 0 or (status < len(statuses) - 1 and random.uniform(0,1) < 0.9)): status = status + 1 else: status = status - 1 # Fix the job. p4_run('fix -s %s -c %d %s' % (statuses[status], change_number, job['Job']), user=user['name'], client=user['client']) # Record the changed status so that we don't have to query Perforce # again. job['Status'] = statuses[status] def run(n_changes = 10): init_clients() init_repository() # Get the set of jobs replicated by the replicator with identifier rid. jobs = p4_run('jobs -e P4DTI-rid=%s' % rid) # Make some changes that fix those jobs. for _ in range(n_changes): make_change(jobs)