909 lines
34 KiB
Ruby
Executable File
909 lines
34 KiB
Ruby
Executable File
#
|
|
# File:: codetest_util.rb
|
|
# Description:: Multipurpose script for use with codetest stats.
|
|
# - Publishes binaries and modifications.xml in perforce - for use by codebuilder.
|
|
# - Also is used to copy these published files.
|
|
# - Also is used to sync data
|
|
# - Also handles aggregating modifications.
|
|
# - Also checks modifications to check if we skip a build.
|
|
# - Also checks executable to see how far it is behind the head.
|
|
# - this should really be split into several files - but I hate to maintain too many, hence a helper script for all related.
|
|
#
|
|
# Author:: Derek Ward <derek.ward@rockstarnorth.com>
|
|
# Date:: 23rd August 2011
|
|
#
|
|
# Passed in :- see OPTIONS ...
|
|
# Passed out :- stderr contains all errors
|
|
# stdout for all other output.
|
|
# Returns :- returns non zero upon detecting any errors
|
|
|
|
#-----------------------------------------------------------------------------
|
|
# Uses / Requires
|
|
#-----------------------------------------------------------------------------
|
|
require 'pipeline/config/projects'
|
|
require 'pipeline/os/getopt'
|
|
require 'pipeline/os/file'
|
|
require 'systemu'
|
|
require 'rexml/document'
|
|
require 'fileutils'
|
|
|
|
include Pipeline
|
|
|
|
#-----------------------------------------------------------------------------
|
|
# Constants
|
|
#-----------------------------------------------------------------------------
|
|
OPTIONS = [
|
|
[ "--help", "-h", OS::Getopt::BOOLEAN, "display usage information." ],
|
|
[ '--filespec', '-f', OS::Getopt::REQUIRED, 'the filespec of the files to publish' ],
|
|
[ '--publish_folder', '-pf', OS::Getopt::REQUIRED, 'the folder the files are published to' ],
|
|
[ '--enable_checkin', '-c', OS::Getopt::OPTIONAL, 'enable checkin - default is off' ],
|
|
[ '--copy_dir', '-cd', OS::Getopt::OPTIONAL, 'the directory to copy everything to' ],
|
|
[ '--sync_data', '-sd', OS::Getopt::BOOLEAN, 'sync to the data of the modifications file source code commits' ],
|
|
[ '--build_shaders', '-sh', OS::Getopt::BOOLEAN, 'sync to the shaders and build them' ],
|
|
[ '--aggregate', '-a', OS::Getopt::BOOLEAN, 'aggregate modifications that were skipped to the current revision we are on' ],
|
|
[ '--check_skip', '-cs', OS::Getopt::OPTIONAL, 'the number of tests from the head in which we would want to skip to the head' ],
|
|
[ '--check_exe', '-ce', OS::Getopt::BOOLEAN, 'tests the exe to see if it was in the CLs processed otherwsie we have nothing to test' ],
|
|
[ '--check_capture', '-cc', OS::Getopt::BOOLEAN, 'tests the capture to see if it was in the CLs processed otherwsie we have nothing to do' ],
|
|
[ '--modifications_file','-m', OS::Getopt::OPTIONAL, 'the modifications file' ],
|
|
[ '--exe', '-e', OS::Getopt::OPTIONAL, 'the filename of the executable' ],
|
|
[ '--capture_file', '-cf', OS::Getopt::OPTIONAL, 'the filename of the capture file' ],
|
|
]
|
|
|
|
INFO = "[colourise=black]INFO_MSG: "
|
|
INFO_BLUE = "[colourise=blue]INFO_MSG: "
|
|
MSG_PREFIX_EMAIL = "[colourise=blue]INFO_EMA: "
|
|
MSG_PREFIX_WEB = "[colourise=blue]INFO_WEB: "
|
|
INFO_GREEN = "[colourise=green]INFO_MSG: "
|
|
INFO_GREY = "[colourise=grey]INFO_MSG: "
|
|
INFO_ORANGE = "[colourise=orange]INFO_MSG: "
|
|
MSG_PREFIX = "#{INFO_GREEN} CodeTestUtil:"
|
|
MSG_PREFIX_PERSIST = "#{INFO_GREEN} CodeTestUtil:"
|
|
MSG_PREFIX_COMMANDLINE = "#{INFO_GREY}"
|
|
|
|
PUBLISH_FOLDER = "cruisecontrol_builds/codebuilder" # the folder for publishing the files to
|
|
REVISION_LIMIT = 32 # the number of revisions to store in p4
|
|
INVALID_EXTENTIONS = ["pdb","xdb","idb","ib_pdb_index"] # extensions of files that will not be published - done this way so that new filetype make it through as they are created.
|
|
VALID_BUILDS = ["psn_beta","psn_bankrelease","psn_release"] # hardcoding the builds that are permitted to run this script.
|
|
MODIFICATIONS_FILE = "modifications.xml"
|
|
AGGREGATE_MODIFICATIONS_FILE = "aggregate_modifications.xml"
|
|
LAST_MODIFICATIONS_FILE_REVISION_FILENAME = "last_build.txt"
|
|
PUBLISH_CL_COMMENT = "Automatically created by codetest_util.rb."
|
|
|
|
|
|
# FYI - the modifications.xml 'schema'
|
|
#
|
|
#<!-- Start of the group of modifications (even if just one). -->
|
|
#<ArrayOfModification xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema">
|
|
# <!-- Start of one modification. -->
|
|
# <Modification>
|
|
# <!-- The change number. -->--filename=
|
|
# <ChangeNumber>... value ...</ChangeNumber>
|
|
# <!-- The comment. -->
|
|
# <Comment>... value ...</Comment>
|
|
# <!-- The user's email address. -->
|
|
# <EmailAddress>... value ...</EmailAddress>
|
|
# <!-- The affected file name. -->
|
|
# <FileName>... value ...</FileName>
|
|
# <!-- The affect file's folder name. -->
|
|
# <FolderName>... value ...</FolderName>
|
|
# <!-- The change timestamp, in yyyy-mm-ddThh:mm:ss.nnnn-hhmm format -->
|
|
# <ModifiedTime>... value ...</ModifiedTime>
|
|
# <!-- The operation type. -->
|
|
# <Type>... value ...</Type>
|
|
# <!-- The user name. -->
|
|
# <UserName>... value ...</UserName>
|
|
# <!-- The related URL. -->
|
|
# <Url>... value ...</Url>
|
|
# <!-- The file version. -->
|
|
# <Version>... value ...</Version>
|
|
# <!-- End of modification. -->
|
|
# </Modification>
|
|
# <!-- End of the group of modifications. -->
|
|
#</ArrayOfModification>
|
|
#Pasted from <http://confluence.public.thoughtworks.org/display/CCNET/Modification+Writer+Task>
|
|
|
|
|
|
#=============================================================================================
|
|
#
|
|
# Copys dir published files to execution build folder
|
|
#
|
|
class CopyDir
|
|
|
|
#
|
|
# constructor
|
|
#
|
|
def initialize()
|
|
|
|
end
|
|
|
|
@@log = nil
|
|
def CopyDir.log
|
|
|
|
@@log = Log.new( 'copy_dir' ) if @@log == nil
|
|
@@log
|
|
end
|
|
|
|
#---------------------------------------------------------------------------------------------------------
|
|
#
|
|
# take the published folder we have synced to and ensure binaries are copied over in the place where they will be run from.
|
|
#
|
|
def copy_dir( p4, publish_folder, copy_dir )
|
|
# ---- copy files over ----
|
|
src = OS::Path::combine(publish_folder,"*.*")
|
|
files = OS::FindEx.find_files(src)
|
|
|
|
CopyDir::log.info "#{MSG_PREFIX}Copying #{files.length} files"
|
|
files.each do |file|
|
|
src = file
|
|
dst = copy_dir
|
|
|
|
CopyDir::log.info "#{MSG_PREFIX} Copy #{src} -> #{dst}"
|
|
FileUtils.cp src, dst
|
|
end
|
|
|
|
CopyDir::log.info "#{MSG_PREFIX} Copied published build to the build directory, ready for use."
|
|
end
|
|
|
|
end # end class CopyDir
|
|
|
|
|
|
#=============================================================================================
|
|
#
|
|
# Syncs data to time of source code commits
|
|
#
|
|
class SyncData
|
|
|
|
#
|
|
# constructor
|
|
#
|
|
def initialize()
|
|
|
|
end
|
|
|
|
@@log = nil
|
|
def SyncData.log
|
|
|
|
@@log = Log.new( 'sync_data' ) if @@log == nil
|
|
@@log
|
|
end
|
|
|
|
#---------------------------------------------------------------------------------------------------------
|
|
#
|
|
# take the published folder we have synced to and ensure binaries are copied over in the place where they will be run from.
|
|
# - also sync to shaders of this change and build these shaders
|
|
#
|
|
def sync_data( p4, publish_folder, build_shaders )
|
|
# ---- sync to data ----
|
|
# we read the modifications file in order that we understand the time of the last commit
|
|
# they we get p4 to sync to the build data of that time
|
|
|
|
mods_filename = OS::Path::combine(publish_folder,MODIFICATIONS_FILE)
|
|
doc = REXML::Document.new(File.new(mods_filename))
|
|
mods = doc.elements.to_a( "//ArrayOfModification/Modification" )
|
|
|
|
# a funky Xpath expression would rid me of this code...
|
|
# we don't want to sync to the data of the binary checkins!
|
|
# rather, we want to sync to the data of the original source code checkins
|
|
changes = []
|
|
mods.each do |mod|
|
|
change_num = mod.elements["ChangeNumber"]
|
|
modification_is_publish_of_binaries = false
|
|
if ( mod.elements["Comment"] and
|
|
mod.elements["Comment"].text and
|
|
mod.elements["Comment"].text.include?PUBLISH_CL_COMMENT)
|
|
modification_is_publish_of_binaries = true
|
|
SyncData::log.info "#{MSG_PREFIX} Ignoring CL #{mod.elements['ChangeNumber'].text} it was the binary checkin and should not be used to sync data to."
|
|
end
|
|
changes << change_num unless modification_is_publish_of_binaries
|
|
end
|
|
|
|
SyncData::log.info "#{MSG_PREFIX_PERSIST} read #{changes.uniq.length} changes in order to derive what we synced to."
|
|
# get the time of this change from perforce
|
|
change_numbers = []
|
|
changes.each do |change|
|
|
SyncData::log.debug "#{MSG_PREFIX} #{change.text}"
|
|
num = change.text.to_i
|
|
change_numbers << num
|
|
end
|
|
|
|
newest_change_number = change_numbers.sort[0]
|
|
SyncData::log.info "#{MSG_PREFIX_PERSIST} #{newest_change_number} is the latest CL of all changes read."
|
|
|
|
#---------------------------------------------------------------------------------------
|
|
build_folder = ENV["RS_BUILDBRANCH"]
|
|
|
|
#build_folder = p4.local2depot( build_folder )
|
|
|
|
sync_cmd = "#{build_folder}...@#{newest_change_number}"
|
|
SyncData::log.info "#{MSG_PREFIX_PERSIST} Syncing to Data : p4 sync #{sync_cmd}"
|
|
|
|
out = p4.run_sync("#{build_folder}/...@#{newest_change_number}")
|
|
SyncData::log.info "#{MSG_PREFIX_PERSIST} The sync of data synced : #{out.length} modifications."
|
|
out.each { |o| SyncData::log.info " Synced : #{MSG_PREFIX_PERSIST} #{o["depotFile"]}\##{o['rev']}" }
|
|
|
|
|
|
if (build_shaders)
|
|
|
|
#---------------------------------------------------------------------------------------
|
|
# shaders : treated like data : but are code really.
|
|
rage_shader_lib = ENV["RAGE_DIR"] + "\\base\\src"
|
|
|
|
rage_shader_lib = p4.local2depot( rage_shader_lib )
|
|
|
|
sync_cmd = "#{rage_shader_lib}/...@#{newest_change_number}"
|
|
SyncData::log.info "#{MSG_PREFIX_PERSIST} Syncing to Shaders : p4 sync #{sync_cmd}"
|
|
|
|
out = p4.run_sync("#{rage_shader_lib}...@#{newest_change_number}")
|
|
SyncData::log.info "#{MSG_PREFIX_PERSIST} The sync of data synced : #{out.length} modifications."
|
|
out.each { |o| SyncData::log.info " Synced : #{MSG_PREFIX_PERSIST} #{o["depotFile"]}\##{o['rev']}" }
|
|
|
|
#---------------------------------------------------------------------------------------
|
|
shader_src = ENV["RS_CODEBRANCH"] + "\\game"
|
|
|
|
shader_src = p4.local2depot( shader_src )
|
|
|
|
sync_cmd = "#{shader_src}/...@#{newest_change_number}"
|
|
SyncData::log.info "#{MSG_PREFIX_PERSIST} Syncing to Shaders : p4 sync #{sync_cmd}"
|
|
|
|
out = p4.run_sync("#{shader_src}...@#{newest_change_number}")
|
|
SyncData::log.info "#{MSG_PREFIX_PERSIST} The sync of data synced : #{out.length} modifications."
|
|
out.each { |o| SyncData::log.info " Synced : #{MSG_PREFIX_PERSIST} #{o["depotFile"]}\##{o['rev']}" }
|
|
|
|
#----------------------------------------------------------------------------------------
|
|
# now build shaders
|
|
# cmd = ENV["RS_CODEBRANCH"] + "\\game\\shader_source\\VS_Project\\batch\\rsm_build_psn.bat"
|
|
# SyncData::log.info "#{MSG_PREFIX_PERSIST} (rebuild shaders) Running #{cmd}"
|
|
# status, stdout, stderr = systemu(cmd)
|
|
# ignore any errors for now... this isnt a shader compiler!
|
|
end
|
|
|
|
end
|
|
|
|
end # end class SyncData
|
|
|
|
#=============================================================================================
|
|
#
|
|
# Publishes binaries in p4
|
|
#
|
|
class PublishFiles
|
|
|
|
|
|
@@log = nil
|
|
def PublishFiles.log
|
|
|
|
@@log = Log.new( 'sync_data' ) if @@log == nil
|
|
@@log
|
|
end
|
|
|
|
|
|
#
|
|
# constructor
|
|
#
|
|
def initialize()
|
|
end
|
|
|
|
#---------------------------------------------------------------------------------------------------------
|
|
#
|
|
# Control loigc to publish binaries in p4.
|
|
#
|
|
def publish_files(p4, publish_folder, filespec, enable_checkin)
|
|
|
|
PublishFiles.log.info"#{MSG_PREFIX_PERSIST} This build will submit its modifications.xml and executables into #{publish_folder}. ( build will take longer )"
|
|
PublishFiles.log.info"#{MSG_PREFIX} publish_folder = #{publish_folder}"
|
|
PublishFiles.log.info"#{MSG_PREFIX} filespec = #{filespec}"
|
|
PublishFiles.log.info"#{MSG_PREFIX} enable_checkin = #{enable_checkin}"
|
|
|
|
# ---- find files ----
|
|
files = find_files(filespec)
|
|
|
|
# ---- create a p4 CL ----
|
|
change_id = create_cl(p4)
|
|
|
|
# ---- copy files to publish folder ----
|
|
copy_files_to_publish_folder(files, p4, publish_folder, change_id)
|
|
|
|
# ---- revert unchanged files ----
|
|
files_in_cl = revert_unchanged(p4, change_id)
|
|
|
|
# ---- checkin ----
|
|
submit(p4, files_in_cl, change_id, enable_checkin)
|
|
end
|
|
|
|
|
|
#---------------------------------------------------------------------------------------------------------
|
|
#
|
|
# create a cl
|
|
#
|
|
def create_cl(p4)
|
|
PublishFiles.log.info"#{MSG_PREFIX} Create p4 CL"
|
|
|
|
change_id = p4.create_changelist( "Automatically created by codetest_util.rb\n" )
|
|
raise Exception if change_id.nil?
|
|
PublishFiles.log.info"#{MSG_PREFIX} Created CL #{change_id}"
|
|
change_id
|
|
end
|
|
|
|
#---------------------------------------------------------------------------------------------------------
|
|
#
|
|
# copy published files in the publishing folder.
|
|
#
|
|
def copy_files_to_publish_folder(files, p4, publish_folder, change_id)
|
|
FileUtils::mkdir_p(publish_folder)
|
|
|
|
PublishFiles.log.info"#{MSG_PREFIX} Copy files to publish folder and edit or add to CL"
|
|
files.each do |file|
|
|
src = file
|
|
dst = publish_folder
|
|
filename_dst = OS::Path::combine(publish_folder, OS::Path::get_filename(src))
|
|
|
|
puts "#{MSG_PREFIX}sync and edit #{filename_dst}"
|
|
p4.run_sync(filename_dst)
|
|
p4.run_edit( '-c', change_id.to_s, filename_dst ) if File.exist?(filename_dst)
|
|
|
|
puts "#{MSG_PREFIX}Copying #{src} -> #{dst}"
|
|
FileUtils.cp src, dst
|
|
|
|
puts "#{MSG_PREFIX} Add or Edit #{filename_dst}"
|
|
p4.run_edit_or_add( '-c', change_id.to_s, filename_dst )
|
|
|
|
fstat = p4.run_fstat( filename_dst ).shift
|
|
basetype, modifiers = fstat['type'].split( '+' )
|
|
|
|
p4.run_reopen( '-c', change_id.to_s, filename_dst )
|
|
end
|
|
end
|
|
|
|
#---------------------------------------------------------------------------------------------------------
|
|
#
|
|
# revert unchanged files in changelist
|
|
#
|
|
def revert_unchanged(p4, change_id)
|
|
PublishFiles.log.info"#{MSG_PREFIX}Reverting unchanged files #{change_id}"
|
|
p4.run_revert( '-a', '-c', change_id.to_s, '//...')
|
|
|
|
files_in_cl = p4.run_opened( '-c', change_id.to_s )
|
|
raise Exception if files_in_cl.nil?
|
|
|
|
PublishFiles.log.info"#{MSG_PREFIX}There are #{files_in_cl.size} files to submit in CL #{change_id}"
|
|
files_in_cl.each do |file|
|
|
PublishFiles.log.info"#{MSG_PREFIX}#{file['depotFile']} has been updated and will be submitted."
|
|
end
|
|
|
|
files_in_cl
|
|
end
|
|
|
|
#---------------------------------------------------------------------------------------------------------
|
|
#
|
|
# submit changes
|
|
#
|
|
def submit(p4, files_in_cl, change_id, enable_checkin )
|
|
if ( enable_checkin )
|
|
if ( files_in_cl.size > 0 )
|
|
PublishFiles.log.info"#{MSG_PREFIX}Submitting file currently in #{change_id}"
|
|
submit_result = p4.run_submit( '-c', change_id.to_s )
|
|
PublishFiles.log.info"#{MSG_PREFIX} Submit result : #{submit_result.to_s}"
|
|
|
|
# revert files in case they became locked
|
|
files_in_cl.each do |file_in_cl|
|
|
p4.run_revert(file_in_cl)
|
|
fstat = p4.run_fstat( file_in_cl ).shift
|
|
PublishFiles.log.info"#{MSG_PREFIX} FSTAT: #{fstat}"
|
|
end
|
|
|
|
elsif ( 0 == files_in_cl.size )
|
|
PublishFiles.log.info"#{MSG_PREFIX}Deleting #{change_id} no files changed."
|
|
p4.run_change('-d', change_id.to_s)
|
|
end
|
|
else
|
|
PublishFiles.log.info"#{MSG_PREFIX}Checkin is disabled the CL is pending."
|
|
end
|
|
end
|
|
|
|
#---------------------------------------------------------------------------------------------------------
|
|
#
|
|
# search for files as per wildcard passed in.
|
|
#
|
|
def find_files(filespec)
|
|
|
|
PublishFiles.log.info"#{MSG_PREFIX} Finding files in #{filespec}"
|
|
|
|
files = OS::FindEx.find_files(filespec)
|
|
|
|
if files.size == 0
|
|
PublishFiles.log.warn("Warning: No files found")
|
|
Process.exit!( 1 )
|
|
else
|
|
PublishFiles.log.info"#{MSG_PREFIX} pruning files"
|
|
files.delete_if do |file|
|
|
ext = OS::Path::get_extension(file).downcase
|
|
prune = INVALID_EXTENTIONS.include?(ext)
|
|
puts "#{MSG_PREFIX} pruned #{file}" if prune
|
|
prune
|
|
end
|
|
|
|
files.each do |file|
|
|
PublishFiles.log.info"#{MSG_PREFIX} Source file: #{file}"
|
|
end
|
|
end
|
|
|
|
files
|
|
end
|
|
|
|
end # end class PublishFiles
|
|
|
|
|
|
|
|
|
|
|
|
|
|
#=============================================================================================
|
|
#
|
|
# Builds an aggregate of modifications, based upon what revision of modifications we have
|
|
# and what modifications we processed last.We collect them all in order to indicate we
|
|
# are testing all these modifications. This means we have to retrieve older revisions of
|
|
# modifications from perforce.
|
|
#
|
|
class AggregateMods
|
|
|
|
#
|
|
# constructor
|
|
#
|
|
def initialize()
|
|
end
|
|
|
|
@@log = nil
|
|
def AggregateMods.log
|
|
|
|
@@log = Log.new( 'aggregate_mods' ) if @@log == nil
|
|
@@log
|
|
end
|
|
|
|
#---------------------------------------------------------------------------------------------------------
|
|
#
|
|
# build an aggregate modifications list in a file for a range of modifications...
|
|
#
|
|
def aggregate_mods(p4, publish_folder, have_modifications_revision, last_modifications_revision, last_mods_revision_filename)
|
|
|
|
mods_filename = OS::Path::combine(publish_folder,MODIFICATIONS_FILE)
|
|
modifications_cumulative = []
|
|
( (last_modifications_revision+1)..(have_modifications_revision-1) ).each do |rev|
|
|
|
|
AggregateMods::log.info "#{MSG_PREFIX} Syncing #{mods_filename}##{rev}"
|
|
p4.run_sync("#{mods_filename}##{rev}")
|
|
# the above CAN fail! since we have limited revisions stored of these files
|
|
if File.exist?(mods_filename)
|
|
File.open(mods_filename) do |file|
|
|
temp_doc = REXML::Document.new(file)
|
|
AggregateMods::log.info "#{MSG_PREFIX} Appending modifications"
|
|
modifications = temp_doc.elements.to_a( "//ArrayOfModification/Modification" )
|
|
|
|
# skip cumuluated binaries
|
|
modifications.each do |mod|
|
|
|
|
modification_is_publish_of_binaries = false
|
|
if ( mod.elements["Comment"] and
|
|
mod.elements["Comment"].text and
|
|
mod.elements["Comment"].text.include?PUBLISH_CL_COMMENT)
|
|
modification_is_publish_of_binaries = true
|
|
end
|
|
|
|
if ( modification_is_publish_of_binaries and not rev.to_i == have_modifications_revision.to_i)
|
|
AggregateMods::log.info "#{MSG_PREFIX} Not cumulating CL #{mod.elements['ChangeNumber'].text}"
|
|
else
|
|
modifications_cumulative << mod
|
|
end
|
|
end
|
|
|
|
temp_doc = nil
|
|
end
|
|
else
|
|
AggregateMods::log.warn "#{MSG_PREFIX} Warning: Sync to revision #{rev} of #{mods_filename} did not get a file - this file has likely been purged from p4?"
|
|
end
|
|
end
|
|
|
|
AggregateMods::log.info "#{MSG_PREFIX} #{mods_filename}##{have_modifications_revision}"
|
|
|
|
p4.run_sync("#{mods_filename}##{have_modifications_revision}")
|
|
if File.exist?(mods_filename)
|
|
doc = REXML::Document.new(File.new(mods_filename))
|
|
|
|
modifications_cumulative.each do |mods|
|
|
doc.elements["ArrayOfModification"] << mods
|
|
end
|
|
|
|
aggregate_filename = OS::Path::combine(publish_folder,AGGREGATE_MODIFICATIONS_FILE)
|
|
|
|
AggregateMods::log.info "#{MSG_PREFIX} Writing aggregate_filename #{aggregate_filename}"
|
|
file = File.open( aggregate_filename,"w+" )
|
|
file << doc
|
|
file.close
|
|
else
|
|
AggregateMods::log.error "#{MSG_PREFIX} Error: Didn't get #{mods_filename}##{have_modifications_revision}"
|
|
end
|
|
|
|
# write out the version we processed
|
|
AggregateMods::log.info "#{MSG_PREFIX} Writing last_mods_revision_filename #{last_mods_revision_filename} with revision #{have_modifications_revision}"
|
|
File.open(last_mods_revision_filename, "w+") { |file| file.write(have_modifications_revision.to_s) }
|
|
end
|
|
|
|
end # end class AggregateModifications
|
|
|
|
|
|
#=============================================================================================
|
|
#
|
|
# Check if we processed a build with an exe in it otherwise what is the point in performing
|
|
# a test, we should not be needlessly be testing the same exe again.
|
|
#
|
|
class CheckExe
|
|
|
|
#
|
|
# constructor
|
|
#
|
|
def initialize()
|
|
end
|
|
|
|
@@log = nil
|
|
def CheckExe.log
|
|
|
|
@@log = Log.new( 'check_exe' ) if @@log == nil
|
|
@@log
|
|
end
|
|
|
|
#---------------------------------------------------------------------------------------------------------
|
|
#
|
|
# Control logic
|
|
#
|
|
def check_exe(p4, modifications_file, exe)
|
|
|
|
CheckExe::log.info "#{MSG_PREFIX_PERSIST} Checking if the exe was in the modifications list - otherwise we have nothing to test"
|
|
CheckExe::log.info "#{MSG_PREFIX} modifications_file = #{modifications_file}"
|
|
CheckExe::log.info "#{MSG_PREFIX} exe = #{exe}"
|
|
|
|
doc = REXML::Document.new(File.new(modifications_file))
|
|
mods = doc.elements.to_a( "//ArrayOfModification/Modification" )
|
|
|
|
CheckExe::log.info "#{MSG_PREFIX} The modification filenames are :-"
|
|
found_exe = false
|
|
mods.each do |mod|
|
|
filename = mod.elements["filename"]
|
|
filename = mod.elements["FileName"] unless filename
|
|
|
|
if not filename
|
|
CheckExe::log.error "#{MSG_PREFIX} no filename element in modification #{mod}"
|
|
next
|
|
end
|
|
|
|
CheckExe::log.info "#{MSG_PREFIX} #{filename.text}"
|
|
found_exe = true if OS::Path.normalise(filename.text) == OS::Path.normalise(exe)
|
|
end
|
|
|
|
if found_exe
|
|
CheckExe::log.info "#{MSG_PREFIX_PERSIST} A new revision of #{exe} has been found and can be tested."
|
|
CheckExe::log.info("#{MSG_PREFIX_PERSIST} This build can continue as it has passed the check exe test")
|
|
return true
|
|
end
|
|
|
|
# DW : this used to be an error for sake of reporting but erroring is emailing unnecessarily.
|
|
CheckExe::log.info("NO NEW EXECUTABLE TO TEST : check_exe determined that there is no NEW executable to test in this build.")
|
|
CheckExe::log.info("Either the associated codebuilder build failed OR reverted upon unchanged the executable <#{exe}>")
|
|
CheckExe::log.info("This build was executed in order that the codetest pipeline is fully executed in order to make reporting clear.")
|
|
return true
|
|
end
|
|
|
|
end # end class CheckExe
|
|
|
|
|
|
#=============================================================================================
|
|
#
|
|
# Check if we processed a build with a capture in it otherwise what is the point in performing
|
|
# a test, we should not be needlessly processing the same file again - might end up with repeat info in database too.
|
|
#
|
|
class CheckCapture
|
|
|
|
#
|
|
# constructor
|
|
#
|
|
def initialize()
|
|
end
|
|
|
|
@@log = nil
|
|
def CheckCapture.log
|
|
|
|
@@log = Log.new( 'check_capture' ) if @@log == nil
|
|
@@log
|
|
end
|
|
|
|
#---------------------------------------------------------------------------------------------------------
|
|
#
|
|
# Control logic
|
|
#
|
|
def check_capture(p4, modifications_file, capture_filename)
|
|
|
|
CheckCapture::log.info "#{MSG_PREFIX_PERSIST} Checking if the capture file was in the modifications list - otherwise we have nothing to work with"
|
|
CheckCapture::log.info "#{MSG_PREFIX} modifications_file = #{modifications_file}"
|
|
CheckCapture::log.info "#{MSG_PREFIX} capture_filename = #{capture_filename}"
|
|
|
|
doc = REXML::Document.new(File.new(modifications_file))
|
|
mods = doc.elements.to_a( "//ArrayOfModification/Modification" )
|
|
|
|
CheckCapture::log.info "#{MSG_PREFIX} The modification filenames are :-"
|
|
found_capture_file = false
|
|
mods.each do |mod|
|
|
filename = mod.elements["filename"]
|
|
filename = mod.elements["FileName"] unless filename
|
|
|
|
if not filename
|
|
CheckCapture::log.error "#{MSG_PREFIX} no filename element in modification #{mod}"
|
|
next
|
|
end
|
|
|
|
CheckCapture::log.info "#{MSG_PREFIX} #{filename.text}"
|
|
found_capture_file = true if OS::Path.normalise(filename.text) == OS::Path.normalise(capture_filename)
|
|
end
|
|
|
|
if found_capture_file
|
|
CheckCapture::log.info "#{MSG_PREFIX_PERSIST} A new revision of #{capture_filename} has been found and can be tested."
|
|
CheckCapture::log.info("#{MSG_PREFIX_PERSIST} This build can continue as it has passed the check capture_filename test")
|
|
return true
|
|
end
|
|
|
|
CheckCapture::log.error("NO NEW CAPTURE FILE #{capture_filename} TO TEST : check_capture determined that there is no NEW capture file to test in this build.")
|
|
CheckCapture::log.error("This means the associated codetester capture failed, which in turn can mean that the codebuilder build didn't create a new executable.")
|
|
CheckCapture::log.error("This build was executed in order that the codetest pipeline is fully executed in order to make reporting clear.")
|
|
return false
|
|
end
|
|
|
|
end # end class CheckCapture
|
|
|
|
#=============================================================================================
|
|
#
|
|
# Check if we should skip this build, we would skip if we are too far behind the head,
|
|
# the danger is that if we fall behind the head too far then since the binaries that are published
|
|
# only store limited number of revisions that we will not be able to retrieve the binary.
|
|
#
|
|
class CheckSkip
|
|
|
|
#
|
|
# constructor
|
|
#
|
|
def initialize()
|
|
end
|
|
|
|
@@log = nil
|
|
def CheckSkip.log
|
|
|
|
@@log = Log.new( 'check_skip' ) if @@log == nil
|
|
@@log
|
|
end
|
|
|
|
#---------------------------------------------------------------------------------------------------------
|
|
#
|
|
# Control logic
|
|
#
|
|
def check_skip(p4, publish_folder, check_skip, exe)
|
|
|
|
CheckSkip::log.info "#{MSG_PREFIX_PERSIST} Checking if we should skip this test"
|
|
CheckSkip::log.info "#{MSG_PREFIX} publish_folder = #{publish_folder}"
|
|
CheckSkip::log.info "#{MSG_PREFIX} check_skip = #{check_skip} ( the number of revisions behind the head that will incur a skip of this test )"
|
|
CheckSkip::log.info "#{MSG_PREFIX} exe = #{exe}"
|
|
|
|
# ---- build exe filename ----
|
|
filename = OS::Path.combine(publish_folder,exe)
|
|
|
|
# ---- find files ----
|
|
fstat = p4.run_fstat( filename ).shift
|
|
if (not fstat)
|
|
CheckSkip::log.error("Error: Bad fstat")
|
|
return false
|
|
end
|
|
|
|
CheckSkip::log.info("#{MSG_PREFIX_PERSIST} We will skip this test if we are #{check_skip} revisions behind the head of #{exe}")
|
|
|
|
have_revision = fstat['haveRev'].to_i
|
|
head_revision = fstat['headRev'].to_i
|
|
|
|
rev_delta = head_revision-have_revision
|
|
if rev_delta >= check_skip.to_i
|
|
CheckSkip::log.error("Error: We are #{rev_delta} revisions of #{filename} behind the head" )
|
|
CheckSkip::log.error("Error: (have revision =#{have_revision} head_revision=#{head_revision}" )
|
|
CheckSkip::log.error("Error: This build will error and stop now so that we do not process this build - this is not a 'true' error per se.")
|
|
return false
|
|
end
|
|
|
|
CheckSkip::log.info("#{MSG_PREFIX_PERSIST} Currently we are #{rev_delta} revisions of #{filename} behind the head")
|
|
CheckSkip::log.info("#{MSG_PREFIX_PERSIST} (have revision=#{have_revision} head_revision=#{head_revision}")
|
|
CheckSkip::log.info("#{MSG_PREFIX_PERSIST} This build can continue as it has passed the check skip test")
|
|
|
|
return true
|
|
end
|
|
|
|
end # end class CheckSkip
|
|
|
|
|
|
|
|
#---------------------------------------------------------------------------------------------------------
|
|
#
|
|
# if this a valid build - only certain build configurations are to be handled ( see VALID_BUILDS )
|
|
#
|
|
def valid_build(publish_folder)
|
|
VALID_BUILDS.each { |build| return true if publish_folder.include?(build) }
|
|
false
|
|
end
|
|
|
|
#---------------------------------------------------------------------------------------------------------
|
|
#
|
|
# get the revisions of modifications xml file, returning the revision we have and the last revisions processed
|
|
# this gives us a range of revisions for which we need to collect all the modifications from.
|
|
#
|
|
def get_modification_revisions(p4, publish_folder,last_mods_revision_filename )
|
|
# ask p4 the revision of the modifications
|
|
last_modifications_revision = 0
|
|
if (File.exist?(last_mods_revision_filename))
|
|
file = File.new(last_mods_revision_filename, "r")
|
|
last_modifications_revision = file.gets.to_i
|
|
PublishFiles::log.info "#{MSG_PREFIX} last_modifications_revision #{last_modifications_revision}"
|
|
else
|
|
PublishFiles::log.info "#{MSG_PREFIX} last_mods_revision_filename #{last_mods_revision_filename} doesn't exist"
|
|
end
|
|
|
|
have_modifications_revision = 0
|
|
modifications_file = OS::Path::combine(publish_folder,MODIFICATIONS_FILE)
|
|
ret = p4.run_files("#{modifications_file}#have")
|
|
have_modifications_revision = ret[0]["rev"].to_i if ret and ret[0]
|
|
|
|
PublishFiles::log.info "#{MSG_PREFIX} #{MODIFICATIONS_FILE} last #{last_modifications_revision} have #{have_modifications_revision}"
|
|
return last_modifications_revision, have_modifications_revision
|
|
end
|
|
|
|
#---------------------------------------------------------------------------------------------------------
|
|
#
|
|
# Create a p4 object
|
|
#
|
|
def create_p4(config)
|
|
PublishFiles::log.debug "#{MSG_PREFIX} Create p4 connection"
|
|
p4 = SCM::Perforce::create( config.sc_server, config.sc_username, config.sc_workspace )
|
|
p4.connect( )
|
|
raise Exception if not p4.connected?
|
|
p4
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
|
|
#-----------------------------------------------------------------------------
|
|
# Application entry point
|
|
#
|
|
# DW - could do with some tidying when I get time!!!!!
|
|
#
|
|
#-----------------------------------------------------------------------------
|
|
if ( __FILE__ == $0 )
|
|
|
|
begin
|
|
g_AppName = File::basename( __FILE__, '.rb' )
|
|
g_ProjectName = ''
|
|
g_BranchName = ''
|
|
g_Project = nil
|
|
g_Config = Pipeline::Config.instance()
|
|
|
|
#--------------------------------------------------------------------
|
|
# --- PARSE COMMAND LINE ---
|
|
#--------------------------------------------------------------------
|
|
opts, trailing = OS::Getopt.getopts( OPTIONS )
|
|
if ( opts['help'] )
|
|
puts OS::Getopt.usage( OPTIONS )
|
|
puts ("Press Enter to continue...")
|
|
$stdin.getc( )
|
|
Process.exit!( 1 )
|
|
end
|
|
|
|
puts "#{MSG_PREFIX_COMMANDLINE} #{$0} #{ARGV.join(" ")}"
|
|
|
|
# ---- create a p4 connection ----
|
|
p4 = create_p4(g_Config)
|
|
|
|
# ---- if in check skip mode, check to see if we should skip
|
|
if (opts['check_exe'])
|
|
modifications_file = opts['modifications_file']
|
|
exe = opts['exe']
|
|
if (modifications_file and exe)
|
|
skip_threshold = opts['check_exe']
|
|
check_exe = CheckExe.new()
|
|
puts"#{MSG_PREFIX} Checking #{modifications_file} for #{exe}"
|
|
ret = check_exe.check_exe( p4, modifications_file, exe )
|
|
puts"#{MSG_PREFIX} Check exe returned : #{ret}"
|
|
Process.exit!(-1) if not ret
|
|
Process.exit!( 0 )
|
|
else
|
|
$stderr.puts"Error: no modifications and/or exe file specified"
|
|
Process.exit!(-1)
|
|
end
|
|
end
|
|
|
|
# ---- check to see if we have a new capture ----
|
|
if (opts['check_capture'])
|
|
modifications_file = opts['modifications_file']
|
|
capture_file = opts['capture_file']
|
|
if (modifications_file and capture_file)
|
|
check_capture = CheckCapture.new()
|
|
puts"#{MSG_PREFIX} Checking #{modifications_file} for #{capture_file}"
|
|
ret = check_capture.check_capture( p4, modifications_file, capture_file )
|
|
puts"#{MSG_PREFIX} Check capture returned : #{ret}"
|
|
Process.exit!(-1) if not ret
|
|
Process.exit!( 0 )
|
|
else
|
|
$stderr.puts"Error: no modifications and/or capture file specified"
|
|
Process.exit!(-1)
|
|
end
|
|
end
|
|
|
|
|
|
# the following modes require the publish folder...
|
|
|
|
Process.exit!( 2 ) unless (opts['publish_folder'])
|
|
publish_folder = opts['publish_folder']
|
|
|
|
# ---- check if to run or not... ----
|
|
if (not valid_build(publish_folder))
|
|
puts "#{MSG_PREFIX} No build publish of executable and modifications is to be performed on this build. ( publish folder = #{publish_folder} ) This is ok."
|
|
Process.exit!( 0 )
|
|
end
|
|
|
|
# ---- if in check skip mode, check to see if we should skip
|
|
if (opts['check_skip'])
|
|
skip_threshold = opts['check_skip']
|
|
exe = opts['exe']
|
|
if (exe)
|
|
check_skip = CheckSkip.new()
|
|
Process.exit!(-1) if not check_skip.check_skip( p4, publish_folder, skip_threshold, exe )
|
|
Process.exit!( 0 )
|
|
else
|
|
Process.exit!(-1)
|
|
end
|
|
end
|
|
|
|
# ---- if in sync data mode, sync the data
|
|
if (opts['sync_data'])
|
|
sync_data = SyncData.new()
|
|
sync_data.sync_data( p4, publish_folder, opts['build_shaders'] )
|
|
Process.exit!( 0 )
|
|
end
|
|
|
|
# ---- if in copy mode, just copy and exit from the publishing - kept as one script for simplicitiy ( for now ) ----
|
|
if (opts['copy_dir'])
|
|
copy_directory = opts['copy_dir']
|
|
copy_dir = CopyDir.new()
|
|
copy_dir.copy_dir( p4, publish_folder, copy_directory )
|
|
Process.exit!( 0 )
|
|
end
|
|
|
|
# ---- aggregate various modifications revisions - since we have skipped to the head...
|
|
last_mods_revision_filename = OS::Path::combine(publish_folder,LAST_MODIFICATIONS_FILE_REVISION_FILENAME)
|
|
last_modifications_revision,have_modifications_revision = get_modification_revisions(p4, publish_folder, last_mods_revision_filename)
|
|
|
|
if (opts['aggregate'])
|
|
aggregate_mods = AggregateMods.new()
|
|
aggregate_mods.aggregate_mods(p4, publish_folder, have_modifications_revision, last_modifications_revision, last_mods_revision_filename)
|
|
Process.exit!( 0 )
|
|
end
|
|
|
|
Process.exit!( 4 ) unless (opts['filespec'])
|
|
filespec = opts['filespec']
|
|
|
|
enable_checkin = false
|
|
enable_checkin = true if (opts['enable_checkin'])
|
|
|
|
# ---- publish files into p4 ( binaries or stats )----
|
|
publish_files = PublishFiles.new()
|
|
publish_files.publish_files(p4, publish_folder, filespec, enable_checkin)
|
|
Process.exit! 0
|
|
|
|
rescue Exception => ex
|
|
$stderr.puts "Error: Unhandled exception: #{ex.message}"
|
|
$stderr.puts "Backtrace:"
|
|
ex.backtrace.each { |m| $stderr.puts "\t#{m}" }
|
|
Process.exit! -1
|
|
end
|
|
end
|