Files
gtav-src/tools_ng/lib/util/CruiseControl/cc_make.rb
T
2025-09-29 00:52:08 +02:00

556 lines
18 KiB
Ruby
Executable File

#
# File:: cc_make.rb
# Description:: processes simple xml files to produce various cruise control configuration files.
#
# https://devstar.rockstargames.com/wiki/index.php?title=Tool_Builder&action=edit&redlink=1
#
# Author:: Derek Ward <derek.ward@rockstarnorth.com>
# Date:: 20th June 2011
#
# Passed in :- see OPTIONS ...
# Passed out :-
#
# Returns :-
#-----------------------------------------------------------------------------
# Uses / Requires
#-----------------------------------------------------------------------------
require 'pipeline/os/path'
require 'pipeline/os/getopt'
require 'pipeline/log/log'
require 'xml'
require "rexml/document"
include REXML
include Pipeline
#-----------------------------------------------------------------------------
# Constants
#-----------------------------------------------------------------------------
INFO_BLUE = "[colourise=blue]INFO_MSG: "
HELP_URL = "https://devstar.rockstargames.com/wiki/index.php?title=Tool_Builder&action=edit&redlink=1"
FUNCTION = "toolbuilder"
#
# solution class - spec for build
class Solution
NAMESPACE = "urn:ccnet.config.builder"
DEFAULT_INCREDICMD = "/build"
PS3_INCREDICMD = "/incredi"
DEFAULT_REBUILD_INCREDICMD = "/rebuild"
PS3_REBUILD_INCREDICMD = "/incredi /rebuild"
CB_SCOPE = 'cb:scope'
CB_NAMESPACE = 'xmlns:cb'
CB_INCLUDE = "cb:include"
SPLIT_CONFIGS = ","
SPLIT_ENV = ","
DEFAULT_TASK_EXEC_FILE = "cruisecontrol/#{FUNCTION}/#{FUNCTION}_task_incredibuild.xml"
PS3_TASK_EXEC_FILE = "cruisecontrol/#{FUNCTION}/#{FUNCTION}_task_vsi.xml"
PS3_BUILD_IDENTIFIER = "ps"
MAX_CUSTOM_ENV_VARIABLES = 4
attr_accessor :name, :platform_compiler, :build_config, :sln, :email_breakers, :owner, :extra_targets, :extra_slns, :config, :fold, :env
def initialize(node)
@name = node["name"]
@config = node["config"]
@sln = node["sln"]
@owner = node["owner"]
@email_breakers = node["email_breakers"]
@env = node["env"]
@rebuild = node["rebuild"]
if @rebuild.nil?
@rebuild = false
else
@rebuild = true if (@rebuild.downcase != "false" and @rebuild.downcase != "0")
end
if @email_breakers.nil?
@email_breakers = false
else
@email_breakers = true if (@email_breakers.downcase != "false" and @email_breakers.downcase != "0")
end
@extra_targets = []
@extra_slns = []
@fold = -1
get_platform_and_config_names()
end
def to_s()
email_breakers = @email_breakers == true ? "email breakers=on" : "email breakers=off"
rebuild = @rebuild == true ? "rebuild=true" : "rebuild=false"
str = "#{@name.ljust(20)}owner= #{@owner.ljust(30)}#{email_breakers.to_s.ljust(20)}#{rebuild.to_s.ljust(15)}\n#{INFO_BLUE}\t\t#{@sln.ljust(80)}#{@config.ljust(20)}"
@extra_targets.each_with_index do |t,i|
str += "\n#{INFO_BLUE}\t\t#{@extra_slns[i].ljust(80)}#{t.ljust(20)}"
end
str
end
def can_fold( solution )
( @name == solution.name and @email_breakers == solution.email_breakers and @owner == solution.owner)
end
#
# Split up the config string and determine what to call this project
def get_platform_and_config_names()
platforms, configs = [],[]
config_split = @config.split(SPLIT_CONFIGS)
config_split.each do |config|
split_config = config.split("|")
if ( split_config.length > 1 )
platforms << split_config[0]
configs << split_config[1]
end
end
platforms.uniq!
configs.uniq!
@platform_compiler = platforms.length > 1 ? platforms.length.to_s : platforms[0]
@build_config = configs.length > 1 ? configs.length.to_s : configs[0]
end
#
# convert solution to xmldoc suitable for CC use.
def to_xml( )
xmldoc = XML::Document.new
xmldoc.encoding = XML::Encoding::UTF_8
xmldoc.root = XML::Node.new( CB_SCOPE )
xmldoc.root.attributes[CB_NAMESPACE] = NAMESPACE
xmldoc.root.attributes['project'] = @name
xmldoc.root.attributes['platform_compiler'] = @platform_compiler
xmldoc.root.attributes['build_config'] = @build_config
xmldoc.root.attributes['toolbuild_owner_email'] = @owner
xmldoc.root.attributes['standard_email'] = "" unless @email_breakers == true
if @rebuild == true
xmldoc.root.attributes['incredicmd'] = @config.downcase.include?(PS3_BUILD_IDENTIFIER) ? PS3_REBUILD_INCREDICMD : DEFAULT_REBUILD_INCREDICMD
else
xmldoc.root.attributes['incredicmd'] = @config.downcase.include?(PS3_BUILD_IDENTIFIER) ? PS3_INCREDICMD : DEFAULT_INCREDICMD
end
xmldoc.root.attributes['task_exec_file'] = @config.downcase.include?(PS3_BUILD_IDENTIFIER) ? OS::Path.combine(ENV["RS_TOOLSCONFIG"],PS3_TASK_EXEC_FILE) : OS::Path.combine(ENV["RS_TOOLSCONFIG"],DEFAULT_TASK_EXEC_FILE)
xmldoc.root.attributes['targets'] = @config
xmldoc.root.attributes['solution_file'] = @sln
if (@env)
env_split = @env.split(SPLIT_ENV)
puts "Warning: Cruise control config needs expended to cater for #{env_split.length} ENV variables" if (env_split.length > MAX_CUSTOM_ENV_VARIABLES)
env_split.each_with_index do |e,idx|
name_val = e.split("=")
if (name_val.length==2)
env_name = name_val[0].strip
env_val = name_val[1].strip
xmldoc.root.attributes["custom_env_variable_name_#{idx+1}"] = env_name
xmldoc.root.attributes["custom_env_variable_value_#{idx+1}"] = env_val
end
end
end
# the solution can have extra configs and solutions it targets ( these are coupled - but linear in the output xml )
@extra_targets.each_with_index do |target,idx|
xmldoc.root.attributes["targets_#{idx+1}"] = target
task_exec_file,incredicmd = nil, nil
if (target.downcase.include?(PS3_BUILD_IDENTIFIER))
task_exec_file = OS::Path.combine(ENV["RS_TOOLSCONFIG"],PS3_TASK_EXEC_FILE)
incredicmd = PS3_REBUILD_INCREDICMD if @rebuild
incredicmd = PS3_INCREDICMD unless @rebuild
else
task_exec_file = OS::Path.combine(ENV["RS_TOOLSCONFIG"],DEFAULT_TASK_EXEC_FILE)
incredicmd = DEFAULT_REBUILD_INCREDICMD if @rebuild
incredicmd = DEFAULT_INCREDICMD unless @rebuild
end
xmldoc.root.attributes["incredicmd_#{idx+1}"] = incredicmd
xmldoc.root.attributes["task_exec_file_#{idx+1}"] = task_exec_file
end
@extra_slns.each_with_index do |sln,idx|
xmldoc.root.attributes["solution_file_#{idx+1}"] = sln
end
include = XML::Node.new(CB_INCLUDE)
include.attributes['href'] = "$(base_file)"
xmldoc.root << include
xmldoc
end
end
# CC make
class CCMaker
DEFAULT_BRANCH = "dev"
REGEXP_BRANCH = /^.*\\(\w*)/i
MACHINE = ENV["COMPUTERNAME"] # "EDIW-CC-8" if run remotely.
REMOTING_PORT = "21234"
REMOTING_FILE = "CruiseManager.rem"
SERVER_URL = "tcp://#{MACHINE}:#{REMOTING_PORT}/#{REMOTING_FILE}"
CCTRAY_PROJECTS_XPATH = "Configuration/Projects"
CCTRAY_ALL_PROJECTS_XPATH = 'Configuration/Projects/Project'
SOLUTIONS_XPATH = "//solutions/solution"
KEEP_CCTRAY_PROJECT_REGEXP = /^(toolbuilder_maker)/
IB_SUPPORTS_MULTIPLE_CONFIGS = false
ENV_SYMBOLS = [ "RAGE_DIR", "RS_TOOLSSRC", "RS_TOOLSCONFIG", "RS_CODEBRANCH", "RAGE_3RDPARTY", "RAGE_3RDPARTY", "RS_PROJECT" ]
MAX_COMBINED_CONFIGS = 4 # if update this you will need to update the cruise control stub also see
def initialize()
@config = Pipeline::Config.instance
@config.project.load_config
@env = Environment.new()
@config.project.fill_env( @env )
ENV_SYMBOLS.each { |sym| @env.add(sym,ENV[sym]) }
@p4 = SCM::Perforce.new()
@p4.port = @config.sc_server
@p4.client = @config.sc_workspace
@p4.user = @config.sc_username
@p4.connect()
@solutions = []
end
#
# main control process
def process(filenames, targetDir, cctray_file, submit )
make( filenames )
return false unless solutions_exist()
save( targetDir )
update_cctray( cctray_file )
submit_or_revert(submit)
true
end
#
# create within the class an xml representation of the history.
def make( filenames )
filenames.each do |filename|
xmldoc = LibXML::XML::Document::file( filename )
nodes = xmldoc.find(SOLUTIONS_XPATH)
# unfortuinately IB wont support mutiple build configs to build at the same time - this would be unnecessary if it could
nodes.each do |node|
if (IB_SUPPORTS_MULTIPLE_CONFIGS)
@solutions << Solution.new(node)
else
config_split = node['config'].split(Solution::SPLIT_CONFIGS)
config_split.each do |config|
node['config'] = config
@solutions << Solution.new(node)
end
end
end
end
fold_solutions()
end
#
# based upon some rules various solutions are combined in a bigger 'mega' ( within limits ) solution.
def fold_solutions()
# one day I'll be able to do this all in one line of ruby - but not on a Friday.
@solutions.each_with_index do |solution,i|
@solutions.each_with_index do |solution2,i2|
solution2.fold = i if (i2 > i and solution2.fold < 0 and solution2.can_fold(solution) )
end
end
@solutions.each_with_index do |solution,i|
if solution.fold > -1
@solutions[solution.fold].extra_targets << solution.config
@solutions[solution.fold].extra_slns << solution.sln
throw RuntimeError.new( "MAX_COMBINED_CONFIGS is #{MAX_COMBINED_CONFIGS} for solution #{solution.to_s}") if @solutions[solution.fold].extra_targets.length > MAX_COMBINED_CONFIGS-1
end
end
# rename them
@solutions.each do |solution|
if (solution.extra_targets.length > 0 or solution.extra_slns.length > 0 )
solution.platform_compiler = (solution.extra_targets.length+1).to_s
solution.build_config = "targets"
end
end
@solutions.delete_if { |solution| solution.fold >= 0 }
puts "#{INFO_BLUE} Folded Solutions comprise..."
@solutions.each { |solution| puts ("#{INFO_BLUE}\t#{full_project_name(solution).ljust(80)}\t#{solution.to_s}") }
end
#
# determine which solutions exist
def solutions_exist()
all_exist = true
@solutions.each do |solution|
filename = @env.subst( solution.sln )
filename = File::expand_path( filename )
if (not File.exists? filename)
$stderr.puts("Error: Solution filename #{filename} not found.")
all_exist = false
end
end
all_exist
end
#
# purge all old target files
def purge(target_dir)
# Purge dir
@p4.run_sync(target_dir)
@p4.run_delete('-c', @change_id.to_s, OS::Path.combine(target_dir,"#{FUNCTION}_target_*.xml") )
end
#
# save files
def save( target_dir )
#FileUtils::mkdir_p( OS::Path::get_directory( filename ) ) unless ( File::directory?( filename ) )
@change_id = @p4.create_changelist( "Automatically generated by cc_make.rb" )
purge(target_dir)
@solutions.each do |solution|
xmldoc = solution.to_xml( )
dest_file = OS::Path.combine(target_dir, "#{FUNCTION}_target_#{unique_project_name(solution)}.xml")
#puts "#{INFO_BLUE} Writing #{dest_file}"
@p4.run_sync( dest_file )
@p4.run_revert( '-c', @change_id.to_s, dest_file)
@p4.run_edit_or_add( '-c', @change_id.to_s, dest_file )
@p4.run_reopen( '-c', @change_id.to_s, dest_file )
xmldoc.save( dest_file )
end
save_target_list( target_dir )
end
# submit or revert files
def submit_or_revert( submit )
@p4.run_revert( '-a', '-c', @change_id.to_s, '//...')
files = @p4.run_opened( '-c', @change_id.to_s )
if ( files.size > 0 )
if (submit)
puts "#{INFO_BLUE} Submitting #{files.size} files in CL #{@change_id}"
puts "Warning: %RS_TOOLSROOT%/script/util/CruiseControl/CCTray/cctray_toolsbuilder.bat should be run to update your cctray."
files.each {|file| puts "#{INFO_BLUE} #{file['depotFile']}" }
submit_result = @p4.run_submit( '-c', @change_id.to_s )
else
puts "#{INFO_BLUE} Pending #{files.size} files in CL #{@change_id}"
end
elsif ( 0 == files.size )
puts "#{INFO_BLUE} No files changed - CL #{@change_id} deleted"
@p4.run_change('-d', @change_id.to_s)
end
end
#
# Get the full project name eg. toolbuilder_gta5_dev_ragebuilder_win32_release
def full_project_name(solution)
project_name = ENV['RS_PROJECT'].downcase
branch_name = DEFAULT_BRANCH
branch_name = $1.downcase if (ENV['RS_CODEBRANCH'] =~ REGEXP_BRANCH)
branch_name = DEFAULT_BRANCH if branch_name.length == 0
"#{FUNCTION}_#{project_name}_#{branch_name}_#{solution.name}_#{solution.platform_compiler}_#{solution.build_config}"
end
#
# Get the unique project name withoput the faff as a prefix eg. ragebuilder_win32_release NOT this toolbuilder_gta5_dev_ragebuilder_win32_release
def unique_project_name(solution)
"#{solution.name}_#{solution.platform_compiler}_#{solution.build_config}"
end
#
# Update the CCTray config file
def update_cctray( cctray_file )
@p4.run_sync( cctray_file )
fstat = @p4.run_fstat( cctray_file ).shift
basetype, modifiers = fstat['headType'].split( '+' ) if (fstat.key?('headType')) # perforce drives me dippy.
basetype, modifiers = fstat['type'].split( '+' ) if (fstat.key?('type'))
@p4.run_edit_or_add( '-c', @change_id.to_s, cctray_file )
@p4.run_reopen( '-c', @change_id, cctray_file )
@p4.run_reopen( '-t', "#{basetype}+w", '-c', @change_id.to_s, cctray_file )
xmldoc = REXML::Document.new File.open( cctray_file, 'r' )
# new working element store
new_projects_elem = Element.new( "Projects" )
# keep some existing elements
xmldoc.elements.each( CCTRAY_ALL_PROJECTS_XPATH ) do |elem|
if elem.attributes["projectName"] =~ KEEP_CCTRAY_PROJECT_REGEXP
#puts "#{INFO_BLUE} Keeping CCtray project #{elem.attributes["projectName"]}"
new_projects_elem.add_element(elem.clone)
end
end
# add all elements as defined by solutions
@solutions.each do |solution|
found = false
project_name = full_project_name(solution)
#puts "#{INFO_BLUE} Adding CCtray project #{project_name}"
elem = Element.new( "Project" )
elem.add_attribute("serverUrl", SERVER_URL)
elem.add_attribute("projectName", project_name)
elem.add_attribute("showProject", "true")
new_projects_elem.add_element(elem)
end
# switch to the new element in the xmldoc
xmldoc.delete_element(CCTRAY_PROJECTS_XPATH)
xmldoc.root.add_element(new_projects_elem)
File.open( cctray_file, 'w' ) do |file|
fmt = REXML::Formatters::Default.new()
fmt.write(xmldoc,file)
end
end
private
#
# Save a list of targets
def save_target_list( target_dir )
xmldoc = target_list_to_xml( )
dest_file = OS::Path.combine(target_dir, "#{FUNCTION}_targets.xml")
#puts "#{INFO_BLUE} Writing #{dest_file}"
@p4.run_sync( dest_file )
@p4.run_edit_or_add( '-c', @change_id.to_s, dest_file )
@p4.run_reopen( '-c', @change_id.to_s, dest_file )
xmldoc.save( dest_file )
end
#
# Get xml doc for target file
def target_list_to_xml( )
xmldoc = XML::Document.new
xmldoc.encoding = XML::Encoding::UTF_8
xmldoc.root = XML::Node.new( Solution::CB_SCOPE )
xmldoc.root.attributes['xmlns:cb'] = Solution::NAMESPACE
@solutions.each do |solution|
include = XML::Node.new(Solution::CB_INCLUDE )
include.attributes['href'] = "$(tools_build_dir)\\#{FUNCTION}_target_#{unique_project_name(solution)}.xml"
xmldoc.root << include
end
xmldoc
end
# log
def self.log( )
@@log = Log.new( 'cc_make' ) if ( @@log.nil? )
@@log
end
@@log = nil
end
#----------------------------------------------------------------------------
# Implementation
#----------------------------------------------------------------------------
if ( __FILE__ == $0 ) then
OPTIONS = [
[ "--help", "-h", OS::Getopt::BOOLEAN, "display usage information." ],
[ '--target_dir', '-t', OS::Getopt::REQUIRED, "directory to output results to" ],
[ '--cctray' , '-c', OS::Getopt::OPTIONAL, "cctray file to update" ],
[ '--submit' , '-s', OS::Getopt::BOOLEAN, "submit or revert" ],
]
begin
g_AppName = File::basename( __FILE__, '.rb' )
#---------------------------------------------------------------------
# Parse Command Line
#---------------------------------------------------------------------
opts, g_Trailing = OS::Getopt.getopts( OPTIONS )
if ( opts['help'] ) then
puts OS::Getopt.usage( OPTIONS )
exit( 1 )
end
if ( not opts['target_dir'] ) then
puts OS::Getopt.usage( OPTIONS )
exit( 1 )
end
if (g_Trailing.length <= 0)
puts OS::Getopt.usage( OPTIONS )
puts "Supply files to read"
exit( 1 )
end
g_TargetDir = opts['target_dir']
g_CCtrayFile = opts['cctray']
g_Submit = opts["submit"]
if (g_CCtrayFile)
filename = File::expand_path( g_CCtrayFile )
if (File.exist? filename)
g_CCtrayFile = filename
else
g_CCtrayFile = nil
$stderr.puts("Error: filename #{filename} not found")
end
end
g_Filenames = []
Pipeline::OS::Path::set_downcase_on_normalise( false )
g_Trailing.each_with_index do |filename, index|
filename = File::expand_path( filename )
if (File.exist? filename)
g_Filenames << filename
else
$stderr.puts("Error: filename #{filename} not found")
end
end
Pipeline::OS::Path::set_downcase_on_normalise( true )
# Make
puts "#{INFO_BLUE} #{HELP_URL}"
cc_maker = CCMaker.new()
ret = cc_maker.process( g_Filenames, g_TargetDir, g_CCtrayFile, g_Submit )
Process.exit! -2 unless ret
rescue Exception => ex
$stderr.puts "Error: Unhandled exception: #{ex.message}"
puts ex.backtrace().join("\n")
Process.exit! -1
end
Process.exit! 0
end
# %RS_TOOLSLIB%/util/CruiseControl/cc_make.rb