x/gtav-src
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
a6770644bb24d2fc66a35fb9db6d70df5d108b1c
gtav-src/tools_ng/lib/util/Assetbuilder/assetbuild_convert_modifications.rb
T
x a6770644bb init gta
2025-09-29 00:52:08 +02:00

566 lines
23 KiB
Ruby
Executable File
Raw Blame History

#
# File:: assetbuild_convert_modifications.rb
# Description:: Takes a modifications xml file ( as produced by Cruise Control ) and sends files contained within into DRb connection/queue to be converted to platform assets.
# after files have been converted the logfile will be parsed for warnings and errors.
#
# It also handles a trailing list of files to process for rebuilds.
#
# Author:: Derek Ward <derek.ward@rockstarnorth.com>
# Date:: 21st May 2010
#
# Passed in :- filename to parse
# Passed out :- stderr contains all errors
# stdout for all other output.
# Returns :- returns non zero upon detecting any errors
#-----------------------------------------------------------------------------
# Uses / Requires
#-----------------------------------------------------------------------------
require 'pipeline/config/projects'
require 'pipeline/os/getopt'
require 'rexml/document'
include Pipeline
require 'drb'
require "source/builder/shell/console"
require "source/engine"
require 'systemu'
#-----------------------------------------------------------------------------
# Constants
#-----------------------------------------------------------------------------
OPTIONS = [
[ "--help", "-h", Getopt::BOOLEAN, "display usage information." ],
[ "--filename", "-f", Getopt::REQUIRED, "Filename to parse." ],
[ '--project', '-p', OS::Getopt::REQUIRED, 'specify project key (e.g. gta5, jimmy)' ],
[ '--branch', '-b', OS::Getopt::REQUIRED, 'specify project branch (e.g. dev, dev_migrate)' ],
[ '--cc_projectname', '-ccp', OS::Getopt::OPTIONAL, 'specify cruise control project name' ],
[ '--cc_rebuild', '-ccr', OS::Getopt::BOOLEAN, 'indicates this build is a cc rebuild project' ],
[ '--skip', '-s', Getopt::BOOLEAN, 'Permit skipping of files if farmed out to another machine - as per hardcoded SKIP_SETTINGS' ],
]
TRAILING_DESC = 'Perforce file paths separated by spaces (e.g. //depot/...). Passing in the string <rebuild> as the first param invokes a rebuild.'
ERROR_REGEXP = "\"^(.*)(Error\\s|Error\\s:|Error:\\s)(.*)$\""
WARNING_REGEXP = "\"^(.*)(Warning:\\s|Checking\sin\sfiles|:Platform|EngineMesssage:\s)(.*)$\""
CHANGELIST_REGEXP = /^(.*)Checking\sin\sfiles\s\([0-9]+\)\sin\sCL\s([0-9]+)\.\.\.$/
PLATFORM_FILE_REGEXP = /^(.*):Platform\s(.*)\sFile\s(.*)$/
ENGINE_MESSAGE_REGEXP = /^(.*)EngineMesssage:(.*)$/
REGEXP_LOGFILE = /Logfiles\s:\s(.*)/
INFO = "[colourise=black]INFO_MSG: "
INFO_BLUE = "[colourise=blue]INFO_MSG: "
MAX_LOGFILE_LINES = 100000
# ORDERED list of machine/regexs pairs for matching p4 modifications.
#
# - The first match picks up the responsibility for this modification.
# - The MAIN server RSGEDICC1 matches everything but will skip a modification
# if a previous machine matches it first.
#
# These settings permit rendundancy of machines
# should you wish to take a machine down then,
# 1) Comment out all of it's entries for the machine.
# 2) Ensure a machine that is still active picks up the remainder,
# eg. this line would be entered LAST. {"RSGEDIABLD..." => /.*/i},
SKIP_SETTINGS = [
{"INVALIDHOST" => /.*anim\/ingame\/*/i}, # DW 14-09-12 associated with non existent machine, this will not be built the new assetbuilder will pick this up. DW updated to take off all anim ingame of hw4 21-06-12.
{"INVALIDHOST" => /.*anim\/cutscene\/*/i}, # LPXO 08-11-12 In preparation for new cutscene pipeline
{"INVALIDHOST" => /.*componentpeds/i}, # DHM 16-10-12 componentpeds now on AP3.
{"INVALIDHOST" => /.*streamedpeds/i}, # DHM 16-10-12 streamedpeds now on AP3.
{"INVALIDHOST" => /.*cutspeds/i}, # DHM 16-10-12 cutspeds now on AP3.
{"INVALIDHOST" => /.*levels\/anim_test\/*/i},
{"INVALIDHOST" => /.*levels\/cptestbed\/*/i},
{"INVALIDHOST" => /.*levels\/generic\/*/i},
{"INVALIDHOST" => /.*levels\/gfx_test\/*/i},
{"INVALIDHOST" => /.*levels\/gta5\/_citye\/*/i}, # JWR 23-01-13 _citye now on AP3
{"INVALIDHOST" => /.*levels\/gta5\/_cityw\/*/i}, # JWR 22-01-13 _cityw now on AP3
{"INVALIDHOST" => /.*levels\/gta5\/_hills\/*/i}, # JWR 23-01-13 _hills now on AP3
{"INVALIDHOST" => /.*levels\/gta5\/_prologue\/*/i}, # JWR 22-01-13 _prologue now on AP3
{"INVALIDHOST" => /.*levels\/gta5\/cloudhats\/*/i}, # JWR 25-01-13 cloudhats now on AP3
{"INVALIDHOST" => /.*levels\/gta5\/destruction\/*/i}, # JWR 25-01-13 destruction now on AP3
{"INVALIDHOST" => /.*levels\/gta5\/interiors\/*/i}, # JWR 22-01-13 interiors now on AP3
{"INVALIDHOST" => /.*levels\/gta5\/outsource\/*/i}, # JWR 25-01-13 outsource now on AP3
{"INVALIDHOST" => /.*levels\/gta5\/props\/*/i}, # JWR 23-01-13 props now on AP3
{"INVALIDHOST" => /.*levels\/gta5\/vehicles_packed\/*/i}, # LPXO 12-12-12 new AP3 data for vehicles.
{"INVALIDHOST" => /.*levels\/nettestbed\/*/i},
{"INVALIDHOST" => /.*levels\/nm_test\/*/i},
{"INVALIDHOST" => /.*levels\/testbed\/*/i},
{"INVALIDHOST" => /.*levels\/tools_test\/*/i},
{"INVALIDHOST" => /.*levels\/vfx_test\/*/i},
{"INVALIDHOST" => /.*levels\/waterbed\/*/i},
{"RSGEDIABLD1" => /.*/i}, # do not remove : this picks up the remainder of files
]
# DW: abit of a hack - more 'skip settings' that apply to ALL, regardless if the commandline doesn't specify --skip
# instead these delete files before skipping so applies to all machines regardless.
ADDITIONAL_DELETE_SETTINGS = [
/.*levels\/gta5\/vehicles_packed\/*/i, # DW 12-12-12 this is required to skip on HW5 as per bugstar 975449
/.*anim\/ingame\/*/i, # DW 13-12-12 this is required to skip on HW5 as per bugstar 964957
/.*anim\/cutscene\/*/i, # DW 09-01-13 as requested by Etienne
/.*componentpeds/i, # DW 13-12-12 this is required to skip on HW5 as per bugstar 964957
/.*streamedpeds/i, # DW 13-12-12 this is required to skip on HW5 as per bugstar 964957
/.*levels\/gta5\/vehicles\/*/i, # DW 21-01-13 Luke asked for this to be removed from all AP2 builders
]
# Do not change this message - since this is recognised by Cruise Control.
SKIP_MSG = "Error : SKIPPED CL - NO BUILD HAS TAKEN PLACE"
DELETE_MSG = "Error : SKIPPED CL - NO BUILD HAS TAKEN PLACE (FORCE)"
#-----------------------------------------------------------------------------
#<!-- Start of the group of modifications (even if just one). -->
#<ArrayOfModification xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema">
# <!-- Start of one modification. -->
# <Modification>
# <!-- The change number. -->--filename=
# <ChangeNumber>... value ...</ChangeNumber>
# <!-- The comment. -->
# <Comment>... value ...</Comment>
# <!-- The user's email address. -->
# <EmailAddress>... value ...</EmailAddress>
# <!-- The affected file name. -->
# <FileName>... value ...</FileName>
# <!-- The affect file's folder name. -->
# <FolderName>... value ...</FolderName>
# <!-- The change timestamp, in yyyy-mm-ddThh:mm:ss.nnnn-hhmm format -->
# <ModifiedTime>... value ...</ModifiedTime>
# <!-- The operation type. -->
# <Type>... value ...</Type>
# <!-- The user name. -->
# <UserName>... value ...</UserName>
# <!-- The related URL. -->
# <Url>... value ...</Url>
# <!-- The file version. -->
# <Version>... value ...</Version>
# <!-- End of modification. -->
# </Modification>
# <!-- End of the group of modifications. -->
#</ArrayOfModification>
#Pasted from <http://confluence.public.thoughtworks.org/display/CCNET/Modification+Writer+Task>
#-----------------------------------------------------------------------------
# Parse local files with a wildcard
# - searches wildcard passed in.
# - return an array of local files which are independent files.
#-----------------------------------------------------------------------------
def parse_wildcard( wildcard, project, branchname )
puts "#{INFO} Parsing wildcard #{wildcard}"
files = []
independent_path = OS::Path.normalise(project.branches[branchname].export)
puts "independent_path #{independent_path}"
ind_wildcard = OS::Path::combine( independent_path, wildcard )
puts "ind_wildcard #{ind_wildcard}"
files = AssetBuild::Engine::get_files( ind_wildcard )
puts "\t#{INFO}Independent files wildcard: #{ind_wildcard}, #{files.size} files found."
files.each do |filename|
puts "\t#{INFO}#{filename}\n"
end
return files
rescue Exception => ex
$stderr.puts "Error :Unexpected exception parsing wildcards: "
$stderr.puts "Error :\t#{ex.message}"
ex.backtrace.each { |m| $stderr.puts "Error :\t#{m}"; }
Process.exit -1
end
#-----------------------------------------------------------------------------
# Parse the Cruise Control XML file for modificatons.
# Return an array of p4 filespecs.
#-----------------------------------------------------------------------------
def parse_cc_modifications( filename, project, branchname )
puts "#{INFO} Parsing #{filename}"
files = []
begin
if not File.exists?(filename)
$stderr.puts "Error : #{filename} does not exist."
return nil
end
#-------------------------------------------------
#----------- Open Source xml ---------------------
#-------------------------------------------------
src_file = File.new( filename )
if not src_file
$stderr.puts "Error : #{filename} could not open."
return nil
end
#-------------------------------------------------
#----------- Read XML doc ------------------------
#-------------------------------------------------
xmldoc = REXML::Document.new( src_file )
if not xmldoc
$stderr.puts "Error : #{filename} could not be opened as an XML document."
return nil
end
independent_path = OS::Path.normalise(project.branches[branchname].export)
puts "independent_path #{independent_path}"
p4 = project.scm( )
p4.connect() unless ( p4.connected?() )
xmldoc.elements.each( 'ArrayOfModification/Modification' ) do |modification|
folder_name = nil
file_name = nil
revision = nil
type = nil
modification.elements.each('FolderName') { |element| folder_name = element.text }
modification.elements.each('FileName') { |element| file_name = element.text }
modification.elements.each('Version') { |element| revision = element.text }
modification.elements.each('Type') { |element| type = element.text }
if folder_name.nil? or file_name.nil? or revision.nil? or type.nil?
$stderr.puts "Error : #{filename} Invalid XML."
return nil
end
local_file = OS::Path.normalise(p4.depot2local( "#{folder_name}/#{file_name}" ))
puts "Local file #{local_file}"
if ( local_file and local_file.include?( independent_path ) )
mod = "#{folder_name}/#{file_name}\##{revision}"
puts "\t#{INFO} #{type} Parsed #{mod} "
files << mod
else
puts "\t#{INFO} : #{local_file} will not be converted as it's path is not in #{independent_path}" if local_file and local_file.length > 0
end
end
rescue Exception => ex
$stderr.puts "Error :Unexpected exception parsing modifications: "
$stderr.puts "Error :\t#{ex.message}"
ex.backtrace.each { |m| $stderr.puts "Error :\t#{m}"; }
Process.exit -1
end
files
end
#-----------------------------------------------------------------------------
# Return true if this modification string should be skipped
#-----------------------------------------------------------------------------
def should_skip_modification(mod, skip_settings)
skip_settings.each do |skip|
skip.each do |machine, regex|
if (mod=~regex)
if (machine.downcase==ENV["COMPUTERNAME"].downcase)
puts "#{INFO} #{mod} is matched with #{machine} by #{regex}"
return false # this computer wishes to process this modification - return false - do not skip.
else
puts "#{INFO} #{mod} is skipped as it it matched with #{machine} by #{regex} which is not this machine #{ENV["COMPUTERNAME"]}"
return true # another machine wishes to process this modification - return true - skip this modification.
end
end
end
end
puts "Warning : no match for mod #{mod} - this shouldn't happen."
return true # skip this modification it was not matched by any machine.
end
#-----------------------------------------------------------------------------
# Return true if this modification string should be deleted
#-----------------------------------------------------------------------------
def should_delete_modification(mod, delete_settings)
delete_settings.each do |regex|
if (mod=~regex)
puts "#{INFO} #{mod} is deleted as it it matched by #{regex}"
return true
end
end
return false
end
#-----------------------------------------------------------------------------
# Kick off the conversion of files on this machine.
# It uses a remote console to the assertbuilder engine,
# It uses the 'build' command. The files are expected to be p4 filespecs only.
# Return the logfile produced.
# the wildcards are a way to tell the engine a summry of what is being built ( for reporting )
#-----------------------------------------------------------------------------
def convert_files( files, is_rebuild = false, all_wildcards = nil, cc_project_name = nil, cc_rebuild = nil )
logfiles = nil
begin
puts "#{INFO} Converting #{all_wildcards} : #{files.length} files \t#{Time.now} "
console = Assetbuild::Builder::Shell::Console.new( )
rebuild = is_rebuild ? "rebuild" : ""
wildcards = all_wildcards ? "wildcard=#{all_wildcards}" : ""
cc_project_name = cc_project_name ? "cc_project_name=#{cc_project_name}" : ""
if (cc_rebuild==true)
cc_rebuild = "cc_rebuild"
else
cc_rebuild = ""
end
command = "build #{rebuild} #{wildcards} #{cc_project_name} #{cc_rebuild} #{files.join(' ')}"
puts "#{INFO} Issuing remote command to engine: #{command} \t#{Time.now}"
puts "#{INFO} This is a REBUILD command." if is_rebuild
puts "#{INFO} This is a regular build command." unless is_rebuild
result = console.run_command( command )
puts "\t#{INFO} Build result = #{result} \t#{Time.now}"
if (result =~ REGEXP_LOGFILE)
logfiles = $1
puts "\t#{INFO} Logfiles : #{logfiles} \t#{Time.now}"
end
puts "#{INFO} Convert completed \t#{Time.now}"
rescue Exception => ex
$stderr.puts "Error :Unexpected exception converting files: "
$stderr.puts "Error :\t#{ex.message}"
ex.backtrace.each { |m| $stderr.puts "Error :\t#{m}"; }
Process.exit -1
end
logfiles
end
#-----------------------------------------------------------------------------
# Parse the logfile, matching on errors and warning.
# Return the number of errors and warnings.
# Errors go to stderr
# Warnings go to stdout.
#-----------------------------------------------------------------------------
def parse_logfiles( log_filenames_list, use_error_parser = true )
total_lines_read = 0
total_error_count = 0
total_warning_count = 0
log_filenames = log_filenames_list.split
found_errors = false
log_filenames.each_with_index do |log_filename, idx|
put_info_in_report = false
puts "#{INFO}Parsing XGE Logfile \##{idx+1}/#{log_filenames.length} #{log_filename}"
puts "========================================================================="
c = Pipeline::Config::instance
cmd = "#{OS::Path.combine(c.toolsbin, "errorparser.exe")}"
args = "#{log_filename} #{ERROR_REGEXP} #{WARNING_REGEXP}"
full_cmd = "#{cmd} #{args}"
puts full_cmd
status, stdout, stderr = systemu(full_cmd)
err_count = 0
stderr.each do |error|
err_count += 1
end
# Caveat
if ( stderr.length > 0 and not found_errors)
$stderr.puts("Error :=====================================================================\n")
$stderr.puts("Error :*** YOUR ASSET(S) MAY NOT HAVE CONVERTED CORRECTLY. - PLEASE READ *** \n")
$stderr.puts("Error :=====================================================================\n")
$stderr.puts("Error :- THESE ERRORS ARE FROM THE ASSETBUILDER / RAGEBUILDER.\n")
$stderr.puts("Error :- THE ISSUE CAN BE WITH YOUR ASSET THAT YOU CHECKED IN.\n")
$stderr.puts("Error :- SOMETIMES IT MAY RELATE TO OTHER BREAKAGES IN DEPENDENCIES ( SHADERS )\n")
$stderr.puts("Error :- PLEASE CONTACT YOUR TOOLS TEAM.\n")
found_errors = true
end
if ( stderr.length > 0 )
$stderr.puts("Error : #{log_filename} contains #{err_count} errors")
$stderr.puts("Error :=====================================================================\n")
stderr.each do |error|
$stderr.puts "Error : #{error}"
end
end
stdout.each do |out|
$stdout.puts out
if out =~ CHANGELIST_REGEXP
puts "\t#{INFO_BLUE}Changelist submitted #{$2.to_s}"
elsif out =~ PLATFORM_FILE_REGEXP
puts "\t#{INFO_BLUE}#{$2.to_s} platform file: #{$3.to_s}"
elsif out =~ ENGINE_MESSAGE_REGEXP
puts "\t#{INFO_BLUE}Engine Message: #{$2.to_s}"
end
end
if (err_count>0)
puts "Errors where found in file #{log_filename}"
else
puts "No errors where found in file #{log_filename}"
end
total_error_count += err_count
end
return total_error_count, total_warning_count
end
#-----------------------------------------------------------------------------
# Application entry point
#-----------------------------------------------------------------------------
error_count = 0
warning_count = 0
begin
g_AppName = File::basename( __FILE__, '.rb' )
g_ProjectName = ''
g_BranchName = ''
g_Project = nil
g_Config = Pipeline::Config.instance()
#---------------------------------------------------------------------
# Parse Command Line.
#---------------------------------------------------------------------
opts, trailing = OS::Getopt.getopts( OPTIONS )
if ( opts['help'] )
puts OS::Getopt.usage( OPTIONS, { 'files' => TRAILING_DESC } )
puts ("Press Enter to continue...")
$stdin.getc( )
Process.exit( 1 )
end
if ( ( 0 == trailing.size ) and ( nil == opts['filename'] ) ) then
puts 'No trailing file arguments specified or modifications filename. Exiting.'
puts OS::Getopt.usage( OPTIONS, { 'files' => TRAILING_DESC } )
Process.exit( 2 )
end
g_ProjectName = ( nil == opts['project'] ) ? '' : opts['project']
project_exists = ( g_Config.projects.has_key?( g_ProjectName ) )
if ( not project_exists ) then
puts OS::Getopt.usage( OPTIONS )
puts "\nError project: #{g_ProjectName} does not exist or its configuration is unreadable."
Process.exit( 3 )
end
g_Project = g_Config.projects[ g_ProjectName ]
g_Project.load_config( )
if ( not g_Project.enabled ) then
puts "\nError project: #{g_ProjectName} is not enabled on this machine. Re-run installer."
Process.exit( 4 )
end
g_BranchName = ( nil == opts['branch'] ) ? g_Project.default_branch : opts['branch']
if ( not g_Project.branches.has_key?( g_BranchName ) ) then
puts "\nError project: #{g_ProjectName} does not have branch #{g_BranchName} defined."
Process.exit( 5 )
end
cc_projectname = opts['cc_projectname'] ? opts['cc_projectname'] : "-"
cc_rebuild = opts['cc_rebuild'] ? opts['cc_rebuild'] : nil
skip = opts['skip'] ? opts['skip'] : nil
puts "#{INFO}Running Script #{g_AppName} for cc project #{cc_projectname}"
#---------------------------------------------------------------------
# Parse the modifications xml file
#---------------------------------------------------------------------
filename = opts['filename']
files = []
files += parse_cc_modifications( filename, g_Project, g_BranchName ) if filename
is_rebuild = ( files.length == 0 )
# DW: Temp hack - as discussed with Luke regarding rebuilds
# some code path must not be getting exercised and the result is that
# the def need_convert? function ( for some reason - possibly timestamps )
# does not think some assets need converted.
# This is experimental for now.
#is_rebuild = true
trailing.each do |wildcard|
files += parse_wildcard( wildcard, g_Project, g_BranchName )
end
#
# Check if any files should be skipped
#
is_gta5 = ENV["RS_PROJECT"].downcase.include?("gta5")
# only gta5 is interested in this skipping hack
if (is_gta5)
#
# For all machines if the file can be matched in ADDITIONAL_DELETE_SETTINGS then remove it from the files to process
files.delete_if do | file |
delete_file = should_delete_modification(file, ADDITIONAL_DELETE_SETTINGS)
puts "#{INFO} #{file} was force skipped." if (delete_file)
delete_file
end
$stderr.puts(DELETE_MSG) if (files.length == 0)
#
# Then if desired *on this machine* we check if this file needs built here, skipped if delegated to another machine.
if (skip)
puts "#{INFO} Checking if files should be skipped"
all_skipped = true
files.delete_if do | file |
skip_file = should_skip_modification(file, SKIP_SETTINGS)
if (skip_file)
puts "#{INFO} #{file} was skipped."
else
all_skipped = false
end
skip_file
end
$stderr.puts(SKIP_MSG) if (all_skipped)
else
puts "#{INFO} no skipping enabled no enabled by commandline" unless skip
puts "#{INFO} no skipping enabled this project #{ENV["RS_PROJECT"]} is not configured for it" unless is_gta5
end
end
if ( files and files.length > 0 ) then
all_wildcards = trailing.join(",")
#---------------------------------------------------------------------
# Convert
#---------------------------------------------------------------------
puts "#{INFO} Converting files for #{cc_projectname} \t#{Time.now}"
log_filenames = convert_files( files, is_rebuild, all_wildcards, cc_projectname, cc_rebuild )
puts "#{INFO} Finished converting files \t#{Time.now}"
#---------------------------------------------------------------------
# Read log file and match errors and warnings.
#---------------------------------------------------------------------
puts "#{INFO} Parsing logfiles \t#{Time.now}"
error_count, warning_count = parse_logfiles( log_filenames ) if log_filenames
puts "#{INFO} Finished parsing logfiles \t#{Time.now}"
else
puts "#{INFO} Warning: no files were processed"
end
rescue Exception => ex
$stderr.puts "Error :Unhandled exception: #{ex.message}"
$stderr.puts "Error :Backtrace:"
ex.backtrace.each { |m| $stderr.puts "Error :\t#{m}" }
Process.exit -1
end
ret_code = ( error_count > 0 ) ? -1 : 0
puts "#{INFO} Script exiting with #{ret_code} since error_count is #{error_count} \t#{Time.now}"
Process.exit ret_code
Reference in New Issue View Git Blame Copy Permalink