# # File:: assetbuild_convert_modifications.rb # Description:: Takes a modifications xml file ( as produced by Cruise Control ) and sends files contained within into DRb connection/queue to be converted to platform assets. # after files have been converted the logfile will be parsed for warnings and errors. # # It also handles a trailing list of files to process for rebuilds. # # Author:: Derek Ward # Date:: 21st May 2010 # # Passed in :- filename to parse # Passed out :- stderr contains all errors # stdout for all other output. # Returns :- returns non zero upon detecting any errors #----------------------------------------------------------------------------- # Uses / Requires #----------------------------------------------------------------------------- require 'pipeline/config/projects' require 'pipeline/os/getopt' require 'rexml/document' include Pipeline require 'drb' require "source/builder/shell/console" require "source/engine" require 'systemu' #----------------------------------------------------------------------------- # Constants #----------------------------------------------------------------------------- OPTIONS = [ [ "--help", "-h", Getopt::BOOLEAN, "display usage information." ], [ "--filename", "-f", Getopt::REQUIRED, "Filename to parse." ], [ '--project', '-p', OS::Getopt::REQUIRED, 'specify project key (e.g. gta5, jimmy)' ], [ '--branch', '-b', OS::Getopt::REQUIRED, 'specify project branch (e.g. dev, dev_migrate)' ], [ '--cc_projectname', '-ccp', OS::Getopt::OPTIONAL, 'specify cruise control project name' ], [ '--cc_rebuild', '-ccr', OS::Getopt::BOOLEAN, 'indicates this build is a cc rebuild project' ], [ '--skip', '-s', Getopt::BOOLEAN, 'Permit skipping of files if farmed out to another machine - as per hardcoded SKIP_SETTINGS' ], ] TRAILING_DESC = 'Perforce file paths separated by spaces (e.g. //depot/...). Passing in the string as the first param invokes a rebuild.' ERROR_REGEXP = "\"^(.*)(Error\\s|Error\\s:|Error:\\s)(.*)$\"" WARNING_REGEXP = "\"^(.*)(Warning:\\s|Checking\sin\sfiles|:Platform|EngineMesssage:\s)(.*)$\"" CHANGELIST_REGEXP = /^(.*)Checking\sin\sfiles\s\([0-9]+\)\sin\sCL\s([0-9]+)\.\.\.$/ PLATFORM_FILE_REGEXP = /^(.*):Platform\s(.*)\sFile\s(.*)$/ ENGINE_MESSAGE_REGEXP = /^(.*)EngineMesssage:(.*)$/ REGEXP_LOGFILE = /Logfiles\s:\s(.*)/ INFO = "[colourise=black]INFO_MSG: " INFO_BLUE = "[colourise=blue]INFO_MSG: " MAX_LOGFILE_LINES = 100000 # ORDERED list of machine/regexs pairs for matching p4 modifications. # # - The first match picks up the responsibility for this modification. # - The MAIN server RSGEDICC1 matches everything but will skip a modification # if a previous machine matches it first. # # These settings permit rendundancy of machines # should you wish to take a machine down then, # 1) Comment out all of it's entries for the machine. # 2) Ensure a machine that is still active picks up the remainder, # eg. this line would be entered LAST. {"RSGEDIABLD..." => /.*/i}, SKIP_SETTINGS = [ {"INVALIDHOST" => /.*anim\/ingame\/*/i}, # DW 14-09-12 associated with non existent machine, this will not be built the new assetbuilder will pick this up. DW updated to take off all anim ingame of hw4 21-06-12. {"INVALIDHOST" => /.*anim\/cutscene\/*/i}, # LPXO 08-11-12 In preparation for new cutscene pipeline {"INVALIDHOST" => /.*componentpeds/i}, # DHM 16-10-12 componentpeds now on AP3. {"INVALIDHOST" => /.*streamedpeds/i}, # DHM 16-10-12 streamedpeds now on AP3. {"INVALIDHOST" => /.*cutspeds/i}, # DHM 16-10-12 cutspeds now on AP3. {"INVALIDHOST" => /.*levels\/anim_test\/*/i}, {"INVALIDHOST" => /.*levels\/cptestbed\/*/i}, {"INVALIDHOST" => /.*levels\/generic\/*/i}, {"INVALIDHOST" => /.*levels\/gfx_test\/*/i}, {"INVALIDHOST" => /.*levels\/gta5\/_citye\/*/i}, # JWR 23-01-13 _citye now on AP3 {"INVALIDHOST" => /.*levels\/gta5\/_cityw\/*/i}, # JWR 22-01-13 _cityw now on AP3 {"INVALIDHOST" => /.*levels\/gta5\/_hills\/*/i}, # JWR 23-01-13 _hills now on AP3 {"INVALIDHOST" => /.*levels\/gta5\/_prologue\/*/i}, # JWR 22-01-13 _prologue now on AP3 {"INVALIDHOST" => /.*levels\/gta5\/cloudhats\/*/i}, # JWR 25-01-13 cloudhats now on AP3 {"INVALIDHOST" => /.*levels\/gta5\/destruction\/*/i}, # JWR 25-01-13 destruction now on AP3 {"INVALIDHOST" => /.*levels\/gta5\/interiors\/*/i}, # JWR 22-01-13 interiors now on AP3 {"INVALIDHOST" => /.*levels\/gta5\/outsource\/*/i}, # JWR 25-01-13 outsource now on AP3 {"INVALIDHOST" => /.*levels\/gta5\/props\/*/i}, # JWR 23-01-13 props now on AP3 {"INVALIDHOST" => /.*levels\/gta5\/vehicles_packed\/*/i}, # LPXO 12-12-12 new AP3 data for vehicles. {"INVALIDHOST" => /.*levels\/nettestbed\/*/i}, {"INVALIDHOST" => /.*levels\/nm_test\/*/i}, {"INVALIDHOST" => /.*levels\/testbed\/*/i}, {"INVALIDHOST" => /.*levels\/tools_test\/*/i}, {"INVALIDHOST" => /.*levels\/vfx_test\/*/i}, {"INVALIDHOST" => /.*levels\/waterbed\/*/i}, {"RSGEDIABLD1" => /.*/i}, # do not remove : this picks up the remainder of files ] # DW: abit of a hack - more 'skip settings' that apply to ALL, regardless if the commandline doesn't specify --skip # instead these delete files before skipping so applies to all machines regardless. ADDITIONAL_DELETE_SETTINGS = [ /.*levels\/gta5\/vehicles_packed\/*/i, # DW 12-12-12 this is required to skip on HW5 as per bugstar 975449 /.*anim\/ingame\/*/i, # DW 13-12-12 this is required to skip on HW5 as per bugstar 964957 /.*anim\/cutscene\/*/i, # DW 09-01-13 as requested by Etienne /.*componentpeds/i, # DW 13-12-12 this is required to skip on HW5 as per bugstar 964957 /.*streamedpeds/i, # DW 13-12-12 this is required to skip on HW5 as per bugstar 964957 /.*levels\/gta5\/vehicles\/*/i, # DW 21-01-13 Luke asked for this to be removed from all AP2 builders ] # Do not change this message - since this is recognised by Cruise Control. SKIP_MSG = "Error : SKIPPED CL - NO BUILD HAS TAKEN PLACE" DELETE_MSG = "Error : SKIPPED CL - NO BUILD HAS TAKEN PLACE (FORCE)" #----------------------------------------------------------------------------- # # # # # --filename= # ... value ... # # ... value ... # # ... value ... # # ... value ... # # ... value ... # # ... value ... # # ... value ... # # ... value ... # # ... value ... # # ... value ... # # # # #Pasted from #----------------------------------------------------------------------------- # Parse local files with a wildcard # - searches wildcard passed in. # - return an array of local files which are independent files. #----------------------------------------------------------------------------- def parse_wildcard( wildcard, project, branchname ) puts "#{INFO} Parsing wildcard #{wildcard}" files = [] independent_path = OS::Path.normalise(project.branches[branchname].export) puts "independent_path #{independent_path}" ind_wildcard = OS::Path::combine( independent_path, wildcard ) puts "ind_wildcard #{ind_wildcard}" files = AssetBuild::Engine::get_files( ind_wildcard ) puts "\t#{INFO}Independent files wildcard: #{ind_wildcard}, #{files.size} files found." files.each do |filename| puts "\t#{INFO}#{filename}\n" end return files rescue Exception => ex $stderr.puts "Error :Unexpected exception parsing wildcards: " $stderr.puts "Error :\t#{ex.message}" ex.backtrace.each { |m| $stderr.puts "Error :\t#{m}"; } Process.exit -1 end #----------------------------------------------------------------------------- # Parse the Cruise Control XML file for modificatons. # Return an array of p4 filespecs. #----------------------------------------------------------------------------- def parse_cc_modifications( filename, project, branchname ) puts "#{INFO} Parsing #{filename}" files = [] begin if not File.exists?(filename) $stderr.puts "Error : #{filename} does not exist." return nil end #------------------------------------------------- #----------- Open Source xml --------------------- #------------------------------------------------- src_file = File.new( filename ) if not src_file $stderr.puts "Error : #{filename} could not open." return nil end #------------------------------------------------- #----------- Read XML doc ------------------------ #------------------------------------------------- xmldoc = REXML::Document.new( src_file ) if not xmldoc $stderr.puts "Error : #{filename} could not be opened as an XML document." return nil end independent_path = OS::Path.normalise(project.branches[branchname].export) puts "independent_path #{independent_path}" p4 = project.scm( ) p4.connect() unless ( p4.connected?() ) xmldoc.elements.each( 'ArrayOfModification/Modification' ) do |modification| folder_name = nil file_name = nil revision = nil type = nil modification.elements.each('FolderName') { |element| folder_name = element.text } modification.elements.each('FileName') { |element| file_name = element.text } modification.elements.each('Version') { |element| revision = element.text } modification.elements.each('Type') { |element| type = element.text } if folder_name.nil? or file_name.nil? or revision.nil? or type.nil? $stderr.puts "Error : #{filename} Invalid XML." return nil end local_file = OS::Path.normalise(p4.depot2local( "#{folder_name}/#{file_name}" )) puts "Local file #{local_file}" if ( local_file and local_file.include?( independent_path ) ) mod = "#{folder_name}/#{file_name}\##{revision}" puts "\t#{INFO} #{type} Parsed #{mod} " files << mod else puts "\t#{INFO} : #{local_file} will not be converted as it's path is not in #{independent_path}" if local_file and local_file.length > 0 end end rescue Exception => ex $stderr.puts "Error :Unexpected exception parsing modifications: " $stderr.puts "Error :\t#{ex.message}" ex.backtrace.each { |m| $stderr.puts "Error :\t#{m}"; } Process.exit -1 end files end #----------------------------------------------------------------------------- # Return true if this modification string should be skipped #----------------------------------------------------------------------------- def should_skip_modification(mod, skip_settings) skip_settings.each do |skip| skip.each do |machine, regex| if (mod=~regex) if (machine.downcase==ENV["COMPUTERNAME"].downcase) puts "#{INFO} #{mod} is matched with #{machine} by #{regex}" return false # this computer wishes to process this modification - return false - do not skip. else puts "#{INFO} #{mod} is skipped as it it matched with #{machine} by #{regex} which is not this machine #{ENV["COMPUTERNAME"]}" return true # another machine wishes to process this modification - return true - skip this modification. end end end end puts "Warning : no match for mod #{mod} - this shouldn't happen." return true # skip this modification it was not matched by any machine. end #----------------------------------------------------------------------------- # Return true if this modification string should be deleted #----------------------------------------------------------------------------- def should_delete_modification(mod, delete_settings) delete_settings.each do |regex| if (mod=~regex) puts "#{INFO} #{mod} is deleted as it it matched by #{regex}" return true end end return false end #----------------------------------------------------------------------------- # Kick off the conversion of files on this machine. # It uses a remote console to the assertbuilder engine, # It uses the 'build' command. The files are expected to be p4 filespecs only. # Return the logfile produced. # the wildcards are a way to tell the engine a summry of what is being built ( for reporting ) #----------------------------------------------------------------------------- def convert_files( files, is_rebuild = false, all_wildcards = nil, cc_project_name = nil, cc_rebuild = nil ) logfiles = nil begin puts "#{INFO} Converting #{all_wildcards} : #{files.length} files \t#{Time.now} " console = Assetbuild::Builder::Shell::Console.new( ) rebuild = is_rebuild ? "rebuild" : "" wildcards = all_wildcards ? "wildcard=#{all_wildcards}" : "" cc_project_name = cc_project_name ? "cc_project_name=#{cc_project_name}" : "" if (cc_rebuild==true) cc_rebuild = "cc_rebuild" else cc_rebuild = "" end command = "build #{rebuild} #{wildcards} #{cc_project_name} #{cc_rebuild} #{files.join(' ')}" puts "#{INFO} Issuing remote command to engine: #{command} \t#{Time.now}" puts "#{INFO} This is a REBUILD command." if is_rebuild puts "#{INFO} This is a regular build command." unless is_rebuild result = console.run_command( command ) puts "\t#{INFO} Build result = #{result} \t#{Time.now}" if (result =~ REGEXP_LOGFILE) logfiles = $1 puts "\t#{INFO} Logfiles : #{logfiles} \t#{Time.now}" end puts "#{INFO} Convert completed \t#{Time.now}" rescue Exception => ex $stderr.puts "Error :Unexpected exception converting files: " $stderr.puts "Error :\t#{ex.message}" ex.backtrace.each { |m| $stderr.puts "Error :\t#{m}"; } Process.exit -1 end logfiles end #----------------------------------------------------------------------------- # Parse the logfile, matching on errors and warning. # Return the number of errors and warnings. # Errors go to stderr # Warnings go to stdout. #----------------------------------------------------------------------------- def parse_logfiles( log_filenames_list, use_error_parser = true ) total_lines_read = 0 total_error_count = 0 total_warning_count = 0 log_filenames = log_filenames_list.split found_errors = false log_filenames.each_with_index do |log_filename, idx| put_info_in_report = false puts "#{INFO}Parsing XGE Logfile \##{idx+1}/#{log_filenames.length} #{log_filename}" puts "=========================================================================" c = Pipeline::Config::instance cmd = "#{OS::Path.combine(c.toolsbin, "errorparser.exe")}" args = "#{log_filename} #{ERROR_REGEXP} #{WARNING_REGEXP}" full_cmd = "#{cmd} #{args}" puts full_cmd status, stdout, stderr = systemu(full_cmd) err_count = 0 stderr.each do |error| err_count += 1 end # Caveat if ( stderr.length > 0 and not found_errors) $stderr.puts("Error :=====================================================================\n") $stderr.puts("Error :*** YOUR ASSET(S) MAY NOT HAVE CONVERTED CORRECTLY. - PLEASE READ *** \n") $stderr.puts("Error :=====================================================================\n") $stderr.puts("Error :- THESE ERRORS ARE FROM THE ASSETBUILDER / RAGEBUILDER.\n") $stderr.puts("Error :- THE ISSUE CAN BE WITH YOUR ASSET THAT YOU CHECKED IN.\n") $stderr.puts("Error :- SOMETIMES IT MAY RELATE TO OTHER BREAKAGES IN DEPENDENCIES ( SHADERS )\n") $stderr.puts("Error :- PLEASE CONTACT YOUR TOOLS TEAM.\n") found_errors = true end if ( stderr.length > 0 ) $stderr.puts("Error : #{log_filename} contains #{err_count} errors") $stderr.puts("Error :=====================================================================\n") stderr.each do |error| $stderr.puts "Error : #{error}" end end stdout.each do |out| $stdout.puts out if out =~ CHANGELIST_REGEXP puts "\t#{INFO_BLUE}Changelist submitted #{$2.to_s}" elsif out =~ PLATFORM_FILE_REGEXP puts "\t#{INFO_BLUE}#{$2.to_s} platform file: #{$3.to_s}" elsif out =~ ENGINE_MESSAGE_REGEXP puts "\t#{INFO_BLUE}Engine Message: #{$2.to_s}" end end if (err_count>0) puts "Errors where found in file #{log_filename}" else puts "No errors where found in file #{log_filename}" end total_error_count += err_count end return total_error_count, total_warning_count end #----------------------------------------------------------------------------- # Application entry point #----------------------------------------------------------------------------- error_count = 0 warning_count = 0 begin g_AppName = File::basename( __FILE__, '.rb' ) g_ProjectName = '' g_BranchName = '' g_Project = nil g_Config = Pipeline::Config.instance() #--------------------------------------------------------------------- # Parse Command Line. #--------------------------------------------------------------------- opts, trailing = OS::Getopt.getopts( OPTIONS ) if ( opts['help'] ) puts OS::Getopt.usage( OPTIONS, { 'files' => TRAILING_DESC } ) puts ("Press Enter to continue...") $stdin.getc( ) Process.exit( 1 ) end if ( ( 0 == trailing.size ) and ( nil == opts['filename'] ) ) then puts 'No trailing file arguments specified or modifications filename. Exiting.' puts OS::Getopt.usage( OPTIONS, { 'files' => TRAILING_DESC } ) Process.exit( 2 ) end g_ProjectName = ( nil == opts['project'] ) ? '' : opts['project'] project_exists = ( g_Config.projects.has_key?( g_ProjectName ) ) if ( not project_exists ) then puts OS::Getopt.usage( OPTIONS ) puts "\nError project: #{g_ProjectName} does not exist or its configuration is unreadable." Process.exit( 3 ) end g_Project = g_Config.projects[ g_ProjectName ] g_Project.load_config( ) if ( not g_Project.enabled ) then puts "\nError project: #{g_ProjectName} is not enabled on this machine. Re-run installer." Process.exit( 4 ) end g_BranchName = ( nil == opts['branch'] ) ? g_Project.default_branch : opts['branch'] if ( not g_Project.branches.has_key?( g_BranchName ) ) then puts "\nError project: #{g_ProjectName} does not have branch #{g_BranchName} defined." Process.exit( 5 ) end cc_projectname = opts['cc_projectname'] ? opts['cc_projectname'] : "-" cc_rebuild = opts['cc_rebuild'] ? opts['cc_rebuild'] : nil skip = opts['skip'] ? opts['skip'] : nil puts "#{INFO}Running Script #{g_AppName} for cc project #{cc_projectname}" #--------------------------------------------------------------------- # Parse the modifications xml file #--------------------------------------------------------------------- filename = opts['filename'] files = [] files += parse_cc_modifications( filename, g_Project, g_BranchName ) if filename is_rebuild = ( files.length == 0 ) # DW: Temp hack - as discussed with Luke regarding rebuilds # some code path must not be getting exercised and the result is that # the def need_convert? function ( for some reason - possibly timestamps ) # does not think some assets need converted. # This is experimental for now. #is_rebuild = true trailing.each do |wildcard| files += parse_wildcard( wildcard, g_Project, g_BranchName ) end # # Check if any files should be skipped # is_gta5 = ENV["RS_PROJECT"].downcase.include?("gta5") # only gta5 is interested in this skipping hack if (is_gta5) # # For all machines if the file can be matched in ADDITIONAL_DELETE_SETTINGS then remove it from the files to process files.delete_if do | file | delete_file = should_delete_modification(file, ADDITIONAL_DELETE_SETTINGS) puts "#{INFO} #{file} was force skipped." if (delete_file) delete_file end $stderr.puts(DELETE_MSG) if (files.length == 0) # # Then if desired *on this machine* we check if this file needs built here, skipped if delegated to another machine. if (skip) puts "#{INFO} Checking if files should be skipped" all_skipped = true files.delete_if do | file | skip_file = should_skip_modification(file, SKIP_SETTINGS) if (skip_file) puts "#{INFO} #{file} was skipped." else all_skipped = false end skip_file end $stderr.puts(SKIP_MSG) if (all_skipped) else puts "#{INFO} no skipping enabled no enabled by commandline" unless skip puts "#{INFO} no skipping enabled this project #{ENV["RS_PROJECT"]} is not configured for it" unless is_gta5 end end if ( files and files.length > 0 ) then all_wildcards = trailing.join(",") #--------------------------------------------------------------------- # Convert #--------------------------------------------------------------------- puts "#{INFO} Converting files for #{cc_projectname} \t#{Time.now}" log_filenames = convert_files( files, is_rebuild, all_wildcards, cc_projectname, cc_rebuild ) puts "#{INFO} Finished converting files \t#{Time.now}" #--------------------------------------------------------------------- # Read log file and match errors and warnings. #--------------------------------------------------------------------- puts "#{INFO} Parsing logfiles \t#{Time.now}" error_count, warning_count = parse_logfiles( log_filenames ) if log_filenames puts "#{INFO} Finished parsing logfiles \t#{Time.now}" else puts "#{INFO} Warning: no files were processed" end rescue Exception => ex $stderr.puts "Error :Unhandled exception: #{ex.message}" $stderr.puts "Error :Backtrace:" ex.backtrace.each { |m| $stderr.puts "Error :\t#{m}" } Process.exit -1 end ret_code = ( error_count > 0 ) ? -1 : 0 puts "#{INFO} Script exiting with #{ret_code} since error_count is #{error_count} \t#{Time.now}" Process.exit ret_code