519 lines
19 KiB
Ruby
Executable File
519 lines
19 KiB
Ruby
Executable File
# File:: update_build_state.rb
|
|
# Description:: updates build state spreadsheet. Interprets a binary, a log file from the game and a version file in order
|
|
# to populate an existing spreadsheet with this data.
|
|
#
|
|
# Author:: Derek Ward <derek.ward@rockstarnorth.com>
|
|
# Date:: 07 October 2010
|
|
#
|
|
|
|
#-----------------------------------------------------------------------------
|
|
# Uses
|
|
#-----------------------------------------------------------------------------
|
|
require 'win32ole'
|
|
require 'pipeline/os/path'
|
|
require 'time'
|
|
require 'pipeline/os/getopt'
|
|
require 'pipeline/os/path'
|
|
require 'fileutils'
|
|
include FileUtils
|
|
require 'dl'
|
|
include Pipeline
|
|
require 'pipeline/log/log'
|
|
require 'util/ExcelTools/excel_tools'
|
|
|
|
#-----------------------------------------------------------------------------
|
|
# Constants
|
|
#-----------------------------------------------------------------------------
|
|
|
|
OPTIONS = [
|
|
[ '--memvis', '-w', Getopt::REQUIRED, 'mem visualize folder' ],
|
|
[ '--xlsfilename', '-x', Getopt::REQUIRED, 'destination xls filename' ],
|
|
[ '--datafilename', '-d', Getopt::REQUIRED, 'source filename from which data is extracted for insertion' ],
|
|
[ '--versionfilename', '-v', Getopt::REQUIRED, 'filename from which version is extracted for insertion' ],
|
|
[ '--ps3bin', '-p', Getopt::REQUIRED, 'where the ps3bin.exe executable exists' ],
|
|
[ '--xbox360bin', '-q', Getopt::REQUIRED, 'where the dumpbin.exe executable exists' ], # NB. VS environment needs to be established to run this.
|
|
[ '--bin', '-b', Getopt::REQUIRED, 'the binary' ],
|
|
[ '--deltas', '-dt', Getopt::BOOLEAN, 'whacks a line in at the bottom of the spreadsheet that is a delta of the last two results.' ],
|
|
[ '--disablecheckin', '-c', Getopt::BOOLEAN, 'prevent checkin ( for development )' ]
|
|
]
|
|
|
|
INFO = "[colourise=black]INFO_MSG: "
|
|
INFO_BLUE = "[colourise=blue]INFO_MSG: "
|
|
|
|
# --- keys that appear as output from ps3bin -dsi ( update if ever changes )
|
|
|
|
KEY_TEXT = "Text (Code)"
|
|
KEY_DATA = "Data"
|
|
KEY_RODATA = "RO-Data"
|
|
KEY_BSS = "BSS (Uninitialised data)"
|
|
KEY_TOTAL = "Total"
|
|
|
|
KEYS = [ KEY_TEXT, KEY_DATA, KEY_RODATA, KEY_BSS, KEY_TOTAL ]
|
|
|
|
# --- relates respectively to what is found in the XLS spreadsheet
|
|
|
|
BUILDSTATE_STRINGS = [ "Date", "Build",
|
|
"CL Game", "CL Rage",
|
|
"Text Size", "Data Size", "RO-Data Size", "BSS Size", "Total",
|
|
"GH Total", "GH Used", "GH Left",
|
|
"Default", "Animation", "Streaming", "World", "Gameplay", "FX", "Rendering", "Physics", "Audio", "Network", "System", "Scaleform", "Script", "Resource", "Debug", "Bounds", "Pools", "AI", "Process", "Pathfinding"
|
|
]
|
|
|
|
# --- a tag in the version.xml file
|
|
|
|
VERSION_TAG = "[VERSION_NUMBER]"
|
|
|
|
# --- regexps for searching checkin comment of executable
|
|
|
|
REGEXP_CL_GAME = /(.*)gameCL([#|\s|:]*)(\d+)/i
|
|
REGEXP_CL_RAGE = /(.*)rageCL([#|\s|:]*)(\d+)/i
|
|
REGEXP_DATE = /(\d+\/\d+\/\d+)/
|
|
REGEXP_VERSION = /(\d+\.*\d*)/
|
|
|
|
#-----------------------------------------------------------------------------
|
|
# Implementation
|
|
#-----------------------------------------------------------------------------
|
|
|
|
# monkey patch the string class
|
|
class String
|
|
def convert_base(from, to)
|
|
self.to_i(from).to_s(to)
|
|
end
|
|
end
|
|
|
|
|
|
#-----------------------------------------------------------------------------
|
|
# Code
|
|
#-----------------------------------------------------------------------------
|
|
begin
|
|
#-------------------------------------------------------------------------
|
|
# Entry-Point
|
|
#-------------------------------------------------------------------------
|
|
|
|
g_AppName = File::basename( __FILE__, '.rb' )
|
|
g_xls_filename = ''
|
|
g_Config = Pipeline::Config.instance()
|
|
g_Log = Log.new( 'exceltools' )
|
|
|
|
#-------------------------------------------------------------------------
|
|
# Parse & validate Command Line
|
|
#-------------------------------------------------------------------------
|
|
opts, trailing = OS::Getopt.getopts( OPTIONS )
|
|
if ( opts['help'] ) then
|
|
puts OS::Getopt.usage( OPTIONS )
|
|
response = message_box( "#{g_AppName} will exit.",g_AppName, BUTTONS_OK, ICON_QUESTION)
|
|
exit( 1 )
|
|
end
|
|
|
|
g_mem_vis = ( nil == opts['memvis'] ) ? nil : opts['memvis']
|
|
g_xls_filename = ( nil == opts['xlsfilename'] ) ? nil : opts['xlsfilename']
|
|
g_data_filename = ( nil == opts['datafilename'] ) ? nil : opts['datafilename']
|
|
g_version_filename = ( nil == opts['versionfilename'] ) ? nil : opts['versionfilename']
|
|
g_ps3bin_filename = ( nil == opts['ps3bin'] ) ? nil : opts['ps3bin']
|
|
g_xbox360bin_filename = ( nil == opts['xbox360bin'] ) ? nil : opts['xbox360bin']
|
|
g_bin_filename = ( nil == opts['bin'] ) ? nil : opts['bin']
|
|
g_enable_checkin = ( nil == opts['disablecheckin'] ) ? true : false
|
|
g_deltas = ( nil == opts['deltas'] ) ? false : true
|
|
|
|
if g_data_filename and not File.exist? g_data_filename
|
|
$stderr.puts "Error : #{g_data_filename} does not exist."
|
|
exit( 2 )
|
|
end
|
|
|
|
if g_version_filename and not File.exist? g_version_filename
|
|
$stderr.puts "Error : #{g_version_filename} does not exist."
|
|
exit( 3 )
|
|
end
|
|
|
|
if g_ps3bin_filename and not File.exist? g_ps3bin_filename
|
|
$stderr.puts "Error : #{g_ps3bin_filename} does not exist."
|
|
exit( 4 )
|
|
end
|
|
|
|
if g_xbox360bin_filename and not File.exist? g_xbox360bin_filename
|
|
$stderr.puts "Error : #{g_xbox360bin_filename} does not exist."
|
|
exit( 6 )
|
|
end
|
|
|
|
if g_bin_filename and not File.exist? g_bin_filename
|
|
$stderr.puts "Error : #{g_bin_filename} does not exist."
|
|
exit( 5 )
|
|
end
|
|
|
|
#-------------------------------------------------------------------------
|
|
# --- Sync, check out file ---
|
|
#-------------------------------------------------------------------------
|
|
|
|
puts "#{INFO_BLUE} Creating p4"
|
|
p4 = SCM::Perforce::create( g_Config.sc_server, g_Config.sc_username, g_Config.sc_workspace )
|
|
p4_rage = SCM::Perforce::create( g_Config.sc_rage_server, g_Config.sc_rage_username, g_Config.sc_rage_workspace )
|
|
|
|
puts "#{INFO_BLUE} Connecting to p4 servers"
|
|
raise Exception if not p4
|
|
p4.connect( )
|
|
raise Exception if not p4.connected?
|
|
|
|
raise Exception if not p4_rage
|
|
p4_rage.connect( )
|
|
raise Exception if not p4_rage.connected?
|
|
|
|
#Why did I do this - this is incorrect - keeping it around until I think if there was a good reason for this --
|
|
# maybe I thought it wasn;t part of the label - but it is in the label.
|
|
# if (g_version_filename)
|
|
# puts "#{INFO_BLUE} Syncing to version file."
|
|
# depot_filename = p4.local2depot(g_version_filename)
|
|
# p4.run_sync( g_version_filename )
|
|
# end
|
|
|
|
if (g_ps3bin_filename)
|
|
puts "#{INFO_BLUE} Syncing to ps3bin file."
|
|
depot_filename = p4_rage.local2depot( g_ps3bin_filename )
|
|
p4_rage.run_sync( g_ps3bin_filename )
|
|
end
|
|
|
|
#-------------------------------------------------------------------------
|
|
# Derive CLs used to build executable from it's checkin comment.
|
|
#-------------------------------------------------------------------------
|
|
|
|
if (g_bin_filename)
|
|
depot_filename = p4.local2depot( g_bin_filename )
|
|
puts "#{INFO_BLUE} EXE= #{depot_filename}"
|
|
change = p4.run_changes( "-m1", "-l", "#{depot_filename}@#{p4.client}" )
|
|
puts "#{INFO_BLUE} EXE CL= #{change[0]['change']}"
|
|
description = change[0]['desc']
|
|
if (description)
|
|
puts "#{INFO_BLUE} EXE descripton= #{description}"
|
|
|
|
trailing[2] = $3 if (trailing[2].nil? and description =~ REGEXP_CL_GAME)
|
|
if trailing[2]
|
|
puts "#{INFO_BLUE} EXE Built with CL #{trailing[2]} from game p4 server"
|
|
else
|
|
puts "Warning: The CL (game P4) could not be derived from the checkin comment #{description}"
|
|
trailing[2] = "unknown"
|
|
#puts "Warning: No update will be done."
|
|
#exit( 0 )
|
|
end
|
|
|
|
trailing[3] = $3 if (trailing[3].nil? and description =~ REGEXP_CL_RAGE)
|
|
if trailing[3]
|
|
puts "#{INFO_BLUE} EXE Built with CL #{trailing[3]} from rage p4 server"
|
|
else
|
|
puts "Warning: The CL (rage p4) could not be derived from the checkin comment #{description}"
|
|
trailing[3] = "unknown"
|
|
#puts "Warning: No update will be done."
|
|
#exit( 0 )
|
|
end
|
|
end
|
|
end
|
|
|
|
puts "#{INFO_BLUE} Creating CL"
|
|
change_id = p4.create_changelist( "Automated Build of 'build state' via Cruise Control @ #{Time.now} on #{ENV["COMPUTERNAME"]}." )
|
|
raise Exception if change_id.nil?
|
|
puts "#{INFO_BLUE}Changelist #{change_id} is created"
|
|
|
|
puts "#{INFO_BLUE} Syncing to memory XLS file."
|
|
depot_filename = p4.local2depot(g_xls_filename)
|
|
p4.run_sync( depot_filename )
|
|
|
|
puts "#{INFO_BLUE}Checking out #{depot_filename} in CL #{change_id.to_s}"
|
|
puts "Checking out #{depot_filename} in CL #{change_id.to_s}"
|
|
p4.run_edit( '-c', change_id.to_s, depot_filename )
|
|
p4.run_reopen( '-c', change_id.to_s, depot_filename )
|
|
|
|
p4.run_edit( '-c', change_id.to_s, g_mem_vis )
|
|
p4.run_reopen( '-c', change_id.to_s, g_mem_vis )
|
|
|
|
#-------------------------------------------------------------------------
|
|
# --- Append the row ---
|
|
#-------------------------------------------------------------------------
|
|
|
|
puts "AppendRow to #{g_xls_filename}"
|
|
if (g_data_filename)
|
|
|
|
#-------------------------------------------------------------------------
|
|
# --- Search for data in log.
|
|
#-------------------------------------------------------------------------
|
|
|
|
buildstate_regexp = [ ]
|
|
|
|
puts "#{INFO_BLUE} building regexps"
|
|
|
|
buildstate_strings = BUILDSTATE_STRINGS
|
|
|
|
buildstate_strings.each do |str|
|
|
buildstate_regexp << Regexp.new("(.*)BuildState(.*)#{str}=(.*)$")
|
|
end
|
|
|
|
buildstate_regexp.each do |regexp|
|
|
puts "#{INFO_BLUE} Regexp #{regexp.to_s}"
|
|
end
|
|
|
|
puts "#{INFO_BLUE} opening #{g_data_filename}"
|
|
File.open(g_data_filename) do |file|
|
|
file.each_line do |line|
|
|
buildstate_regexp.each_with_index do |regexp, idx|
|
|
if line =~ regexp
|
|
puts "#{INFO_BLUE} Matched #{line}"
|
|
trailing[idx] = $3
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
#-------------------------------------------------------------------------
|
|
#--- Derive missing data not found in log, starting here with time/date and
|
|
#--- version.
|
|
#-------------------------------------------------------------------------
|
|
|
|
trailing[0] = Time.now.strftime("%Y/%m/%d") if trailing[0].nil?
|
|
|
|
if trailing[1].nil?
|
|
puts "#{INFO_BLUE} Opening #{g_version_filename}"
|
|
File.open(g_version_filename) do |file|
|
|
|
|
capture = false
|
|
file.each_line do |line|
|
|
line.gsub!(/[\n]+/, "");
|
|
if capture
|
|
puts "#{INFO_BLUE} Version read #{line}"
|
|
trailing[1] = "dev build #{line} (automated)"
|
|
break
|
|
end
|
|
|
|
capture = true if line.include?VERSION_TAG
|
|
end
|
|
end
|
|
end
|
|
|
|
#-------------------------------------------------------------------------
|
|
# Derive executable memory segmentation sizes from executable.
|
|
#-------------------------------------------------------------------------
|
|
|
|
error_count = 0
|
|
|
|
if ( trailing[4].nil? or
|
|
trailing[5].nil? or
|
|
trailing[6].nil? or
|
|
trailing[7].nil? or
|
|
trailing[8].nil?)
|
|
|
|
if (g_ps3bin_filename)
|
|
cmd = "#{g_ps3bin_filename} #{g_bin_filename} -dsi"
|
|
puts "#{INFO_BLUE} Executing #{cmd}...\n"
|
|
status, stdout, stderr = systemu(cmd)
|
|
puts "#{INFO_BLUE} \n...Completed"
|
|
|
|
$stderr.puts( cmd ) if ( stderr.length > 0 )
|
|
|
|
stderr.each do |err|
|
|
error_count += 1
|
|
$stderr.puts "Error: #{err}"
|
|
end
|
|
|
|
stdout.each_with_index do |out, idx|
|
|
puts out
|
|
if (idx==1)
|
|
vals = out.split
|
|
|
|
hash = {}
|
|
KEYS.each_with_index do |key,idx|
|
|
puts "#{INFO_BLUE} Bin stats : #{key} = #{vals[idx]}"
|
|
hash[key] = vals[idx]
|
|
end
|
|
|
|
trailing[4] = hash[KEY_TEXT]
|
|
trailing[5] = hash[KEY_DATA]
|
|
trailing[6] = hash[KEY_RODATA]
|
|
trailing[7] = hash[KEY_BSS]
|
|
trailing[8] = hash[KEY_TOTAL]
|
|
end
|
|
end
|
|
elsif (g_xbox360bin_filename)
|
|
cmd = "#{g_xbox360bin_filename} #{g_bin_filename} /Summary"
|
|
puts "#{INFO_BLUE} Executing #{cmd}...\n"
|
|
status, stdout, stderr = systemu(cmd)
|
|
puts "#{INFO_BLUE} \n...Completed"
|
|
|
|
$stderr.puts( cmd ) if ( stderr.length > 0 )
|
|
|
|
stderr.each do |err|
|
|
error_count += 1
|
|
$stderr.puts "Error: #{err}"
|
|
end
|
|
|
|
total = 0
|
|
stdout.each_with_index do |out, idx|
|
|
if out =~ /^\s*([0-9A-F]*)\s*\.(.*)$/i
|
|
hex_size = $1
|
|
section = $2
|
|
size = hex_size.convert_base(16,10).to_i
|
|
|
|
if (section=="text")
|
|
trailing[4] = size
|
|
puts "Section #{section} is #{size}"
|
|
elsif (section=="data")
|
|
trailing[5] = size
|
|
puts "Section #{section} is #{size}"
|
|
elsif (section=="rdata")
|
|
trailing[6] = size
|
|
puts "Section #{section} is #{size}"
|
|
else
|
|
puts "Warning: section #{section} #{size} is ignored"
|
|
end
|
|
|
|
# DW: NB. only text & data are read here, as they are really the only sections that make much sense to the end user in the final report
|
|
# anybody interested or knowledgable about these sections though can analyse by hand.
|
|
# The XEX signing process hopefully will take care of many optimisations to such sections.?!
|
|
|
|
#rdata section
|
|
#In MS Windows PE file terms, the .rdata section is used for at least four things:
|
|
#- First, in EXEs produced by Microsoft Link, the .rdata section holds the debug directory;
|
|
#- Second, the description string - intended to hold a useful text string describing the file;
|
|
#- Third, describe GUIDs objects used in OLE programming.
|
|
#- Forth, place to put the TLS (Thread Local Storage) directory.
|
|
|
|
|
|
#Pdata Section
|
|
#The .pdata section contains an array of function table entries used for exception handling and is pointed to by the exception table entry in the image data directory.
|
|
|
|
|
|
#reloc section
|
|
#In MS Windows PE file terms, the .reloc section holds a table of base relocations - a list of places in the image where the difference between the linker-assumed load address and the actual load address needs to be taken into account.
|
|
#A base relocation is an adjustment to an instruction or initialized variable value (locations of static variables, string literals and so on).
|
|
#The relocation directory is a sequence of chunks where each chunk contains the relocation information for 4 KB of the image.
|
|
#If the loader can load the image at the linker's preferred base address, the loader ignores the relocation information in this section.
|
|
|
|
#trailing[7] = hash[KEY_BSS]
|
|
total += size
|
|
else
|
|
puts "skipped #{out}"
|
|
end
|
|
end
|
|
|
|
trailing[8] = total
|
|
puts "Total is #{total}"
|
|
end
|
|
end
|
|
|
|
buildstate_strings.each_with_index do |str, idx|
|
|
trailing[idx] = "0" if trailing[idx].nil?
|
|
puts "#{INFO_BLUE} #{str} #{trailing[idx]}"
|
|
end
|
|
end
|
|
|
|
#-------------------------------------------------------------------------
|
|
# -- Finally append this row to the spreadsheet & compute a delta.
|
|
#-------------------------------------------------------------------------
|
|
|
|
last_row = ExcelTools::Row(g_xls_filename, -1)
|
|
puts "#{INFO_BLUE} last row was #{last_row.join(" ")}"
|
|
if (last_row[0] =~ /DELTA/i )
|
|
puts "#{INFO_BLUE} Delta found, delete delta row"
|
|
ExcelTools::DeleteRow(g_xls_filename, -1)
|
|
end
|
|
|
|
# DW: It has been noticed that the delete row above sometimes doesn't delete the row.
|
|
# Not sure why, perhaps saving the document on another thread, but it;s not the time or place to rewrite this, so...
|
|
2.times do
|
|
Kernel.sleep(5)
|
|
|
|
last_row = ExcelTools::Row(g_xls_filename, -1)
|
|
puts "#{INFO_BLUE} last row was #{last_row.join(" ")}"
|
|
if (last_row[0] =~ /DELTA/i )
|
|
puts "#{INFO_BLUE} Error : Delta found, delete delta row, this should not happen, I'll delete it anyway"
|
|
ExcelTools::DeleteRow(g_xls_filename, -1)
|
|
else
|
|
break
|
|
end
|
|
end
|
|
|
|
#now replace the entry if it is the same version on the same day
|
|
last_row = ExcelTools::Row(g_xls_filename, -1)
|
|
puts "#{INFO_BLUE} Previous row was #{last_row.join(" ")}"
|
|
|
|
if (last_row[0] =~ /DELTA/i )
|
|
puts "#{INFO_BLUE} Error : A delta row has persisted in the xls document, depsite attempts to remove it. This document will need to be hand edited. (#{g_xls_filename})"
|
|
end
|
|
|
|
last_date = last_row[0] =~ REGEXP_DATE ? $1 : nil
|
|
this_date = trailing[0] =~ REGEXP_DATE ? $1 : nil
|
|
|
|
puts "#{INFO_BLUE}last_date #{last_date}"
|
|
puts "#{INFO_BLUE}this_date #{this_date}"
|
|
|
|
if (last_date and this_date and (last_date == this_date) )
|
|
puts "#{INFO_BLUE}Last row was of the same date"
|
|
#strip out version numbers - comments are not important
|
|
|
|
num1 = last_row[1] =~ REGEXP_VERSION ? $1 : nil
|
|
num2 = trailing[1] =~ REGEXP_VERSION ? $1 : nil
|
|
|
|
if (num1 and num2 and (num1 == num2) )
|
|
puts "#{INFO_BLUE} Deleting previous row since it is of same date #{last_row[0]} and version #{last_row[1]}"
|
|
ExcelTools::DeleteRow(g_xls_filename, -1)
|
|
end
|
|
end
|
|
|
|
puts "#{INFO_BLUE}Appending row"
|
|
ExcelTools::AppendRow(g_xls_filename, trailing)
|
|
|
|
if (g_deltas)
|
|
begin
|
|
puts "#{INFO_BLUE} Delta is being computed"
|
|
last_rows = [ ExcelTools::Row(g_xls_filename, -1), ExcelTools::Row(g_xls_filename, -2) ]
|
|
deltas = [ "DELTA", "", "", "" ]
|
|
length_to_skip = deltas.length - 1
|
|
|
|
last_rows[0].each_with_index do |num, idx|
|
|
if (idx > length_to_skip)
|
|
delta = num-last_rows[1][idx]
|
|
deltas << delta
|
|
end
|
|
end
|
|
|
|
puts "#{INFO_BLUE} Delta is #{deltas.join(" ")}"
|
|
ExcelTools::AppendRow(g_xls_filename, deltas)
|
|
rescue Exception => ex
|
|
|
|
$stderr.puts "Error : #{g_AppName} unhandled exception: #{ex.message}"
|
|
$stderr.puts "Call stack:"
|
|
$stderr.puts ex.backtrace.join( "\n\t" )
|
|
end
|
|
end
|
|
|
|
#-------------------------------------------------------------------------
|
|
# --- Submit changes
|
|
#-------------------------------------------------------------------------
|
|
|
|
puts "#{INFO_BLUE}Reverting unchanged files #{change_id}"
|
|
p4.run_revert( '-a', '-c', change_id.to_s, '//...')
|
|
|
|
files = p4.run_opened( '-c', change_id.to_s )
|
|
raise Exception if files.nil?
|
|
|
|
puts "#{INFO_BLUE}There are #{files.size} files to submit."
|
|
files.each do |file|
|
|
puts "#{INFO_BLUE}#{file['depotFile']} has been updated and will be submitted."
|
|
end
|
|
|
|
if ( g_enable_checkin )
|
|
if ( files.size > 0 )
|
|
puts "#{INFO_BLUE}Submitting file currently in #{change_id}"
|
|
submit_result = p4.run_submit( '-c', change_id.to_s )
|
|
puts submit_result.to_s
|
|
elsif ( 0 == files.size )
|
|
puts "#{INFO_BLUE}Deleting #{change_id} no files changed."
|
|
p4.run_change('-d', change_id.to_s)
|
|
end
|
|
else
|
|
puts "#{INFO_BLUE}Checkin is disabled the CL is pending."
|
|
end
|
|
|
|
rescue Exception => ex
|
|
|
|
puts "#{g_AppName} unhandled exception: #{ex.message}"
|
|
puts "Call stack:"
|
|
puts ex.backtrace.join( "\n\t" )
|
|
end
|
|
|
|
# Exceltools.rb |