# File:: update_build_state.rb
# Description:: updates build state spreadsheet. Interprets a binary, a log file from the game and a version file in order
# to populate an existing spreadsheet with this data.
#
# Author:: Derek Ward <derek.ward@rockstarnorth.com>
# Date:: 07 October 2010
#

#-----------------------------------------------------------------------------
# Uses
#-----------------------------------------------------------------------------
require 'win32ole'
require 'pipeline/os/path'
require 'time'
require 'pipeline/os/getopt'
require 'pipeline/os/path'
require 'fileutils'
include FileUtils
require 'dl'
include Pipeline
require 'pipeline/log/log'
require 'util/ExcelTools/excel_tools'

#-----------------------------------------------------------------------------
# Constants
#-----------------------------------------------------------------------------

OPTIONS =	[
			[ '--memvis', 		'-w', Getopt::REQUIRED, 'mem visualize folder' ],
			[ '--xlsfilename', 	'-x', Getopt::REQUIRED, 'destination xls filename' ],
			[ '--datafilename', 	'-d', Getopt::REQUIRED, 'source filename from which data is extracted for insertion' ],
			[ '--versionfilename', 	'-v', Getopt::REQUIRED, 'filename from which version is extracted for insertion' ],
			[ '--ps3bin', 		'-p', Getopt::REQUIRED, 'where the ps3bin.exe executable exists' ],
			[ '--xbox360bin', 	'-q', Getopt::REQUIRED, 'where the dumpbin.exe executable exists' ], # NB. VS environment needs to be established to run this.
			[ '--bin', 		'-b', Getopt::REQUIRED, 'the binary' ],
			[ '--deltas', 		'-dt', Getopt::BOOLEAN,	'whacks a line in at the bottom of the spreadsheet that is a delta of the last two results.' ],
			[ '--disablecheckin', 	'-c', Getopt::BOOLEAN,	'prevent checkin ( for development )' ]			
		]

INFO			= "[colourise=black]INFO_MSG: "
INFO_BLUE		= "[colourise=blue]INFO_MSG: "

# --- keys that appear as output from ps3bin -dsi ( update if ever changes )

KEY_TEXT		= "Text (Code)"
KEY_DATA		= "Data"
KEY_RODATA		= "RO-Data"
KEY_BSS			= "BSS (Uninitialised data)"
KEY_TOTAL		= "Total"

KEYS 			= [ KEY_TEXT, KEY_DATA, KEY_RODATA, KEY_BSS, KEY_TOTAL ]

# --- relates respectively to what is found in the XLS spreadsheet

BUILDSTATE_STRINGS 	= [ 	"Date", "Build", 
				"CL Game", "CL Rage", 
				"Text Size", "Data Size", "RO-Data Size", "BSS Size", "Total", 
				"GH Total", "GH Used", "GH Left", 
				"Default", "Animation", "Streaming", "World", "Gameplay", "FX", "Rendering", "Physics", "Audio", "Network", "System", "Scaleform", "Script", "Resource", "Debug", "Bounds", "Pools", "AI", "Process", "Pathfinding" 
			]

# --- a tag in the version.xml file

VERSION_TAG		= "[VERSION_NUMBER]"

# --- regexps for searching checkin comment of executable

REGEXP_CL_GAME 		= /(.*)gameCL([#|\s|:]*)(\d+)/i	
REGEXP_CL_RAGE 		= /(.*)rageCL([#|\s|:]*)(\d+)/i
REGEXP_DATE		= /(\d+\/\d+\/\d+)/
REGEXP_VERSION		= /(\d+\.*\d*)/ 

#-----------------------------------------------------------------------------
# Implementation
#-----------------------------------------------------------------------------

# monkey patch the string class
class String   
	def convert_base(from, to)     
		self.to_i(from).to_s(to)   
	end 
end 


	#-----------------------------------------------------------------------------
	# Code
	#-----------------------------------------------------------------------------
	begin
		#-------------------------------------------------------------------------
		# Entry-Point
		#-------------------------------------------------------------------------

		g_AppName = File::basename( __FILE__, '.rb' )
		g_xls_filename = ''
		g_Config = Pipeline::Config.instance()
		g_Log = Log.new( 'exceltools' )

		#-------------------------------------------------------------------------
		# Parse & validate Command Line
		#-------------------------------------------------------------------------
		opts, trailing = OS::Getopt.getopts( OPTIONS )
		if ( opts['help'] ) then
			puts OS::Getopt.usage( OPTIONS )
			response = message_box( "#{g_AppName} will exit.",g_AppName, BUTTONS_OK, ICON_QUESTION)
			exit( 1 )
		end

		g_mem_vis 		= ( nil == opts['memvis'] ) 		? nil  	: opts['memvis']
		g_xls_filename 		= ( nil == opts['xlsfilename'] ) 	? nil  	: opts['xlsfilename']
		g_data_filename 	= ( nil == opts['datafilename'] ) 	? nil 	: opts['datafilename']
		g_version_filename 	= ( nil == opts['versionfilename'] ) 	? nil 	: opts['versionfilename']
		g_ps3bin_filename 	= ( nil == opts['ps3bin'] ) 		? nil 	: opts['ps3bin']
		g_xbox360bin_filename 	= ( nil == opts['xbox360bin'] ) 	? nil 	: opts['xbox360bin']
		g_bin_filename 		= ( nil == opts['bin'] ) 		? nil 	: opts['bin']
		g_enable_checkin	= ( nil == opts['disablecheckin'] ) 	? true 	: false
		g_deltas		= ( nil == opts['deltas'] ) 		? false : true
		
		if g_data_filename and not File.exist? g_data_filename
			$stderr.puts "Error : #{g_data_filename} does not exist."
			exit( 2 )
		end

		if g_version_filename and not File.exist? g_version_filename
			$stderr.puts "Error : #{g_version_filename} does not exist."
			exit( 3 )
		end

		if g_ps3bin_filename and not File.exist? g_ps3bin_filename 
			$stderr.puts "Error : #{g_ps3bin_filename} does not exist."
			exit( 4 )
		end

		if g_xbox360bin_filename and not File.exist? g_xbox360bin_filename 
			$stderr.puts "Error : #{g_xbox360bin_filename} does not exist."
			exit( 6 )
		end
		
		if g_bin_filename and not File.exist? g_bin_filename 
			$stderr.puts "Error : #{g_bin_filename} does not exist."
			exit( 5 )
		end

		#-------------------------------------------------------------------------
		# --- Sync, check out file ---
		#-------------------------------------------------------------------------

		puts "#{INFO_BLUE} Creating p4"
		p4 	= SCM::Perforce::create( g_Config.sc_server, g_Config.sc_username, g_Config.sc_workspace )
		p4_rage = SCM::Perforce::create( g_Config.sc_rage_server, g_Config.sc_rage_username, g_Config.sc_rage_workspace )
		
		puts "#{INFO_BLUE} Connecting to p4 servers"
		raise Exception if not p4
		p4.connect( )
		raise Exception if not p4.connected?

		raise Exception if not p4_rage
		p4_rage.connect( )
		raise Exception if not p4_rage.connected?

#Why did I do this - this is incorrect - keeping it around until I think if there was a good reason for this --
# maybe I thought it wasn;t part of the label - but it is in the label.
#		if (g_version_filename)
#			puts "#{INFO_BLUE} Syncing to version file."
#			depot_filename = p4.local2depot(g_version_filename)
#			p4.run_sync( g_version_filename )
#		end

		if (g_ps3bin_filename)
			puts "#{INFO_BLUE} Syncing to ps3bin file."
			depot_filename = p4_rage.local2depot( g_ps3bin_filename )
			p4_rage.run_sync( g_ps3bin_filename )
		end

		#-------------------------------------------------------------------------
		# Derive CLs used to build executable from it's checkin comment.
		#-------------------------------------------------------------------------

		if (g_bin_filename)
			depot_filename = p4.local2depot( g_bin_filename )
			puts "#{INFO_BLUE} EXE= #{depot_filename}"
			change = p4.run_changes( "-m1", "-l", "#{depot_filename}@#{p4.client}" )
			puts "#{INFO_BLUE} EXE CL= #{change[0]['change']}"
			description = change[0]['desc']	
			if (description)
				puts "#{INFO_BLUE} EXE descripton= #{description}"
				
				trailing[2] = $3 if (trailing[2].nil? and description =~ REGEXP_CL_GAME)
				if trailing[2]
					puts "#{INFO_BLUE} EXE Built with CL #{trailing[2]} from game p4 server" 
				else
					puts "Warning: The CL (game P4) could not be derived from the checkin comment #{description}"
					trailing[2] = "unknown"
					#puts "Warning: No update will be done."
					#exit( 0 )
				end
				
				trailing[3] = $3 if (trailing[3].nil? and description =~ REGEXP_CL_RAGE)
				if trailing[3]
					puts "#{INFO_BLUE} EXE Built with CL #{trailing[3]} from rage p4 server" 
				else
					puts "Warning: The CL (rage p4) could not be derived from the checkin comment #{description}"
					trailing[3] = "unknown"
					#puts "Warning: No update will be done."
					#exit( 0 )
				end
			end
		end
			
		puts "#{INFO_BLUE} Creating CL"
		change_id = p4.create_changelist( "Automated Build of 'build state' via Cruise Control @ #{Time.now} on #{ENV["COMPUTERNAME"]}." )
		raise Exception if change_id.nil?
		puts "#{INFO_BLUE}Changelist #{change_id} is created"

		puts "#{INFO_BLUE} Syncing to memory XLS file."
		depot_filename = p4.local2depot(g_xls_filename)
		p4.run_sync( depot_filename )
		
		puts "#{INFO_BLUE}Checking out #{depot_filename} in CL #{change_id.to_s}"
		puts "Checking out #{depot_filename} in CL #{change_id.to_s}"
		p4.run_edit( '-c', change_id.to_s, depot_filename )	
		p4.run_reopen( '-c', change_id.to_s, depot_filename )
		
		p4.run_edit( '-c', change_id.to_s, g_mem_vis )	
		p4.run_reopen( '-c', change_id.to_s, g_mem_vis )
		
		#-------------------------------------------------------------------------
		# --- Append the row ---
		#-------------------------------------------------------------------------

		puts "AppendRow to #{g_xls_filename}"
		if (g_data_filename)

			#-------------------------------------------------------------------------
			# --- Search for data in log.
			#-------------------------------------------------------------------------

			buildstate_regexp = [ ]

			puts "#{INFO_BLUE} building regexps"
			
			buildstate_strings = BUILDSTATE_STRINGS

			buildstate_strings.each do |str|
				buildstate_regexp << Regexp.new("(.*)BuildState(.*)#{str}=(.*)$")
			end

			buildstate_regexp.each do |regexp|
				puts "#{INFO_BLUE} Regexp #{regexp.to_s}"	
			end

			puts "#{INFO_BLUE} opening #{g_data_filename}"
			File.open(g_data_filename) do |file| 
				file.each_line do |line|
					buildstate_regexp.each_with_index do |regexp, idx|											
						if line =~ regexp
							puts "#{INFO_BLUE} Matched #{line}"
							trailing[idx] = $3 							
						end
					end					
				end
			end
	
			#-------------------------------------------------------------------------
			#--- Derive missing data not found in log, starting here with time/date and 
			#--- version.
			#-------------------------------------------------------------------------

			trailing[0] = Time.now.strftime("%Y/%m/%d") if trailing[0].nil?
			
			if trailing[1].nil?
				puts "#{INFO_BLUE} Opening #{g_version_filename}"
				File.open(g_version_filename) do |file|
					
					capture = false
					file.each_line do |line|
						line.gsub!(/[\n]+/, ""); 
						if capture 
							puts "#{INFO_BLUE} Version read #{line}"
							trailing[1] = "dev build #{line} (automated)"
							break
						end

						capture = true if line.include?VERSION_TAG								
					end
				end	
			end

			#-------------------------------------------------------------------------
			# Derive executable memory segmentation sizes from executable.
			#-------------------------------------------------------------------------

			error_count = 0

			if (	trailing[4].nil? or 
				trailing[5].nil? or 
				trailing[6].nil? or 
				trailing[7].nil? or 
				trailing[8].nil?)

				if (g_ps3bin_filename)
					cmd = "#{g_ps3bin_filename} #{g_bin_filename} -dsi"
					puts "#{INFO_BLUE} Executing #{cmd}...\n"
					status, stdout, stderr = systemu(cmd)
					puts "#{INFO_BLUE} \n...Completed"

					$stderr.puts( cmd ) if ( stderr.length > 0 )

					stderr.each do |err|
						error_count += 1
						$stderr.puts "Error: #{err}"
					end

					stdout.each_with_index do |out, idx|									
						puts out
						if (idx==1)
							vals = out.split

							hash = {}
							KEYS.each_with_index do |key,idx|
								puts "#{INFO_BLUE} Bin stats : #{key} = #{vals[idx]}" 
								hash[key] = vals[idx]
							end

							trailing[4] = hash[KEY_TEXT]
							trailing[5] = hash[KEY_DATA]
							trailing[6] = hash[KEY_RODATA]
							trailing[7] = hash[KEY_BSS]
							trailing[8] = hash[KEY_TOTAL]
						end
					end
				elsif (g_xbox360bin_filename)
					cmd = "#{g_xbox360bin_filename} #{g_bin_filename} /Summary"
					puts "#{INFO_BLUE} Executing #{cmd}...\n"
					status, stdout, stderr = systemu(cmd)
					puts "#{INFO_BLUE} \n...Completed"

					$stderr.puts( cmd ) if ( stderr.length > 0 )

					stderr.each do |err|
						error_count += 1
						$stderr.puts "Error: #{err}"
					end

					total = 0 
					stdout.each_with_index do |out, idx|																				
						if out =~ /^\s*([0-9A-F]*)\s*\.(.*)$/i
							hex_size = $1
							section = $2							
							size = hex_size.convert_base(16,10).to_i
							
							if (section=="text")
								trailing[4] = size
								puts "Section #{section} is #{size}" 
							elsif (section=="data")												
								trailing[5] = size
								puts "Section #{section} is #{size}"
							elsif (section=="rdata")												
								trailing[6] = size
								puts "Section #{section} is #{size}"
							else
								puts "Warning: section #{section} #{size} is ignored"
							end
							
							# DW: NB. only text & data are read here, as they are really the only sections that make much sense to the end user in the final report
							# anybody interested or knowledgable about these sections though can analyse by hand.
							# The XEX signing process hopefully will take care of many optimisations to such sections.?!
							
							#rdata section
							#In MS Windows PE file terms, the .rdata section is used for at least four things:
							#- First, in EXEs produced by Microsoft Link, the .rdata section holds the debug directory;
							#- Second, the description string - intended to hold a useful text string describing the file;
							#- Third, describe GUIDs objects used in OLE programming.
							#- Forth, place to put the TLS (Thread Local Storage) directory.


							#Pdata Section
							#The .pdata section contains an array of function table entries used for exception handling and is pointed to by the exception table entry in the image data directory. 
							

							#reloc section
							#In MS Windows PE file terms, the .reloc section holds a table of base relocations - a list of places in the image where the difference between the linker-assumed load address and the actual load address needs to be taken into account.
							#A base relocation is an adjustment to an instruction or initialized variable value (locations of static variables, string literals and so on).
							#The relocation directory is a sequence of chunks where each chunk contains the relocation information for 4 KB of the image.
							#If the loader can load the image at the linker's preferred base address, the loader ignores the relocation information in this section. 
							
							#trailing[7] = hash[KEY_BSS]													
							total += size
						else
							puts "skipped #{out}"
						end
					end				
					
					trailing[8] = total
					puts "Total is #{total}"
				end
			end

			buildstate_strings.each_with_index do |str, idx|
				trailing[idx] = "0" if trailing[idx].nil? 
				puts "#{INFO_BLUE} #{str} #{trailing[idx]}"
			end	
		end		
		
		#-------------------------------------------------------------------------
		# -- Finally append this row to the spreadsheet & compute a delta.
		#-------------------------------------------------------------------------

		last_row = ExcelTools::Row(g_xls_filename, -1)
		puts "#{INFO_BLUE} last row was #{last_row.join(" ")}"
		if (last_row[0] =~ /DELTA/i )
			puts "#{INFO_BLUE} Delta found, delete delta row"
			ExcelTools::DeleteRow(g_xls_filename, -1)	
		end

		# DW: It has been noticed that the delete row above sometimes doesn't delete the row. 
		# Not sure why, perhaps saving the document on another thread, but it;s not the time or place to rewrite this, so...
		2.times do			
			Kernel.sleep(5) 
		
			last_row = ExcelTools::Row(g_xls_filename, -1)
			puts "#{INFO_BLUE} last row was #{last_row.join(" ")}"
			if (last_row[0] =~ /DELTA/i )
				puts "#{INFO_BLUE} Error : Delta found, delete delta row, this should not happen, I'll delete it anyway"
				ExcelTools::DeleteRow(g_xls_filename, -1)	
			else
				break
			end
		end

		#now replace the entry if it is the same version on the same day
		last_row = ExcelTools::Row(g_xls_filename, -1)
		puts "#{INFO_BLUE} Previous row was #{last_row.join(" ")}"
		
		if (last_row[0] =~ /DELTA/i )
			puts "#{INFO_BLUE} Error : A delta row has persisted in the xls document, depsite attempts to remove it. This document will need to be hand edited. (#{g_xls_filename})"
		end

		last_date = last_row[0] =~ REGEXP_DATE ? $1 : nil
		this_date = trailing[0] =~ REGEXP_DATE ? $1 : nil

		puts "#{INFO_BLUE}last_date #{last_date}" 
		puts "#{INFO_BLUE}this_date #{this_date}" 

		if (last_date and this_date and (last_date == this_date) )
			puts "#{INFO_BLUE}Last row was of the same date"
			#strip out version numbers - comments are not important

			num1 = last_row[1] =~ REGEXP_VERSION ? $1 : nil
			num2 = trailing[1] =~ REGEXP_VERSION ? $1 : nil
					
			if (num1 and num2 and (num1 == num2) )
				puts "#{INFO_BLUE} Deleting previous row since it is of same date #{last_row[0]} and version #{last_row[1]}"
				ExcelTools::DeleteRow(g_xls_filename, -1)
			end
		end

		puts "#{INFO_BLUE}Appending row"
		ExcelTools::AppendRow(g_xls_filename, trailing)

		if (g_deltas)
			begin
				puts "#{INFO_BLUE} Delta is being computed"
				last_rows = [ ExcelTools::Row(g_xls_filename, -1), ExcelTools::Row(g_xls_filename, -2) ]
				deltas = [ "DELTA", "", "", "" ]
				length_to_skip = deltas.length - 1		

				last_rows[0].each_with_index do |num, idx|			
					if (idx > length_to_skip)	
						delta = num-last_rows[1][idx]					
						deltas << delta
					end
				end
			
				puts "#{INFO_BLUE} Delta is #{deltas.join(" ")}"
				ExcelTools::AppendRow(g_xls_filename, deltas)
			rescue Exception => ex

				$stderr.puts "Error : #{g_AppName} unhandled exception: #{ex.message}"
				$stderr.puts "Call stack:"
				$stderr.puts ex.backtrace.join( "\n\t" )
			end	
		end

		#-------------------------------------------------------------------------
		# --- Submit changes
		#-------------------------------------------------------------------------

		puts "#{INFO_BLUE}Reverting unchanged files #{change_id}"
		p4.run_revert( '-a', '-c', change_id.to_s, '//...')

		files = p4.run_opened( '-c', change_id.to_s )
		raise Exception if files.nil?

		puts "#{INFO_BLUE}There are #{files.size} files to submit."
		files.each do |file|
			puts "#{INFO_BLUE}#{file['depotFile']} has been updated and will be submitted."			
		end

		if ( g_enable_checkin )
			if ( files.size > 0 )		
				puts "#{INFO_BLUE}Submitting file currently in #{change_id}"	
				submit_result = p4.run_submit( '-c', change_id.to_s )
				puts submit_result.to_s
			elsif ( 0 == files.size )
				puts "#{INFO_BLUE}Deleting #{change_id} no files changed."
				p4.run_change('-d', change_id.to_s)
			end
		else
			puts "#{INFO_BLUE}Checkin is disabled the CL is pending."	
		end

	rescue Exception => ex

		puts "#{g_AppName} unhandled exception: #{ex.message}"
		puts "Call stack:"
		puts ex.backtrace.join( "\n\t" )
	end		

# Exceltools.rb