#
# File:: Engine.rb
# Description:: Main autobuild engine component.
#
# Author:: David Muir <david.muir@rockstarnorth.com>
# Date:: 25 June 2008
#

#-----------------------------------------------------------------------------
# Uses
#-----------------------------------------------------------------------------
require "source/builder/reporting/report_ragebuilder"
require "source/builder/reporting/report_changelist"
require "source/builder/reporting/report_rebuild"
require "source/builder/reporting/report_writer"
require "source/builder/shell/console"
require "source/builder/shell/command"
require "source/builder/shell/command_queue"
require 'pipeline/config/projects'
require 'pipeline/content/content_core'
require 'pipeline/log/log'
require 'pipeline/os/file'
require 'pipeline/os/path'
require 'pipeline/os/start'
require 'pipeline/projectutil/data_content'
require 'pipeline/projectutil/data_convert'
require 'pipeline/projectutil/data_convert_map_dependencies'
require 'pipeline/projectutil/data_get_latest'
require 'pipeline/projectutil/tools_get_latest'
require 'pipeline/scm/monitor'
require 'pipeline/scm/perforce'
require 'pipeline/util/environment'
require 'pipeline/util/rexml_write_fix'
require 'pipeline/win32/stdin_nonblock_gets'
require 'pipeline/util/memory_profiler'
require 'log4r/outputter/fileoutputter'
include Pipeline
require 'socket'
require 'fileutils' 
require 'pp'
require 'xml'

#-----------------------------------------------------------------------------
# Implementation
#-----------------------------------------------------------------------------

module AssetBuild

	#
	# Used for logging just a single build triggered by Cruise Control
	class EachBuildOutputter < Log4r::FileOutputter
		
		def initialize( _name, hash={} )
			super( _name, hash )
			
			@filename_base = OS::Path.get_basename( @filename )
			@filename_ext = OS::Path.get_extension( @filename )
		end
			
		def roll()			
			@out.close()
			@out = File.new( @filename, "w" )
		end
	end
		
	#
	# == Description
	# The main asset building engine.  This class handles all of the asset
	# build logic.  There are utility methods (typically in the 
	# Pipeline::ProjectUtil module) that actually handle most of the work.  
	# 
	# Ideally this is just glue between those components and our generic
	# Perforce SCM monitoring utility.
	#
	class Engine
		include Assetbuild::Builder::Shell::CommandQueue
		
		#---------------------------------------------------------------------
		# Constants
		#---------------------------------------------------------------------
		VERSION			= 220

		ERROR_REGEXP		= "\"^(.*)(Error\\s|Error\\s:|Error:\\s)(.*)$\""
		WARNING_REGEXP		= "\"^(.*)(Warning:\\s)(.*)$\""
		
		# Regexs that match bundled information when cruise control sends commands to the engine.
		# * these should not match the names of modification files.
		# * the modifications list is stripped of these commands.
		REGEX_CC_REBUILD = /^cc_rebuild/
		REGEX_WILDCARD = /^wildcard=(.+)/
		REGEX_CC_PROJECTNAME = /^cc_project_name=(.+)/
		REGEX_REBUILD = /^rebuild$/				
		
		MAX_ADDITIONAL_DESCRIPTIONS = 50
		
		# If this character is placed in the wildcard filespec of files to build, 
		# then no recursion will be used when finding files.
		NO_RECURSE_CHARACTER = "<" # this character needs to be an invalid filename character.

		# To be derived from something... this needs unhardcoded when I get time			
		EVENT_FILENAME 		= "N:\\RSGEDI\\Builders\\CruiseControl\\live\\assetbuilder_#{ENV["RS_PROJECT"]}_$(branch)_hw\\events.xml"
		EVENT_FILENAME_USER	= "N:\\RSGEDI\\Builders\\CruiseControl\\live\\assetbuilder_#{ENV["RS_PROJECT"]}_$(branch)_user\\events.xml"				
			
		#---------------------------------------------------------------------
		# Attributes
		#---------------------------------------------------------------------	
		
		attr_reader :project
		attr_reader :branch
		attr_reader :config
		attr_reader :running
		attr_reader :rebuild
		attr_reader :num_builds
		attr_reader :commands
		
		# Generic options Hash object containing the following option keys:
		#   disabled: Bool (default: false)
		#   checkin : Bool (default: true)
		#   reports : Bool (default: output path exists status, for local testing)
		attr_reader :options
		
		#---------------------------------------------------------------------
		# Public Methods
		#---------------------------------------------------------------------

		def initialize( project, branch, config )
			puts "Version: #{VERSION/100.0}"
			throw ArgumentError.new( 'Invalid project object (#{project.class}).' ) \
				unless ( ( nil != project ) and ( project.is_a?( Project ) ) )
			throw ArgumentError.new( 'Invalid branch object (#{branch.class}).' ) \
				unless ( branch.is_a?( Branch ) )
			
			@lock_problem_counter = 0
			@build_lock = Mutex.new()
			@project = project
			@branch = branch
			@config = config
			@shell_log = Log.new( 'shell' )			
			@build_log = Log.new( 'build')
			
			# add an each build outputter to the root log.
			@each_build_outputter = EachBuildOutputter.new( get_main_log(), :filename => get_main_log_each_build() )
			Pipeline::LogSystem::instance.rootlog.add( @each_build_outputter )
								
			@running = true
			@options = {
				'checkin' => true,
				'reports' => ::File::directory?( config.report.path ),
			}
			@num_builds = 0
			
			pipeline_config = Pipeline::Config.instance( )
			if not pipeline_config.user.is_builder_server()
				@events_filename = EVENT_FILENAME_USER
			else			
				@events_filename = EVENT_FILENAME
			end
			
			@branch.in_env do |env|
				@events_filename = env.subst( @events_filename )
			end
			
			log_info "Using Event filename #{@events_filename}"

			env_init( )
			@p4 = @project.scm( )
			commands_init( )
			init_queue( @commands, ENV['COMPUTERNAME'], @config.command_queue.port )
			@shell = Assetbuild::Builder::Shell::Console.new( @commands, ENV['COMPUTERNAME'], @config.command_queue.port )
		end
		
		# Helper method...
		# puts the msg and log it too.
		def log_info( msg, cc = false )			
			msg = "#{Time.now.strftime('%Y-%m-%d %H:%M:%S')} #{msg}"
			
			# make sure cc messages are always pumped out despite logging level fix for asset builder slowness
#			config = Pipeline::Config.instance( )
#			log_level= @build_log.level
#			if cc
#				@build_log.level
#				msg = "INFO_MSG: #{msg}"				
#				@build_log.level = Logger::INFO	
#			end
			@build_log.info( msg )
			puts "INFO_MSG: #{msg}" # DW - this should remain - otherwise we can't see twhat is going on when we have set the logging level to warning for optimising the speed fo the assetbuilder temporarily.
			
#			if cc
#				@build_log.level = log_level 
#			end
		end		
															
		def commands_init( )
		
			# Initialise command Proc objects
			sync_proc 		= Proc.new do |arg| 
							event_time = report_event_started("sync", arg, false, "-");									
							result = sync( arg ); 
							report_event_finished(result, event_time);
							result;
						  end
			view_proc 		= Proc.new do |arg| view_files( arg ); end
			rebuild_proc 		= Proc.new do |arg| rebuild_files( arg ); end
			reload_proc 		= Proc.new do reload( ); end
			set_proc 		= Proc.new do |arg, val| set_option( arg, val ); end
			get_proc 		= Proc.new do |arg| get_option( arg ); end
			config_proc 		= Proc.new do get_config( ); end
			queue_proc 		= Proc.new do get_queue( ); end
			status_proc 		= Proc.new do get_status( ); end
			exit_proc 		= Proc.new do exit( ); end
			pause_proc 		= Proc.new do ; end
			resume_proc 		= Proc.new do ; end
			build_proc 		= Proc.new do |arg| build( arg ); end
			help_proc 		= Proc.new do help_func(); end
			threads_proc		= Proc.new do threads_func(); end
			close_logfiles_proc 	= Proc.new do close_logfiles(); end
			reopen_logfiles_proc 	= Proc.new do reopen_logfiles(); end
			print_class_counts_proc = Proc.new do print_class_counts( ); end
			toggle_mem_prof_proc 	= Proc.new do toggle_mem_prof(); end					
			print_mem_proc		= Proc.new do print_mem(); end					
			purge_cache_proc	= Proc.new do 
							event_time = report_event_started("purge_cache", "", false, "-");									
							result = purge_cache(); 
							report_event_finished(result, event_time);
							result;
			 			  end					
				
			@commands = [
				Assetbuild::Builder::Shell::Command.new( 	'sync', 
										Assetbuild::Builder::Shell::Command::NO_SHORTCUTS, 
										'Sync Perforce path.',
										[ Assetbuild::Builder::Shell::Param.new( 'path', String ) ], 
										sync_proc),
				Assetbuild::Builder::Shell::Command.new( 	'view', 
										Assetbuild::Builder::Shell::Command::NO_SHORTCUTS,
										'View assets from wildcard or changelist ID.',
										[ Assetbuild::Builder::Shell::Param.new( 'wildcard/changelist', String ) ], 
										view_proc ),
				Assetbuild::Builder::Shell::Command.new( 	'rebuild', 
										Assetbuild::Builder::Shell::Command::NO_SHORTCUTS,
										'Rebuild asset with wildcard or changelist ID.',
										[ Assetbuild::Builder::Shell::Param.new( 'wildcard/changelist', String ) ], 
										rebuild_proc ),
				Assetbuild::Builder::Shell::Command.new( 	'reload', 
										Assetbuild::Builder::Shell::Command::NO_SHORTCUTS, 
										'Reload Asset Builder configuration file.',
										Assetbuild::Builder::Shell::Param::NONE, 
										reload_proc ),
				Assetbuild::Builder::Shell::Command.new( 	'pause', 
										Assetbuild::Builder::Shell::Command::NO_SHORTCUTS, 
										'Pause Perforce monitor.',
										Assetbuild::Builder::Shell::Param::NONE, 
										pause_proc ),
				Assetbuild::Builder::Shell::Command.new( 	'resume', 
										Assetbuild::Builder::Shell::Command::NO_SHORTCUTS, 
										'Resume Perforce monitor.',
										Assetbuild::Builder::Shell::Param::NONE, 
										resume_proc ),
				Assetbuild::Builder::Shell::Command.new( 	'set', 
										Assetbuild::Builder::Shell::Command::NO_SHORTCUTS, 
										'Set Asset Builder variable.',
										[ Assetbuild::Builder::Shell::Param.new( 'option', String ), Assetbuild::Builder::Shell::Param.new( 'value', String ) ], 
										set_proc ),
				Assetbuild::Builder::Shell::Command.new( 	'get', 
										Assetbuild::Builder::Shell::Command::NO_SHORTCUTS, 
										'Get Asset Builder variable.',
										[ Assetbuild::Builder::Shell::Param.new( 'option', String ) ], 
										get_proc ),
				Assetbuild::Builder::Shell::Command.new( 	'config', 
										[ 'c' ], 
										'Display Asset Builder configuration information.',
										Assetbuild::Builder::Shell::Param::NONE, 
										config_proc ),
				Assetbuild::Builder::Shell::Command.new( 	'queue', 
										[ 'qu' ],
										'Display Asset Builder command queue information.',
										Assetbuild::Builder::Shell::Param::NONE, 
										queue_proc ),
				Assetbuild::Builder::Shell::Command.new( 	'status', 
										[ 's' ],
										'Display Asset Builder status information.',
										Assetbuild::Builder::Shell::Param::NONE, 
										status_proc ),
				Assetbuild::Builder::Shell::Command.new( 	'build', 
										Assetbuild::Builder::Shell::Command::NO_SHORTCUTS,
										'Build independent assets listed with p4 filespec.',
										[ Assetbuild::Builder::Shell::Param.new( 'files', String ) ], 
										build_proc ),
				Assetbuild::Builder::Shell::Command.new(	'exit', 
										[ 'quit', 'q' ], 
										'Exit Asset Builder',
										Assetbuild::Builder::Shell::Param::NONE, 
										exit_proc ),
				Assetbuild::Builder::Shell::Command.new( 	'help', 
										Assetbuild::Builder::Shell::Command::NO_SHORTCUTS, 
										'Display help',
										Assetbuild::Builder::Shell::Param::NONE, 
										help_proc ),
				Assetbuild::Builder::Shell::Command.new( 	'close_logfiles', 
										Assetbuild::Builder::Shell::Command::NO_SHORTCUTS, 
										'Closes log files',
										Assetbuild::Builder::Shell::Param::NONE, 
										close_logfiles_proc ),				
				Assetbuild::Builder::Shell::Command.new( 	'reopen_logfiles', 
										Assetbuild::Builder::Shell::Command::NO_SHORTCUTS, 
										'Reopens log files',
										Assetbuild::Builder::Shell::Param::NONE, 
										reopen_logfiles_proc ),				
				Assetbuild::Builder::Shell::Command.new( 	'mem_print_class_count', 
										[ 'mem', 'm' ], 
										'Debug helper for memory leaks.',
										Assetbuild::Builder::Shell::Param::NONE, 
										print_class_counts_proc ),
				Assetbuild::Builder::Shell::Command.new( 	'mem_log', 
										Assetbuild::Builder::Shell::Command::NO_SHORTCUTS, 
										'Toggle leak tracking.',
										Assetbuild::Builder::Shell::Param::NONE, 
										toggle_mem_prof_proc ),
				Assetbuild::Builder::Shell::Command.new( 	'mem_print', 
										Assetbuild::Builder::Shell::Command::NO_SHORTCUTS, 
										'Display memory used.',
										Assetbuild::Builder::Shell::Param::NONE, 
										print_mem_proc ),
				Assetbuild::Builder::Shell::Command.new( 	'threads', 
										Assetbuild::Builder::Shell::Command::NO_SHORTCUTS, 
										'Display threads',
										Assetbuild::Builder::Shell::Param::NONE, 
										threads_proc ),
				Assetbuild::Builder::Shell::Command.new( 	'purge_cache', 
										Assetbuild::Builder::Shell::Command::NO_SHORTCUTS, 
										'Purge Cache',
										Assetbuild::Builder::Shell::Param::NONE, 
										purge_cache_proc )																			
				]	
		end
					
		#
		# Toggle mem profiling ( leak tracking ) 
		#		
		def toggle_mem_prof()
			if ( @mem_prof_started )
				@mem_prof_started = false
				puts "stopping leak tracking"
				Pipeline::Util::MemoryProfiler::stop( )
			else
				@mem_prof_started = true
				puts "starting leak tracking"
				Pipeline::Util::MemoryProfiler::start( )				
			end
			Assetbuild::Builder::Shell::CommandResultCompleted.new( "completed toggle_mem_prof" )
		end			
		
		#
		# Print memory
		#
		def print_mem()
			Pipeline::Util::MemoryProfiler::start( ) if @mem_prof_started
			Pipeline::Util::MemoryProfiler::snapshot( )
			Pipeline::Util::MemoryProfiler::stop( ) if @mem_prof_started
			
			Assetbuild::Builder::Shell::CommandResultCompleted.new( "completed print_mem" )
		end
		
		
		#
		# purge cache files
		#
		def purge_cache()
			begin
				#convert = Pipeline::ConvertSystem::instance()	
			 	#convert.setup( @project )				
				#puts "Purging #{convert.cache_root}"								
				#FileUtils.rm_f(convert.cache_root)
				
				puts "Purging cache..."
				# ideally I should derive the cache directory form the convert system - but it doesn;t work - no time to investigate
				cache_folder = OS::Path.combine(ENV['RS_PROJROOT'],"cache")
				puts "Purging #{cache_folder}..."
				FileUtils.rm_rf(cache_folder)						
				Assetbuild::Builder::Shell::CommandResultCompleted.new( "Completed Purge Cache in #{cache_folder}" )
			rescue P4Exception => ex
				puts ex.message
				result = CommandResultError.new( "Purge Cache failed #{ex.message}" )	
			end
		end									
		
		#
		# Print class counts
		#
		def print_class_counts( )
			Pipeline::Util::MemoryProfiler::start( ) if @mem_prof_started
			result = Pipeline::Util::MemoryProfiler::snapshot( :count ) 
			Pipeline::Util::MemoryProfiler::stop( ) if @mem_prof_started
			
			Assetbuild::Builder::Shell::CommandResultCompleted.new( "completed print_class_counts" )
		end
		
		#
		# Shell sync command handler.
		#
		# This command can do a Perforce sync on a particular known-token
		# (tools, common) or a local file/directory path.
		#
		def sync( param )
			message = ''
			if ( :tools_config == param or :tools_config.to_s == param ) then
				message = "\nSyncing #{@project.uiname} tools config...\n"
				puts message
				
				tools = ProjectUtil::tools_config_get_latest( ) do
					|client_file, synced, errors|
					if ( synced ) then
						message << "\tSynced #{client_file}\n"						
					else
						message << "Failed to sync #{client_file}\n"
						puts "\t#{message}"
						errors.each do |error| @build_log.error( error ); end
					end
				end
				message << "#{tools.size} files synced.\n"

			elsif ( :tools_bin == param or :tools_bin.to_s == param ) then
				message = "\nSyncing #{@project.uiname} tools bin...\n"
				puts message
				
				tools = ProjectUtil::tools_bin_get_latest( ) do
					|client_file, synced, errors|
					if ( synced ) then
						message << "\tSynced #{client_file}\n"						
					else
						message << "Failed to sync #{client_file}\n"
						puts "\t#{message}"
						errors.each do |error| @build_log.error( error ); end
					end
				end
				message << "#{tools.size} files synced.\n"
				
			elsif ( :tools == param or :tools.to_s == param ) then
				message = "\nSyncing #{@project.uiname} tools...\n"
				puts message
				
				tools = ProjectUtil::tools_get_latest( ) do
					|client_file, synced, errors|
					if ( synced ) then
						message << "\tSynced #{client_file}\n"						
					else
						message << "Failed to sync #{client_file}\n"
						puts "\t#{message}"
						errors.each do |error| @build_log.error( error ); end
					end
				end
				message << "#{tools.size} files synced.\n"

			elsif ( :shaders == param or :shaders.to_s == param ) then				
				message = "\nSyncing #{@project.uiname} shaders...\n"
				puts message
				
				shaders = ProjectUtil::data_get_latest_build_shaders( @project, @branch.name ) do
					|client_file, synced, errors|
					
					if ( synced ) then
						message << "Synced #{client_file}\n"
					else
						message << "Failed to sync #{client_file}\n"
						errors.each do |error| @build_log.error( error ); end
					end
				end
				message << "#{shaders.size} files synced."
				
			elsif ( :tools_lib_util_ragebuilder == param or :tools_lib_util_ragebuilder.to_s == param ) then
				message = "\nSyncing #{@project.uiname} tools/lib/util/ragebuilder...\n"
				puts message
				
				tools = ProjectUtil::tools_lib_util_ragebuilder_get_latest( ) do
					|client_file, synced, errors|
					if ( synced ) then
						message << "\tSynced #{client_file}\n"						
					else
						message << "Failed to sync #{client_file}\n"
						puts "\t#{message}"
						errors.each do |error| @build_log.error( error ); end
					end
				end
				message << "#{tools.size} files synced.\n"


			elsif ( :platform_data == param or :platform_data.to_s == param ) then
				message = "\nSyncing #{@project.uiname} platform data...\n"
				puts message
				
				plaform_data = ProjectUtil::data_get_latest_platform_build_labelled( @project, @branch.name  ) do
					|client_file, synced, errors|
					if ( synced ) then
						message << "\tSynced #{client_file}\n"						
					else
						message << "Failed to sync #{client_file}\n"
						puts "\t#{message}"
						errors.each do |error| @build_log.error( error ); end
					end
				end
				message << "#{plaform_data.size} files synced.\n"
						
			elsif ( :common == param or :common.to_s == param ) then				
				message = "\nSyncing #{@project.uiname} common...\n"
				puts message
				
				shaders = ProjectUtil::data_get_latest_build_common( @project, @branch.name ) do
					|client_file, synced, errors|
					
					if ( synced ) then
						message << "Synced #{client_file}\n"
					else
						message << "Failed to sync #{client_file}\n"
						errors.each do |error| @build_log.error( error ); end
					end
				end
				message << "#{shaders.size} files synced."
			elsif ( :assets == param or :assets.to_s == param ) then
				message = "\nSyncing #{@project.uiname} assets...\n"
				puts message
				
				assets = ProjectUtil::data_get_latest_assets( @project, @branch.name  ) do
					|client_file, synced, errors|
					if ( synced ) then
						message << "\tSynced #{client_file}\n"						
					else
						message << "Failed to sync #{client_file}\n"
						puts "\t#{message}"
						errors.each do |error| @build_log.error( error ); end
					end
				end
				message << "#{assets.size} files synced.\n"
			elsif ( :assets_metadata == param or :assets_metadata.to_s == param ) then
				message = "\nSyncing #{@project.uiname} assets_metadata...\n"
				puts message
				
				assets_metadata = ProjectUtil::data_get_latest_assets_metadata( @project, @branch.name  ) do
					|client_file, synced, errors|
					if ( synced ) then
						message << "\tSynced #{client_file}\n"						
					else
						message << "Failed to sync #{client_file}\n"
						puts "\t#{message}"
						errors.each do |error| @build_log.error( error ); end
					end
				end
				message << "#{assets_metadata.size} files synced.\n"
			elsif ( :assets_maps_parenttxds == param or :assets_maps_parenttxds.to_s == param ) then
				message = "\nSyncing #{@project.uiname} assets_maps_parenttxds...\n"
				puts message
				
				assets_maps_parenttxds = ProjectUtil::data_get_latest_assets_maps_parenttxds( @project, @branch.name  ) do
					|client_file, synced, errors|
					if ( synced ) then
						message << "\tSynced #{client_file}\n"						
					else
						message << "Failed to sync #{client_file}\n"
						puts "\t#{message}"
						errors.each do |error| @build_log.error( error ); end
					end
				end
				message << "#{assets_maps_parenttxds.size} files synced.\n"										
			elsif ( :assets_processed == param or :assets_processed.to_s == param ) then
				message = "\nSyncing #{@project.uiname} assets_processed...\n"
				puts message
				
				assets_processed = ProjectUtil::data_get_latest_assets_processed( @project, @branch.name  ) do
					|client_file, synced, errors|
					if ( synced ) then
						message << "\tSynced #{client_file}\n"						
					else
						message << "Failed to sync #{client_file}\n"
						puts "\t#{message}"
						errors.each do |error| @build_log.error( error ); end
					end
				end
				message << "#{assets_processed.size} files synced.\n"					
			else
				# Arbitary Perforce path.
				throw RuntimeError.new( "Not implemented fetching normal Perforce paths (yet!)" )
			end
			
			Assetbuild::Builder::Shell::CommandResultCompleted.new( message )
		end	
		
		
		#
		#
		#
		def set_option( opt, val )
			# Fail if its not a recognised option.
			return CommandResultError.new( "Option #{opt} not known." ) \
				unless ( @options.has_key?( opt ) )
			
			# Set option, attempt to coerce into the current option type.
			# Falling back to String if required.
			case @options[opt].class
			when Integer
				@options[opt] = val.to_i
			when FalseClass
				@options[opt] = ( val ? true : false )
			when TrueClass
				@options[opt] = ( val ? true : false )
			else
				@options[opt] = val.to_s
			end
			Assetbuild::Builder::Shell::CommandResultCompleted.new( 'ok' )
		end
		
		#
		#
		#
		def get_option( opt )
			# Fail if its not a recognised option.
			return CommandResultError.new( "Option #{opt} not known." ) \
				unless ( @options.has_key?( opt ) )
			
			# Get option; returning as String.
			Assetbuild::Builder::Shell::CommandResultCompleted.new( @options[opt].to_s )
		end
		
		#
		# Handles the special mangled wildcard format for rebuilding non recursively
		# really this should be a new command but since it would take a fair bit of 
		# work and testing to do this have taken the 'easy' option since the design of the engine doesn;t 
		# lend itself to this too easily.
		#
		def self.get_files( wildcard )
			if (wildcard.include?(NO_RECURSE_CHARACTER))		
				OS::FindEx::find_files(wildcard.gsub(NO_RECURSE_CHARACTER,""))
			else
				OS::FindEx::find_files_recurse(wildcard)
			end		
		end
				
		#
		# View independent data files from changelist or wildcard string.
		#
		def view_files( wildcard )
			message = ''
			if ( wildcard =~ /^([0-9]+)$/ ) then
				cl = $1
				message << "Changelist #{cl} files:\n"
				begin
					export_files = []
					@p4.connect() unless ( @p4.connected?() )
					local_files = @p4.files_from_changelist( cl )
					local_files.each do |filename|
						local_filename = OS::Path::normalise( @p4.depot2local( filename ) )
						next unless ( @project.branches[@branch.name].is_export_file?( local_filename ) )
						export_files << local_filename
					end
					export_files.each do |filename|
						message << "\t#{filename}\n"
						puts filename
					end
					
					message << "End.\n\n"
					return CommandResultCompleted.new( message )
				rescue P4Exception => ex
					Assetbuild::Builder::Shell::CommandResultError.new( "Changelist #{cl} was not found." )	
				end				
			else
				export = @env.subst( @project.branches[@branch.name].export )
				ind_wildcard = @env.subst( OS::Path::combine( export, wildcard ) )
				files = Engine::get_files( ind_wildcard )
				
				message = "Export files wildcard: #{ind_wildcard}, #{files.size} files found."
				files.each do |filename|
					message << "\t#{filename}\n"
				end
				message << "End.\n\n"
				return Assetbuild::Builder::Shell::CommandResultCompleted.new( message )
			end		
		end
				
		#
		# Helper method to prettify a rebuild duration for events.
		#
		def prettify_rebuild_duration(build_duration)
			difference = build_duration
			seconds    =  difference % 60
			difference = (difference - seconds) / 60
			minutes    =  difference % 60
			difference = (difference - minutes) / 60
			hours      =  difference % 24
			difference = (difference - hours)   / 24
			days       =  difference % 7
			weeks      = (difference - days)    /  7								

			build_duration = ""											
			build_duration += " #{weeks.to_i} weeks" if weeks > 0
			build_duration += " #{days.to_i} days" if days > 0
			build_duration += " #{hours.to_i} hrs" if hours > 0
			build_duration += " #{minutes.to_i} mins" if minutes > 0
			build_duration += " #{seconds.to_i} secs" if seconds > 0
			
			build_duration
		end
		
		
		EVENT_KEYS = [ "status", "build_date", "sort_date", "build_duration", "project", "description", "cc_project_name" ]
		
		#
		# Publish the event that has started
		#
		def report_event_started(name, wildcard, cc_rebuild = false, cc_project_name = "")
			begin
				log_info "=== Event #{name} started ==="
				time = Time.now
				event_time_formatted = time.strftime("%H:%M:%S %d-%m")
				sort_date = time.strftime("%Y-%m-%d %H:%M:%S")

				xmldoc = LibXML::XML::Document::file( @events_filename )
				if (xmldoc)
					events = xmldoc.find_first("//events")
					if (not events)
						events = XML::Node.new( 'events' )
						xmldoc.root << events
					end	

					event_node = XML::Node.new( 'event' )
					EVENT_KEYS.each { |key| event_node.attributes[key] = "-" }								
					
					event_node.attributes["project"]		= name
					if (name == "rebuild")
						event_node.attributes["project"]		= cc_rebuild ? "CC Rebuild" : "Manual Rebuild" 
					end
					
					event_node.attributes["description"] 		= "'#{name} #{wildcard}' "
					event_node.attributes["status"] 		= "Running"
					event_node.attributes["build_date"] 		= event_time_formatted
					event_node.attributes["sort_date"] 		= sort_date					
					event_node.attributes["cc_project_name"]	= "#{ENV["COMPUTERNAME"]} #{cc_project_name}"

					events << event_node 
					xmldoc.save( @events_filename, :indent => true )
				end
				
			rescue Exception => ex
				puts "Exception during report_event_started: #{ex.message}"
				puts "Call stack:"
				puts ex.backtrace.join( "\n\t" )						
			end					
			sort_date
		end				
		
		#
		# Publish the event that a rebuild has finished
		# - it will search the xmldoc for the event of the same time
		# - if it can be found then this event will be finalised.
		# - stale events are cleaned up.
		#
		def report_event_finished(result, event_time)
			begin
				log_info "=== Event finished ==="
				time = Time.now
				event_end_time_formatted = time.strftime("%H:%M:%S %d-%m")
				
				xmldoc = LibXML::XML::Document::file( @events_filename )				
				if (xmldoc)

					events = xmldoc.find_first("//events")
					if (events)

						event_nodes = events.find("event")
						if (event_nodes)
						
							event_nodes.each do |event|	
								event_hash = {}
								EVENT_KEYS.each { |key| event_hash[key] = event.attributes[key].nil? ? "" : event.attributes[key] }
								
								if (event_hash["sort_date"]==event_time)
									event_hash["status"] = result.to_s.downcase
									event_hash["status"] = "Success" if (event_hash["status"].include?("completed") )									
									event_hash["status"] = "Failure" if (event_hash["status"].include?("error") )

									event.attributes["build_completed"] 	= event_end_time_formatted
					
									event_hash["build_duration"] = (Time.parse(event_end_time_formatted)-Time.parse(event_hash["build_date"]))																		
									event_hash["build_duration"] = prettify_rebuild_duration(event_hash["build_duration"]) #prettify the duration
								else	
									# Only one event can be running at any time - fix up stale events. ( should only happen if engine prematurely exits )
									event_hash["status"] = "Stale - was running" if (event_hash["status"].downcase=="running")
								end
								
								EVENT_KEYS.each { |key| event.attributes[key] = event_hash[key].nil? ? "" : event_hash[key] }								
							end					
						end
					end								

					xmldoc.save( @events_filename, :indent => true)
				end		
			rescue Exception => ex
				puts "Exception during report_event_finished: #{ex.message}"
				puts "Call stack:"
				puts ex.backtrace.join( "\n\t" )						
			end				
		end
		
		#
		# Rebuild independent data files from changelist or wildcard string.
		#
		def rebuild_files( wildcard )
			result = nil
									
			@build_lock.synchronize do
			
				event_time = report_event_started("rebuild", wildcard, false, "-")
			
				if ( wildcard =~ /^([0-9]+)$/ ) then
					cl = $1
					#enqueue_command_result( Assetbuild::Builder::Shell::CommandResultProgress.new( "Changelist #{cl} rebuild..." ) )
					begin
						export_files = []
						@p4.connect() unless ( @p4.connected?() )
						local_files = @p4.files_from_changelist( cl )
						local_files.each do |filename|
							local_filename = OS::Path::normalise( @p4.depot2local( filename ) )
							next unless ( @project.branches[@branch.name].is_export_file?( local_filename ) )
							export_files << local_filename
						end
						#enqueue_command_result( Assetbuild::Builder::Shell::CommandResultProgress.new( "Rebuilding assets and checking-in..." ) )
						rebuild_data( export_files )
						result = CommandResultCompleted.new( "Rebuild of changelist #{cl} complete." )
					rescue P4Exception => ex
						result = CommandResultError.new( "Changelist #{cl} was not found." )	
					end				
				else
					export = @env.subst( @project.branches[@branch.name].export )
					ind_wildcard = @env.subst( OS::Path::combine( export, wildcard ) )
					files = Engine::get_files( ind_wildcard )

					message = "Export files wildcard: #{ind_wildcard}, #{files.size} files found."
					#enqueue_command_result( Assetbuild::Builder::Shell::CommandResultProgress.new( message ) )

					# By creating a new thread we shall hopefully 
					# maintain the shell's interactive ability.
					#enqueue_command_result( Assetbuild::Builder::Shell::CommandResultProgress.new( "Rebuilding assets and checking-in..." ) )
					rebuild_data( files )
					result = Assetbuild::Builder::Shell::CommandResultCompleted.new( "Rebuild complete." )			
				end	
				
				report_event_finished(result, event_time)
			end
			return result
		end

		#
		#
		#
		def reload( )
			@build_lock.synchronize do
				@config.reload( )
			end
			Assetbuild::Builder::Shell::CommandResultCompleted.new( "Configuration XML reloaded." )
		end
		
		#
		#
		#
		def get_config( )
			Assetbuild::Builder::Shell::CommandResultCompleted.new( @config.pretty_string( 1, '   ' ) )
		end
		
		#
		#
		#
		def get_queue( )
			num_items_in_q = 0
			@shell.queue_walk do |queued_command|
				Assetbuild::Builder::Shell::CommandResultProgress.new( queued_command.to_s )	
				num_items_in_q += 1
			end			
			return Assetbuild::Builder::Shell::CommandResultCompleted.new( "Queue peek complete. \##{num_items_in_q} items." )
		end
		
		#
		#
		#
		def get_status( )
			c = Pipeline::Config.instance()
			message =  "Project:         #{@project.uiname}\n"
			message << "Branch:          #{@branch.name}\n"
			message << "Build lock:      #{@build_lock.locked? ? 'locked' : 'unlocked'}\n"
			message << "P4 Local:        #{@p4.connected?() ? 'connected' : 'not connected'}\n"
			message << "XGE:             #{c.use_xge ? 'enabled' : 'disabled'}\n"
			message << "Checkin:         #{@options['checkin'] ? 'enabled' : 'disabled' }\n"
			message << "Targets:\n"
			@project.branches[@branch.name].targets.each_pair do |name, tgt|
				message << "\tTarget: #{name} #{tgt.enabled}\n"
			end
			
			Assetbuild::Builder::Shell::CommandResultCompleted.new( message )
		end
		
		#
		# Common help function.
		#
		def help_func( )
			
			message = "Shell help and registered commands:\n"
			@commands.each do |command|
				
				message << "\t#{command.name}"
				command.shortcuts.each do |s|
					message << ",#{s}"
				end
				message << "  "
				command.parameters.each do |p|
					message << "<#{p.name}> "
				end
				message << "\t\t#{command.help}\n"
			end

			Assetbuild::Builder::Shell::CommandResultCompleted.new( message )
		end
		
		#
		# Common threads view function.
		#
		def threads_func( )
			message = ''
			Thread.list.each do |thr| 
				message += "\t#{thr.inspect}: #{thr[:name]}\n"
			end
			Assetbuild::Builder::Shell::CommandResultCompleted.new( message )
		end		
		
		#
		# Get logfile function.
		#		
		def get_logfile_func() 
			Assetbuild::Builder::Shell::CommandResultCompleted.new( get_xge_log() )
		end
		
		#
		# Build a string of modifications using p4 filespecs or local paths.
		#
		def build( modifications_string )
			throw ArgumentError.new( 'Invalid modifications object.' ) \
				unless ( ( nil != modifications_string ) and ( modifications_string.is_a?( String ) ) )
			
			result = nil
			# Protect from multiple simultaneous builds by using a mutex lock.
			log_info "Acquiring build lock"
			@build_lock.synchronize do
				log_info "Acquired build lock"

				@each_build_outputter.roll()			
				
				clear_log_files()
							
				logfiles_dst = []			
			
				@num_builds += 1
				log_info "============ Started Build \##{@num_builds} @ #{Time.now} ============="
				modifications = modifications_string.split				
				
				# strip out the wildcard info ( for event reporting ) and rebuild flag
				
				# DW : Not ideal however, the main Cruise Control CI process passes p4 paths into this.
				# In order that I can differentiate what a CC rebuild is and a CC CI is without adding extra command obsfucation 
				# then this can be inferred whether there are soley p4 paths passed here. 
				# The rebuild flag USED to differentiate these properly, but since all builds became
				# rebuilds due to the TCS deps I can no longer use this - and to retrospectively define some 'true rebuild' flag is just going to confuse people.
				
				wildcard = ""					
				rebuild  = false
				is_a_cc_rebuild = false
				cc_project_name = ""
								
				if ( modifications.length > 0) 					
					modifications.delete_if do |mod|
						# check if a cruise control rebuild
						if (mod=~REGEX_CC_REBUILD)
							is_a_cc_rebuild = true
							true
						elsif (mod=~REGEX_WILDCARD) # check if we are telling the engine what cruise control wildcard was used
							wildcard = $1
							
							# The special syntax for recursion needs to be presented nicely.
							if (wildcard.include?(NO_RECURSE_CHARACTER))
								wildcard = wildcard.gsub(NO_RECURSE_CHARACTER,"")
								wildcard += " (non-recursive)"
							else
								wildcard += " (recursive)"
							end
							
							true
						elsif (mod=~REGEX_CC_PROJECTNAME)
							cc_project_name = $1
							true
						elsif (mod=~REGEX_REBUILD) # check a rebuild of the assets is to be done.
							rebuild = true
							true						
						end
					
					end				
				end

				
				log_info "=== is_a_cc_rebuild #{is_a_cc_rebuild}"													
				event_time = report_event_started("rebuild", wildcard, is_a_cc_rebuild, cc_project_name) if (is_a_cc_rebuild)
				
				log_info "BUILD requested #{modifications.length} modifications : #{modifications_string}"			
			
				# If we lose our Perforce connection, server down for 
				# example, attempt to reconnect.
				@p4.connect() unless ( @p4.connected?() )
				logfile_dst = nil			
							
				begin
					build_start = Time.now
													
					sync( :tools_config )		# contains anim related compression stuff					
					sync( :tools_lib_util_ragebuilder ) # contains convert.rbs					
					sync( :tools_bin )		# contains stuff like animcombine.exe
					sync( :common )			# contains shaders for asset conversion					
					
					pipeline_config = Pipeline::Config.instance( )
					if pipeline_config.user.is_builder_server() # do not sync to platform data unless a builder server - since we do not submit CLs on the 'user' machines - across CC rebuilds data would simply be lost.								
						sync( :platform_data )		# synced in order that users that submit platform versions along with their independent files don;t then prevent a checkin of the platform data since we would not be at the head revision and would need to resolve.
					end
					#sync( :assets )		# ragebuilder has deps on this folder DW - 08-07-2010
					sync( :assets_metadata )	# ragebuilder has deps on this folder - UPDATED TO BE JUST THIS FOLDER 10/03/11
					sync( :assets_maps_parenttxds )	# DW UPDATED 18-11-11
					sync( :assets_processed )	# DW UPDATED 20-12-11
					

					# Main Build
					begin
						files, deleted_files = [], [] 
						
						fstat_results = @p4.run_fstat_escaped( modifications )
						fstat_results.compact! # paranoia 
						
						highest_change_number = -1
						
						additional_description = nil
						max_additional_descriptions = MAX_ADDITIONAL_DESCRIPTIONS
						
						fstat_results.each do |fstat|
							# DW - these are not actually the 'headAction' and 'headChange' since the fstats was a query of revision specifying filespecs.
							filename		= fstat['clientFile']
							depot_filename		= fstat['depotFile']
							action			= fstat['headAction']
							change_number		= fstat['headChange'].to_i
							revision		= fstat['headRev'].to_i

							if (revision<=0)						
								$stderr.puts("Revision of #{filename} is bad! ( #{revision} ) ")	 
							else
								highest_change_number = change_number if change_number > highest_change_number
							
								if ( 0 == 'delete'.casecmp( action ) )
							
									log_info "deleted #{filename}"
				
									deleted_files << OS::Path::normalise( filename )
								else
									log_info "add or edit #{filename}"
																
									files << OS::Path::normalise( filename )						
								end
								
								if ( max_additional_descriptions > 0 )
									additional_description = "Independent files built (truncates at #{MAX_ADDITIONAL_DESCRIPTIONS}):\n" if additional_description.nil?									
									additional_description += "#{change_number} #{action} #{depot_filename}\##{revision}\n" if rebuild
									additional_description += "#{action} #{depot_filename}\##{revision}\n" unless rebuild
									max_additional_descriptions -= 1
								end
							end
						end
						
						log_info "Getting Changelist for change #{highest_change_number}"
								
						changelist = @p4.run_describe( '-s', highest_change_number.to_s ).shift if highest_change_number >= 0 
						
						if (changelist.nil? and not rebuild)
							$stderr.puts "changelist is nil for changelist #{highest_change_number.to_s} !!!!"
						else														
							changelist = {} if ( rebuild )
							
							log_info "Converting"
							
							do_rebuild = rebuild
							do_rebuild = true # DW - hack for rebuild for now... its a hack because of the ocde path rebuild  or build takes - we don't want a rebuild to sync to files so we force it to rebuild at the last minute...
							log_info("******* - THIS IS A REBUILD - DUE TO TCS DEPENDENCIES - *******", true) if do_rebuild
							convert_data( nil, files, deleted_files, do_rebuild, changelist, true, additional_description )
							log_info "Converted"
						end
													
					rescue Exception => ex
						log_exception( ex, "Exception during asset build:" )
					end

					# Log time taken and finalise reports.
					log_time_msg( Time.now )

				rescue Exception => ex
					log_exception( ex, "build() exception:" )
				end

				# Force garbage collection
				GC::start()
				
				# Move the logfile before the build lock is released to ensure no log file ever gets overwritten.
				log_info "move log files"
								
				move_log_files( logfiles_dst ) 
				
				log_info "============ Completed Build \##{@num_builds} @ #{Time.now}============="
				
				@each_build_outputter.roll()							
				
				result = Assetbuild::Builder::Shell::CommandResultCompleted.new( "Build completed. Logfiles : #{logfiles_dst.join(' ')}" )
				
				if (is_a_cc_rebuild)
					report_event_finished(result, event_time)
				end
				
			end
			
			return result
		end
				
		#
		# Move the logfile before the build lock is released to ensure no log file ever gets overwritten.
		def move_log_files( logfiles_dst )
			logfiles_src = []
			begin				
				puts "adding log files"
				logfiles_src << get_xge_log()
				logfiles_src << get_main_log_each_build()															
				puts "added #{logfiles_src.length} log files"		
				
				# logfiles must be closed ( temporarily ) before we can read em.
				close_logfiles()
				
				logfiles_src.each do |logfile_src|												
					if (File.exist?(logfile_src))
						
						logfile_dst = logfile_src.gsub(".log",".cruisecontrol.log") # #{now.strftime('%Y-%m-%d.%H-%M-%S')}.log
						logfiles_dst << logfile_dst
						
						# DW - I put this in because Fileutils.mv kept hanging, so did FileUtils.cp.
						# eventually it seems that a file.rename is robust, however the sleep may in fact be unnecessary.
						Kernel.sleep(5) 		

						puts "Copying #{logfile_src} to #{logfile_dst}"
														
						FileUtils::Verbose.copy(logfile_src, logfile_dst)
						
						puts "Log File : #{logfile_dst}"
					else	
						puts "Log File : empty"	
					end												
				end
				
				reopen_logfiles()
				
			rescue Exception => ex
				puts "Unhandled exception: #{ex.message}"
				puts "Call stack:"
				puts ex.backtrace.join( "\n\t" )					
				log_exception( ex, "Monitor exception:" )
			end
		end
		
		# Reopen logfiles
		# 		
		def reopen_logfiles()					
			@shell_log.reopen_file()
			@build_log.reopen_file()
			Pipeline::LogSystem::instance().rootlog.reopen_file()
			Assetbuild::Builder::Shell::CommandResultCompleted.new( "Logfiles reopened.") 
		end
		
		# Closes logfiles
		# 		
		def close_logfiles()
			log_info "close_logfiles"	
			@shell_log.close_file()
			@build_log.close_file()			
			Pipeline::LogSystem::instance().rootlog.close_file()
			Assetbuild::Builder::Shell::CommandResultCompleted.new( "Logfiles closed.")
		end
		
		# Trys to get an exclusive lock on a file
		# if it can't get it an idx is incremented
		# thereafter the filename will not be used again.
		#
		def get_lock( filename )
			ok = true
			if File.exist? filename
				File.open(filename, 'r+') do |f|
					# get exclusive lock
					if f.flock File::LOCK_EX | File::LOCK_NB
						# release the lock
						f.flock File::LOCK_UN
					else
						@lock_problem_counter = @lock_problem_counter + 1 # ensures we dont use the same log file again					    
						log_error "file was not able to be exclusively locked #{filename}"
						ok = false
					end					    					    
				end				
			end
			ok
		end
		
		# Obliterate logfiles - don;t wabnt stale log file data coming through.
		# - Not elegant but does the job.		
		# logfiles are a total complete and utter nightmare! - TODO: review logfiles and how they work and come up with something better!
		def clear_log_files()
		
			begin
				if (not get_lock(get_xge_log()))
					return
				end
				
				if (not get_lock(get_main_log_each_build()))				
					return
				end
								
				puts "deleting log files #{get_xge_log()} #{get_main_log_each_build()}"
				close_logfiles()
				if File.exist? get_xge_log()
					puts "delete #{get_xge_log()}"
					File.delete(get_xge_log()) 
					puts "new #{get_xge_log()}"
					File.new(get_xge_log(), "w") 						
				end
				if File.exist? get_main_log_each_build()
					puts "delete #{get_main_log_each_build()}"
					File.delete(get_main_log_each_build())
					puts "new #{get_main_log_each_build()}"
					File.new(get_main_log_each_build(), "w")
				end
				reopen_logfiles()
			rescue Exception => ex
				puts "Exception during log delete: #{ex.message}"
				puts "Call stack:"
				puts ex.backtrace.join( "\n\t" )						
			end	
		end
																
		#
		# Start build engine, spawning a separate thread for the build loop.
		# This allows the interpreter to kill the thread, and restart the 
		# build loop should the config change or a build stall.
		#
		def start( )
			# Start up our command processing.
			start_queue_processing( )

			# Start up our local Console shell.
			@shell.start( )
		end
		
		#
		# Shutdown our AutoBuild engine.
		#
		def exit( )
			puts "Shutting down..."
			@shell.stop( )
			@running = false
			shutdown( )
		end

		#
		# Temporary measure to get the logifle.
		#
		def get_xge_log_dir( )
			xge_folder = XGE::get_temp_dir( @project.name, @branch.name )
			xge_packet_folder = OS::Path::combine( xge_folder, 'convert' )			
		end
		
		def get_xge_log_filename( )
			counter = (@lock_problem_counter==0) ? "" : @lock_problem_counter.to_s
			'convert#{counter}.log'
		end
		
		def get_xge_log( )			
			OS::Path.combine( get_xge_log_dir( ), get_xge_log_filename( ) )
		end
		
		def get_main_log_filename( )
			counter = (@lock_problem_counter==0) ? "" : @lock_problem_counter.to_s
			"assetbuilder#{counter}.log"
		end
		
		# DW - TODO : these directories can be derived better; rootLog = Pipeline::LogSystem::instance.rootlog
		def get_main_log_dir ( )
			OS::Path::combine(Pipeline::Config::instance().toolsroot, "logs")
		end
		
		def get_main_log( )
			OS::Path.combine( get_main_log_dir( ), get_main_log_filename( ) )
		end		
		
		def get_main_log_each_build( )			
			counter = (@lock_problem_counter==0) ? "" : @lock_problem_counter.to_s
			"#{get_main_log( )}.each_build#{counter}.log"
		end		
		
		#---------------------------------------------------------------------
		# Private Methods
		#---------------------------------------------------------------------	
	private
		#
		# Asset Builder environment initialisation.
		#
		def env_init( )
			@env = Environment.new()
			@project.branches[@branch.name].fill_env( @env )
		end		
	
		#---------------------------------------------------------------------
		# Build Loop Utility Methods
		#---------------------------------------------------------------------
	
		#
		# Rebuild data function.  The content parameter may be nil, a single content node
		# or an Array of content nodes.
		#
		def rebuild_data( files )
			throw ArgumentError.new( "Invalid filenames Array (#{files.class})." ) \
				unless ( files.is_a?( Array ) )
			begin
				rebuild_start = Time.now
				msg = "Manual data rebuild:"
				files.each do |filename| msg += "\n#{filename}"; end
				@build_log.info( msg )
				report = ReportRebuildRagebuilder.new( @project, @branch.name, msg, files )
				
				# We don't fetch latest shaders for asset conversion as this
				# may be required to be done manually.
				
				# Rebuild our content
				convert_data( report, files, [], true, {}, true )
				
				# Log time taken and finalise reports.
				log_time_msg( rebuild_start )
				report_finalise( report )

				# Force garbage collection
				GC::start()
			rescue Exception => ex
				log_exception( ex, "Rebuild data exception:" )
			end
		end	


		#
		# Given a list of platform files that are new ( files) and are requiring deleted ( deleted_files )
		# return a list of platform added, edited and deleted files that truly represents what action
		# is required on these files. To clarify a file that is wished to be deleted might already be delted, so no action is required.
		# A file that is new could be either edited or added so determine which.
		def get_added_edited_deleted_platform_files( files, deleted_files )
			
			platform_added_files, platform_edited_files, platform_deleted_files = [], [], []

			#------------------------------------------------------
			# Get possible Added or Edited files 
								
			if (files and files.length > 0)
				fstat_results = @p4.run_fstat_noncompact( files )
				
				if fstat_results
					fstat_results.each do |fstat|
					
						# DW - these are not actually the 'headAction' and 'headChange' since the fstats was a query of revision specifying filespecs.
						filename		= fstat['clientFile']
						action			= fstat['headAction']
						revision		= fstat['headRev'].to_i
				
						if 	( 0 == 'delete'.casecmp( action ) or revision <= 0 )
							platform_added_files << OS::Path::normalise( filename ) 	
						else								
							platform_edited_files << OS::Path::normalise( filename )
						end						
					end
				end
			end

			#------------------------------------------------------
			# Get possible Deleted Files 
			if (deleted_files and deleted_files.length > 0)
				fstat_results = @p4.run_fstat_noncompact( deleted_files )
				
				if fstat_results
					fstat_results.each do |fstat|
					
						# DW - these are not actually the 'headAction' and 'headChange' since the fstats was a query of revision specifying filespecs.
						filename		= fstat['clientFile']
						action			= fstat['headAction']
						revision		= fstat['headRev'].to_i
				
						if 	( 0 == 'delete'.casecmp( action ) or revision <= 0 )
							# already deleted or doesn't exist - no action required.
						else	
							platform_deleted_files << OS::Path::normalise( filename ) # the file we want to delete exists, so just delete it.
						end						
					end
				end
			end
			
			# This output will be put into report - do not remove.
			platform_added_files.each 	{ |file| log_info  ":Platform Added File #{file}"  }
			platform_edited_files.each 	{ |file| log_info  ":Platform Edited File #{file}"  }
			platform_deleted_files.each 	{ |file| log_info  ":Platform Deleted File #{file}"  }
			
			return platform_added_files, platform_edited_files, platform_deleted_files
		end

		#
		# Return three arrays of filenames that are all outputs.  The output
		# arrays are in: added, edited, deleted respectively.
		#
		def get_output_files( export_files, export_deleted_files )

			begin
				raise ArgumentError.new( "Invalid export_files Array specified (#{export_files.class})." ) \
					unless ( export_files.is_a?( Array ) )			
				raise ArgumentError.new( "Invalid export_deleted_files Array specified (#{export_deleted_files.class})." ) \
					unless ( export_deleted_files.is_a?( Array ) )			

				files_content = ProjectUtil::data_content_for_files( @project, export_files )
				deleted_files_content = ProjectUtil::data_content_for_files( @project, export_deleted_files )
				
				puts "EXPORT FILES: "
				puts export_files.join("\n")
				
				# Determine our content-tree defined outputs; this is typically
				# for the processed files that will need to be checked out etc.
				# We also use this array to determine our platform files.
				output_files = []
				
				# we want to exclude certain files from the conversion
				files_to_exclude = []
				
				if ( files_content.nil? or 0 == files_content.size ) then
					# This case will likely not be hit, as data_content_for_files returns nil entries
					# for files not in the content tree (see special case below).
					puts "CONTENT NIL"
					output_files += ProjectUtil::data_convert_platform_filenames( @project, @branch, export_files )
				else
					puts "ELSE"
					
					# here we need to get any dependencies before we continue - JWR
					
					files_content = get_content_with_dependencies( files_content )
					
					files_content.each_with_index do |content, index|
										
						if ( content.nil? ) then
						
							# This case handles files that do not appear in our content tree.
							puts "\tFILE: #{export_files[index]}"
							output_files += ProjectUtil::data_convert_platform_filenames( @project, @branch, export_files[index] )
							
						else
							# JWR - B* 692193 - prevent scene xml files making it into the platform data
							if content.is_a?( Pipeline::Content::MapSceneXml ) then
								files_to_exclude << export_files[index]
							else
								output_files += ProjectUtil::data_content_get_output_files( @project, @branch, content, true )
							end
						end
					end
				end
				
				puts "OUTPUT FILES: "
				puts output_files.join("\n")
				
				output_files.uniq!
			
				# Determine the platform defined outputs; as we did before.
				deleted_files = ProjectUtil::data_convert_platform_filenames( @project, @branch, export_deleted_files )
				platform_added_files, platform_edited_files, platform_deleted_files = get_added_edited_deleted_platform_files( output_files, deleted_files )

				return platform_added_files, platform_edited_files, platform_deleted_files, files_to_exclude
			rescue Exception => ex
				puts "UNHANDLE EX: #{ex.message}"
				puts ex.backtrace.join( "\n" )
			end
		end

		def get_content_with_dependencies( input_nodes )
			dependency_nodes = []
		
			#get dependencies for the input_filenames
			input_nodes.each do |input_node|
				if( input_node != nil and (input_node.is_a?( Pipeline::Content::MapZip )) ) then
					ProjectUtil::get_lod_dependency_nodes( input_node, dependency_nodes )
				end
			end
		
			#combine and remove duplicates - do not use uniq!								
			dependency_nodes.delete_if do |node|			
				node.nil? or input_nodes.include?(node)
			end

			output_nodes = input_nodes + dependency_nodes
			
						
			output_nodes
		end

		#
		# Get the modified times of files storing in a preexisting hash 
		# against a key that describes an event on that file. Used for pre/post convert times.
		#
		def get_modified_times_of_files(modified_times, files, key)
			
			log_info("get modified times of files")
			
			files.each do |filename|
				filename = OS::Path::normalise(filename)
				modified_times[filename] = { key => File.mtime(filename) } if File.exist?(filename)
			end	
		end	
		
		#
		# Build data function.  Checks out the files and does a data build.
		#
		def convert_data( report, files, deleted_files, rebuild = false, changelist = {}, revert = false, additional_description = nil )			
			raise ArgumentError.new( "Invalid report specified (#{report.class})." ) \
				unless ( report.nil? or report.is_a?( Pipeline::Builder::ReportBase ) )
			raise ArgumentError.new( "Invalid files Array specified (#{files.class})." ) \
				unless ( files.is_a?( Array ) )
			raise ArgumentError.new( "Invalid deleted_files Array specified (#{deleted_files.class})." ) \
				unless ( deleted_files.is_a?( Array ) )
			
			begin
				pipeline_config = Pipeline::Config.instance( )
				project = pipeline_config.project
				project.load_content( @branch.name, true )
				
				if not pipeline_config.user.is_builder_server()
					log_info "******* - Current user is not a builder_server(), so no changelist will be created and no platform data will be committed - ******* "
				end
				
				env = Environment.new()
				@project.branches[@branch.name].fill_env( env )

				time_format = "%m/%d/%Y %H:%M:%S"

				time_of_change = changelist.has_key?( 'time' ) ? Time.at(changelist['time'].to_i).strftime(time_format) : "Unknown date/time"

				desc = ""
				desc = changelist['desc'] unless changelist.empty?
				desc = desc.gsub(/Error/i,	"E rror") if desc
				desc = desc.gsub(/Warning/i,	"W arning") if desc

				comment = "#{@project.uiname} Asset Builder platform asset build [ processed at #{Time.now.strftime(time_format)} ]\n" + \
						  "Branch: #{@branch.name}\n" + \
						  "Built because of changelist: #{changelist['change']} by #{changelist['user']}@#{changelist['client']} on #{time_of_change}.\n\n" + \
						  "Changelist description:\n#{desc}\n" \
						  unless ( changelist.empty? )
				comment = "#{@project.uiname} Asset Builder platform asset build #{Time.now.strftime(time_format)}.\n" +
						  "Branch: #{@branch.name}\n" + \
						  "No associated changelist." if ( changelist.empty? )			

				comment += "\n#{additional_description}" if ( additional_description )

				# Checkout platform files.
				@p4.connect( ) unless ( @p4.connected? )
				change_id = @p4.create_changelist( comment ) if pipeline_config.user.is_builder_server()
				log_info "Changelist #{change_id} is created : #{comment}" if pipeline_config.user.is_builder_server()

				platform_added_files, platform_edited_files, platform_deleted_files, files_to_exclude = []
				begin
					# DW - work out which platform files require added, edited and deleted.
					# since the actions on the ind files don't necessitate that same action on the platform files
					# because the platform files CAN be in any state : eg. ind file is edited but plat file is deleted, ind file is deleted but plat file already deleted.
					
					platform_added_files, platform_edited_files, platform_deleted_files, files_to_exclude = get_output_files( files, deleted_files )

					puts "PLATFORM ADD:"
					platform_added_files.each do |filename|
						puts "\t#{filename}"
					end
					puts "PLATFORM EDIT:"
					platform_edited_files.each do |filename|
						puts "\t#{filename}"
					end
					puts "PLATFORM DEL:"
					platform_deleted_files.each do |filename|
						puts "\t#{filename}"
					end
					puts "EXCLUDE:"
					files_to_exclude.each do |filename|
						puts "\t#{filename}"
						file_to_exclude_index = files.find_index(filename)
						if (file_to_exclude_index != nil) then
							files.delete_at(file_to_exclude_index)
							puts "\twas removed"
						end
					end
				rescue Exception => ex
					puts "Ex: #{ex.message}"
					puts ex.backtrace.join("\n")
				end
				
				# DHM TESTING -- DONT CHECKOUT FILES UNTIL THEY ARE BUILT.
				# HOPEFULLY FIX THE FILETYPE ISSUES
				log_info "Checking out edited platform files (#{platform_edited_files.length})" if pipeline_config.user.is_builder_server()
				checkout_platform_files( change_id, [], platform_edited_files ) if pipeline_config.user.is_builder_server()
				
				# get the modified time of files before convert
				modified_times = {}
				get_modified_times_of_files(modified_times, platform_edited_files, "preconvert")

				# Do actual platform data conversion.
				ProjectUtil::data_convert_file( files, rebuild ) do |node, success|
					next unless ( node.is_a?( Pipeline::Content::File ) )

					if ( success ) then
						message = "Converted: #{node.filename}"
						log_info message
					else
						message = "Failed to convert, (however revert is disabled): #{node.filename}"
						log_info(message, true)
						@build_log.error( message )
						## @p4.run_revert( node.filename )
					end
				end if ( files.size > 0 )
			
				# get the modified time of files after convert
				get_modified_times_of_files(modified_times, platform_edited_files, "postconvert") if pipeline_config.user.is_builder_server()
			
				log_info "Checking out added platform files (#{platform_added_files.length})" if pipeline_config.user.is_builder_server()
				checkout_platform_files( change_id, platform_added_files, [] ) if pipeline_config.user.is_builder_server()

				log_info "Delete platform files #{platform_deleted_files.length}" if pipeline_config.user.is_builder_server()
				delete_platform_files( report, change_id, platform_deleted_files ) if pipeline_config.user.is_builder_server()

				# Checkout/add files again so that we can add new files.  This is
				# in case the content configuration is updated with new files.			
				log_info "checkin revert files" if pipeline_config.user.is_builder_server()
				
				checkin_revert_files( report, change_id, modified_times, changelist, revert ) if pipeline_config.user.is_builder_server()

				# Parse our converter output to append it to our report.				
				if report
					log_info "Parse Report"
					parse_report( report ) 
				end

				@p4.disconnect()
			rescue Exception => ex
				log_exception( ex, "convert_data exception:" )
			end
		end
		
		#
		# Parse our converter output to append it to our report.
		def parse_report( report )
			raise ArgumentError.new( "Invalid report specified (#{report.class})." ) \
				unless ( report.is_a?( ReportRebuildRagebuilder ) )
				
			log_info "Parsing convertor output"	
			
			# Now use the C# parser since ruby would run out of memory 
			# and be too slow...
			begin								
				# just pass the errors on...				
				cmd = OS::Path.combine(Pipeline::Config::instance.toolsbin, "errorparser.exe")				
				log_info "running cmd #{cmd}"

				output, errors, warnings = Resourcing::ConvertSystem::instance().get_converter_output( cmd, ERROR_REGEXP, WARNING_REGEXP )
				
				log_info "Setting report errors and warnings"
				report.set_output_errors_warnings( output, errors, warnings )

			rescue Exception => ex
				log_exception( ex, "Exception during error parsing:" )
			end			
		end
			
		#
		# Write out report data.
		#
		def report_finalise( report, changelist = {}, max_id = -1, subject = nil )
			# Ensure our output directory exists
			if ( not File.directory?( @config.report.path ) ) then
				FileUtils.mkdir_p( @config.report.path )
			end

			user = ( changelist.empty? ? Socket.gethostname() : changelist['user'] )
			id = ( changelist.empty? ? 'rebuild' : changelist['change'] )
			
			now = Time.now
			report_date = now.strftime( '%Y-%m-%d' )
			report_filename = ''
			if ( changelist.empty? ) then
				report_filename = "rebuild_#{now.strftime('%Y-%m-%d %H-%M-%S')}_#{project.name}_report.xml".gsub( ' ', '_' )
			else
				report_filename = "#{id}_#{project.name}_report.xml".gsub( ' ', '_' )
			end
			report_path = OS::Path.combine( @config.report.path, report_date, report_filename )
			
			report_path = report_path.gsub("assetbuilder","assetbuilder_hw")
			
			report_url = @config.report.webserver + '/' + OS::Path::combine( report_date, OS::Path::replace_ext( report_filename, 'html' ) )
			report.save( report_path, true, report_url )
								
			# Run XSLT and write as HTML
			xslt_report = Pipeline::Builder::XSLTReportWriter.new( report_path )
			html_filename = OS::Path.replace_ext( report_path, 'html' )
			xslt_report.write( @config.report.xslt_html, html_filename )
			
			# Send email report
			email_report_html = Pipeline::Builder::HTMLEmailReportWriter.new( report_path )
			if ( report.error_count > 0 ) then
				subject = "#{@project.uiname} #{@branch.name}: #{id}: *** ERRORS *** [#{user}|#{id}/#{max_id}]" if subject.nil?
			else
				subject = "#{@project.uiname} #{@branch.name}: #{id}: OK [#{user}|#{id}/#{max_id}]" if subject.nil?
			end
			
			email_report_html.write( @config.report.email.xslt, subject,
								@config.report.email.addresses[:from],
								@config.report.email.addresses[:list], 
								@config.report.email.addresses[:maintainer],
								@config.report.email.aliases[:from],
								@config.report.email.aliases[:list], 
								@config.report.email.aliases[:maintainer] )
			puts "HTML report saved to #{html_filename}"
			report.close()
		end

		#
		# Log and print the generic synced to changelist message.
		#
		def log_sync_msg( monitor, changelist, skipped )

			message = "Synced #{monitor.root_folder}@#{changelist['change']}, skipped: #{skipped}"
			log_info message
		end

		#
		# Log an exception.
		#
		def log_exception( ex, prefix = 'Error : Unhandled exception:' )
		
			message = "#{prefix} #{ex.message}"			
			enqueue_command_result( Assetbuild::Builder::Shell::CommandResultProgress.new( message ) )
			@build_log.error( message )
			ex.backtrace.each do |m|
				puts m
				@build_log.error( "Error : #{m}" )
			end
		end

		#
		# Log and print the build done with time message.
		#
		def log_time_msg( build_start )
		
			build_time = ( Time.now - build_start )
			message = "Build done: #{build_time} seconds"			
			#enqueue_command_result( Assetbuild::Builder::Shell::CommandResultProgress.new( message ) )
			@build_log.info( message )
		end

		#
		# Checkout platform files for our project given the array of files
		# to checkout (platform files, Array).
		#
		def checkout_platform_files( change_id, add_files, edit_files, lock = false )
			throw ArgumentError.new( "Invalid files list (#{add_files.class})." ) \
				unless ( add_files.is_a?( Array ) )
			throw ArgumentError.new( "Invalid files list (#{edit_files.class})." ) \
				unless ( edit_files.is_a?( Array ) )
			
			begin
				log_info "Checking out files (#{add_files.size}) for add & convert..."
				add_files.each do |filename|
					if (not File.file?(filename) )
						log_info "Error: The file #{filename} does not exist yet we are about to add it. This is called AFTER conversion."
						next
					end
				
					log_info "Add: #{filename} CL #{change_id.to_s}"
					@p4.run_edit_or_add( '-c', change_id.to_s, filename )
					log_info "Added: #{filename} in CL #{change_id.to_s}"
					# Get basetype.					
					fstat = @p4.run_fstat_escaped( filename ).shift				
					log_info "fstat succeeded for: #{filename}"
					basetype, modifiers = fstat['type'].split( '+' ) 
					puts "Basetype: #{basetype}"
					puts "Modifiers: #{modifiers}"
                    			@p4.run_reopen( '-t', "#{basetype}+S16w", filename )
				end

				@build_log.info( "Checking out files (#{edit_files.size}) for edit & convert..." ) 
				edit_files.each do |filename|
					
					#if (not File.file?(filename))
					#	log_info "Error: The file #{filename} does not exist to checkout for edit."
					#	next
					#end

					log_info "Checkout: #{filename} CL #{change_id.to_s}"
					@p4.run_edit( filename )
                    			# Removing "-t +S10w" so we can maintain the modifiers.
					@p4.run_reopen( '-c', change_id.to_s, filename )
				end
								
			rescue P4Exception => ex
				log_exception( ex, "Error : Perforce exception during checkout:" )
			end
		end
		
		#
		# Delete platform files for our project.
		#
		def delete_platform_files( report, change_id, files )
			throw ArgumentError.new( "Invalid report specified (#{report.class})." ) \
				unless ( report.nil? or report.is_a?( Pipeline::Builder::ReportBase ) )
			throw ArgumentError.new( "Invalid files list (#{files.class})." ) \
				unless ( files.is_a?( Array ) )
				
			begin
			
				log_info "Deleting files (#{files.size})..."
				files.each do |filename|
					log_info "Deleting: #{filename} CL #{change_id.to_s}"
					@p4.run_delete( '-c', change_id.to_s, filename )
				end
			rescue P4Exception => ex
			
				log_exception( ex, "Perforce exception during delete:" )
			end
		end
		
		#
		# Checkin currently opened files.
		#
		def checkin_revert_files( report, change_id, modified_times, changelist = {}, revert = true )
			log_info "Checkin is enabled" if @options['checkin']
			log_info "reverting!" if revert
			
			return unless @options['checkin']
			throw ArgumentError.new( "Invalid report specified (#{report.class})." ) \
				unless ( report.nil? or report.is_a?( Pipeline::Builder::ReportBase ) )

			begin
				revert_results = @p4.run_revert( '-a', '-c', change_id.to_s ) if revert

				log_info( "Reverting unchanged  #{revert_results.length} files...", true ) if revert				

				if ( ( not revert_results.nil? ) ) then
					report.additional << 'The following files were reverted:' \
						unless ( report.nil? or revert_results.empty? )
					revert_results.each do |revert|
						next unless ( revert.is_a?( Hash ) )				
						report.additional << revert['clientFile'] unless report.nil?
						 
						# check the modified time - it might be worth issuing a warning that this file was not modified.
						filename = OS::Path::normalise(revert['clientFile'])
						
						log_info "EngineMesssage: *** REVERTED (unchanged) #{filename} ***"
						
						if (	modified_times and
							modified_times[filename] and 
							modified_times[filename]["preconvert"] == modified_times[filename]["postconvert"] )

							@build_log.warn("Warning: Revert unchanged #{filename} : Contents AND Timestamp unchanged. File did not convert?")
						end						
					end
				end
				
				# Submit will raise an exception if there are no files to submit
				# so we detect here if we need to submit.
				files = @p4.run_opened( '-c', change_id.to_s, '-C', @p4.client )
				
				if ( files.size > 0 ) then
					log_info "Check in (#{files.size}) files currently in CL #{change_id.to_s}...( *** which may not be the final CL *** )"

					begin
						# Attempt a submit, if a P4Exception is raised then we
						# assume its because we have unresolved files.
						#@p4.run_unlock( '-c', change_id.to_s )
						submit_result = @p4.run_submit( '-c', change_id.to_s )												
						
						submit_result.each do |sr|							
							log_info "Checking in files (#{files.size}) in CL #{sr['submittedChange'].to_s}..." if sr.has_key?('submittedChange')
						end
				
					rescue P4Exception => ex
					
						log_info "Error : Checking in files hit a P4Exception - are the files locked? or are they needing resolved?"

						begin

							# We assume that the submit failed because files need 
							# to be resolved so lets try that, accepting ours.

							log_info "Resolving"
							resolve_results = @p4.run_resolve( '-ay' )

							if ( ( not resolve_results.nil? ) ) then
								report.additional << 'The following files were resolved:' \
									unless ( report.nil? or resolve_results.empty? )
								resolve_results.each do |resolve|
									next unless ( resolve.is_a?( Hash ) )
									report.additional << resolve['clientFile'] unless report.nil?
								end
							end

							#@p4.run_unlock( '-c', change_id.to_s )	
							log_info "Submitting (post resolve) - it still may not work - if say for example the file was locked."
							@p4.run_submit( '-c', change_id.to_s )				

						rescue P4Exception => ex
							log_info "Error : Dealing with a failed submit or resolve, has resulted in further errors. - was the target file locked?"
						end
					end
				else
					# Remove empty changelist.
					#@p4.run_unlock( '-c', change_id.to_s )
					@p4.delete_changelist( change_id )
					log_info "No files to checkin. CL is now deleted #{change_id.to_s}"
					@build_log.debug( "No files to checkin." )
				end

			rescue P4Exception => ex
				log_exception( ex, "Error: Perforce exception during revert/checkin:" )
			end
		end
	end

end # AssetBuild module

# End of engine.rb