#
# File:: parcodegen_ci.rb
# Description:: Continuous Integration of ParCodeGen psc files.
#
# 		This script takes a modifications xml file as produced by cruise control,
#		It updates p4 with the associated .PSC file.
#
#		Pseudo-Pseudo-Code
#		==================
# 		- For all the files modified they are converted by parcodegen if they are sourcecode files (cpp, c)
# 			- If there are psc files they are just copied.
# 		- They all go to a tmp directory. 
#		- Then for all files we processed 
#			- if an associated psc file doesn't exist in the tmp dir delete the psc file from the p4 build/metadata dir.
# 			- if it does exist it will be added/edited to the build/metadata directory.
#
# Author:: Derek Ward <derek.ward@rockstarnorth.com>
# Date:: 08th April 2011
#
# Passed in  :-  dest directory, filename/widcards or modification.xml(cruise control) to parse.
# Passed out :-  stderr contains all errors 
#                stdout for all other output. 
# Returns    :-  returns non zero upon detecting any errors 

#-----------------------------------------------------------------------------
# Uses / Requires
#-----------------------------------------------------------------------------
require 'pipeline/config/projects'
require 'pipeline/os/getopt'
require 'rexml/document'
include Pipeline
require 'systemu'
require 'fileutils'

#-----------------------------------------------------------------------------
# Constants
#-----------------------------------------------------------------------------

OPTIONS = [
	[ "--help", 			"-h", OS::Getopt::BOOLEAN, 	"display usage information." ],
	[ '--project', 			'-p', OS::Getopt::REQUIRED, 	'specify project key (e.g. gta5, jimmy)' ],
	[ '--branch', 			'-b', OS::Getopt::REQUIRED, 	'specify project branch (e.g. dev, dev_migrate)' ],
	[ '--dest_folder', 		'-d', OS::Getopt::REQUIRED, 	'dest folder eg. x/gta5/build/dev/metadata' ],	
	[ '--disablecheckin', 		'-c', OS::Getopt::BOOLEAN,	'prevent checkin ( for development )' ],			
	[ '--disable_revert_unchanged', '-r', OS::Getopt::BOOLEAN,	'prevent revert unchanged files ( for development )' ],			
	[ '--pre_convert_script', 	'-ps', OS::Getopt::BOOLEAN, 	'execute script on each file eg. blah.rb, this will be called like this; blah.rb file.cpp' ],	
]

TRAILING_DESC 		= 'File paths separated by spaces (e.g. x:/gta5/src/dev). and/or a cruise control modifications.xml filename.'

INFO			= "" # comment back in for a verbose report "[colourise=grey]INFO_MSG: "
INFO_BLACK		= "[colourise=black]INFO_MSG: "
INFO_BLUE		= "[colourise=blue]INFO_MSG: "
INFO_GREEN		= "[colourise=green]INFO_MSG: "

PAR_CODE_GEN_CI_VERSION	= "1.0"

PAR_CODE_GEN_COMMAND 	= "$(toolsroot)\\bin\\coding\\python\\parCodeGen.exe"
PRE_CONVERT_SCRIPT_COMMAND = "$(toolsroot)\\lib\\util\\parcodegen_ci\\parcodegen_ci_pre_convert.rb"
TMP_FOLDER	     	= "$(toolsroot)\\tmp\\parcodegen_ci"

PAR_CODE_GEN_NO_XML_TOKEN = "No XML metadata"

COPY_EXTENSIONS     	= [ "psc" ]
CONVERT_EXTENSIONS   	= [ ]
PAR_CODE_GEN_EXTENSION 	= "psc"

#-----------------------------------------------------------------------------
# General helper methods
# - log files added in retrospect of the fact they could be useful!
#-----------------------------------------------------------------------------

$gLog = Log.new( 'parcodegenci' )

def info(msg)
	$gLog.info("#{INFO}#{msg}")
end

def info_black(msg)
	$gLog.info("#{INFO_BLACK}#{msg}")
end

def info_blue(msg)
	$gLog.info("#{INFO_BLUE}#{msg}")
end

def info_green(msg)
	$gLog.info("#{INFO_GREEN}#{msg}")
end

def warning(msg)
	$gLog.warn("#{INFO}Warning: #{msg}")
end

def error(msg)
	$gLog.error("Error: #{msg}")
end

#-----------------------------------------------------------------------------
# ParCodeGen helper class
#-----------------------------------------------------------------------------
class ParCodeGenCi

	def initialize(pre_convert_script = false)
		@p4 = Pipeline::Config::instance().scm 
		@p4.connect()	
		
		@p4_rage = Pipeline::Config::instance().ragescm
		@p4_rage.connect()
		
		@pre_convert_script = false # DW - removed for now, not working... @pre_convert_script = pre_convert_script
	end
	
	#-----------------------------------------------------------------------------
	# Parse the Cruise Control XML file for modificatons. 
	# Return an array of p4 filespecs.
	#
	# FYI - the modifications.xml 'schema'
	#
	#<!-- Start of the group of modifications (even if just one). -->
	#<ArrayOfModification xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema">
	#  <!-- Start of one modification. -->
	#  <Modification>
	#    <!-- The change number. -->--filename=
	#    <ChangeNumber>... value ...</ChangeNumber>
	#    <!-- The comment. -->
	#    <Comment>... value ...</Comment>
	#    <!-- The user's email address. -->
	#    <EmailAddress>... value ...</EmailAddress>
	#    <!-- The affected file name. -->
	#    <FileName>... value ...</FileName>
	#    <!-- The affect file's folder name. -->
	#    <FolderName>... value ...</FolderName>
	#    <!-- The change timestamp, in yyyy-mm-ddThh:mm:ss.nnnn-hhmm format -->
	#    <ModifiedTime>... value ...</ModifiedTime>
	#    <!-- The operation type. -->
	#    <Type>... value ...</Type>
	#    <!-- The user name. -->
	#    <UserName>... value ...</UserName>
	#    <!-- The related URL. -->
	#    <Url>... value ...</Url>
	#    <!-- The file version. -->
	#    <Version>... value ...</Version>
	#    <!-- End of modification. -->
	#  </Modification>
	# <!-- End of the group of modifications. -->
	#</ArrayOfModification>
	#Pasted from <http://confluence.public.thoughtworks.org/display/CCNET/Modification+Writer+Task> 	
	#-----------------------------------------------------------------------------
	
	def parse_cc_modifications( filename, project, branchname)
		info "Parsing #{filename}"
		files = []
		begin
			if not File.exists?(filename)
				error "#{filename} does not exist."
				return nil
			end				
			
			#----------- Open Source xml ---------------------						
			src_file = File.new( filename )
			if not src_file
				error "#{filename} could not open."
				return nil
			end		

			#----------- Read XML doc ------------------------
			xmldoc	= REXML::Document.new( src_file )
			if not xmldoc
				error "#{filename} could not be opened as an XML document."
				return nil
			end	

			xmldoc.elements.each( 'ArrayOfModification/Modification' ) do |modification|
				folder_name		= nil
				file_name		= nil
				revision		= nil
				type			= nil
				modification.elements.each('FolderName')	{ |element| folder_name = element.text	}
				modification.elements.each('FileName')		{ |element| file_name	= element.text	}
				modification.elements.each('Version')		{ |element| revision	= element.text	}
				modification.elements.each('Type')		{ |element| type	= element.text	}
 
				if folder_name.nil? or file_name.nil? or revision.nil? or type.nil?
					error "#{filename} Invalid XML."
					return nil
				end	

				local_file = OS::Path.normalise(@p4.depot2local( "#{folder_name}/#{file_name}" ))								
				local_file = OS::Path.normalise(@p4_rage.depot2local( "#{folder_name}/#{file_name}" )) if ( local_file.nil? or local_file.length==0 )
				
				# its ok for files not to be resolved - might be outside the workspace.
				#error "Can't resolve #{folder_name}/#{file_name} locally" if local_file.nil? or local_file.length==0
				
				# there is no requirement for the file to exist though since this may be a delete operation 
				# but the result will be that the file is processed and finally deleted - we need to process it in order to 
				# know that it needs deleting from perforce.
				if ( local_file and local_file.length > 0)			
					info "\t#{type} Local file parsed #{local_file} "
					files << local_file
				else
					warning "#{folder_name}/#{file_name} will not be converted."
				end
			end

		rescue Exception => ex
			error "Unexpected exception parsing modifications: "
			error "\t#{ex.message}"
			ex.backtrace.each { |m| error "\t#{m}"; }
			Process.exit -1
		end	

		files
	end

	#--------------------------------------------------------------------------------
	# Kick off the conversion/copy of files - they will be built/copied to tmp folder
	#--------------------------------------------------------------------------------
	def process_files( env, files, error_count, dst_folder_root )	
		begin						
			processed_files = []

			# --- report ---
			report = { "copied" => 0, "copied_ok" => 0, "not_copied_ok" => 0, "converted" => 0, "converted_ok" => 0, "skipped" => 0, "non_existent" => 0, "deleted" => 0 }

			# --- copy directly / convert ---
			
			# --- first strip the files down to more concise list ---
			stripped_files = []			
			files.uniq.each do |file|
				
				if (not file.is_a?( String ))
					error_count += 1
					error "file is a #{file.class} #{file.to_s}"
					next
				end
				
				if file.nil?
					error_count += 1
					error "nil file encountered."	
					next
				end

				if not File.exist?(file)	
					report["non_existent"] += 1
				else

					filename = OS::Path.get_filename(file)					

					ext = OS::Path::get_extension(filename)	

					is_copy 	= COPY_EXTENSIONS.include?(ext)
					is_convert 	= CONVERT_EXTENSIONS.include?(ext)

					if (not is_copy and not is_convert)
						report["skipped"] += 1
						info "\tSkipping #{file}"# as #{ext} not in #{CONVERT_EXTENSIONS.join(" ")} or #{COPY_EXTENSIONS.join(" ")}"
						next
					end															
				end											
				
				stripped_files << file
			end	
						
			src_path 	= OS::Path.normalise( ENV['RS_CODEBRANCH'] )
			
			# --- now copy or convert ---
			stripped_files.each do |file|
				
				filename = OS::Path.get_filename(file)											
				ext = OS::Path::get_extension(filename)	
				
				is_copy, is_convert, is_delete = false, false, false
				
				if (File.exists?(file))
					is_copy 	= COPY_EXTENSIONS.include?(ext)
					is_convert 	= CONVERT_EXTENSIONS.include?(ext)							
				else
					is_delete	= true
				end			
				# --- compute subfolder in tmp dir and make it---									
				sub_folder 	= OS::Path.normalise(OS::Path.get_directory(file))
				if (sub_folder.sub!(src_path,"") == nil)
					report["skipped"] += 1
					info "\tSkipping #{file} as #{src_path} is not in #{sub_folder}"
					next
				end
				dst_folder = OS::Path.combine(dst_folder_root,sub_folder)	
				if not File.exists?(dst_folder)	
					info "\tMaking directory #{dst_folder}"
					FileUtils::mkdir_p(dst_folder)
				end
				
				dst = OS::Path.combine(dst_folder,filename)
				dst = OS::Path.replace_ext(dst,PAR_CODE_GEN_EXTENSION)
						
				# --- choose to copy, convert or skip. ---										
				if ( is_copy )
					report["copied"] += 1

					info"\tCopying #{file} to #{dst}"									
					FileUtils::cp(file, dst)	
					
					# --- did a file get copied ok? ---
					if File.exist?(dst)
						report["copied_ok"] += 1
						# --- register as a file that was processed --
						processed_files << { "filename" => filename, "tmp_filename" => dst }	
					else
						report["not_copied_ok"] += 1
					end
					
				elsif ( is_convert )
					report["converted"] += 1

					# --- build cmd ---
					info"\tConverting #{file}"					
					cmd = OS::Path::normalise( env.subst("#{PAR_CODE_GEN_COMMAND} --rebuild --savemetadata #{dst_folder} #{file}"))
					info("\t#{cmd}")

					if (@pre_convert_script)					
						# -- execute pre convert script cmd ---
						cmd = OS::Path::normalise(env.subst("#{PRE_CONVERT_SCRIPT_COMMAND} #{file}"))
						info"\t----------------> PRE CONVERT #{cmd}"
						
						status, stdout, stderr = systemu(cmd)
						
						# --- interpret output ---
						error "\n\n\n#{stderr.length} ERRORS FOUND!\n\n\n" if (stderr.length > 0 || status != 0)					
						stderr.each { |err| error "#{err}"; error_count += 1 }
						stdout.each do |out| 
							info("\t\t#{cmd} : #{out}") 
						end
					else
						info"\t----------------> NO PRE CONVERT #{cmd}"
					end
					
					# -- execute convert cmd ---
					status, stdout, stderr = systemu(cmd)
					
					generated_file = true
					
					# --- interpret output ---
					error "\n\n\n#{stderr.length} ERRORS FOUND!\n\n\n" if (stderr.length > 0 || status != 0)					
					stderr.each { |err| error "#{err}"; error_count += 1 }
					stdout.each do |out| 
						info("\t\t#{cmd} : #{out}") 
						if out.include?(PAR_CODE_GEN_NO_XML_TOKEN)
							generated_file = false 
						end
					end										
					
					info("\t\treturned #{status}")	
					
					if (status != 0)
						error "A parcodegen error occured"
						error "Parcodegen didn't write the error to stderr" if stderr.length==0
						error_count += 1
					elsif (status==0 && stderr.length>0)
						error "A parcodegen error occured : Parcodegen didn't return a non zero return code though"
					end
					
					# --- did a file get created? ---
					if generated_file and File.exist?(dst)
						report["converted_ok"] += 1
						# --- register as a file that was processed --
						processed_files << { "filename" => filename, "tmp_filename" => dst }
						info_black("\t\tCreated #{dst}")
					else
						info("\t\tDest file not created or invalid #{dst}")
					end
					
				elsif is_delete
					report["deleted"] += 1					
					processed_files << { "filename" => filename, "tmp_filename" => dst }
				end
			end

			info ""
			info "\tProcess completed. #{Time.now}"
			info ""
			report.sort.each { |key,val| info_black "\t*** Report: #{key} #{val}" }
			
			# if any files have not been copied this may indicate a disk fault - out of diskspace etc
			# we signal to abort - no files should be checked in.
			
			abort = false
			if (report["not_copied_ok"] > 0)
				abort = true
				error "A file did not get copied ok"
				error_count += 1
			end
			
		rescue Exception => ex
			error "Unexpected exception converting or copying files: "
			error "\t#{ex.message}"
			ex.backtrace.each { |m| error "\t#{m}"; }
			Process.exit -1
		end	

		return processed_files, error_count, abort
	end

	#------------------------------------------------------------------------------------------
	# Build a changelist of the final destination files to submit.
	#------------------------------------------------------------------------------------------
	def build_changelist(env, processed_files, dst_folder, error_count )		
		
		info "\t\t========build_changelist============"
		
		# --- create a changelist ---
		change_id = nil
		@p4.connect()
		raise Exception if not @p4.connected?
		change_id = @p4.create_changelist( comment() )
		raise Exception if change_id.nil?

		# --- sync on the dst folder to prevent possible sync/resolve issues ---
		@p4.run_sync( "#{dst_folder}/*.#{PAR_CODE_GEN_EXTENSION}" )

		# --- For each determine the equivalent filename and add it to the CL, but if it doesn't exist delete it  ---
		# * this means that if we change the RULES of the equivalent mapping there would be work in perforce to do by hand.
		# i.e. a stale file could remain in p4.

		info "\t\t========build_changelist : #{processed_files.length} processed_files"

		processed_files.each do |processed_file| 

			tmp_filename = processed_file["tmp_filename"]
			dest_filename = get_dest_filename(env, tmp_filename, dst_folder)

			if (File.exist?(tmp_filename) )			
				info "\t\tProcessed file exists #{tmp_filename}"					
				add_file( tmp_filename, dest_filename, change_id)
			else
				info "\t\tProcessed file missing #{tmp_filename} if it exists in p4 then it needs deleted."					
				delete_file( tmp_filename, dest_filename, change_id)
			end
		end

		change_id
	end

	#------------------------------------------------------------------------------------------
	# comment for changelist
	#------------------------------------------------------------------------------------------
	def comment()		
		"Automatically generated by #{__FILE__} v#{PAR_CODE_GEN_CI_VERSION}\n"
	end

	#------------------------------------------------------------------------------------------
	# delete a file in p4 and add into CL
	#------------------------------------------------------------------------------------------
	def delete_file( tmp_filename, filename, change_id)

		info "Filename  #{filename} deleted in CL"
		@p4.run_revert( filename )
		@p4.run_sync( filename )	
		@p4.run_delete( '-c', change_id.to_s, filename )														
	end

	#------------------------------------------------------------------------------------------
	# add a file to the CL
	#------------------------------------------------------------------------------------------
	def add_file( tmp_filename, dest_filename, change_id)
		
		# -- create the dir so we can copy the file here if required --
		dst_dir = OS::Path.get_directory(dest_filename)
		FileUtils::mkdir_p(dst_dir) if (not File.exists?(dst_dir) )			

		fstat = @p4.run_fstat( dest_filename ).shift	
		
		# --- edit or add file to CL ---
		if (fstat.nil? or (fstat['headAction'] and fstat['headAction'].include?("delete")))				
			info "\t\t\tCopy #{tmp_filename} to #{dest_filename}"
			
			FileUtils.rm(dest_filename, :force => true) if File.exists?(dest_filename)			
			FileUtils::cp(tmp_filename, dest_filename)
			info "\t\t\t#{dest_filename} added to CL"
			@p4.run_add( '-c', change_id.to_s, dest_filename )
		else	
			info "\t\t\tDest_filename  #{dest_filename} edited in CL"
						
			@p4.run_revert( dest_filename )
			@p4.run_sync( dest_filename )
			@p4.run_edit( '-c', change_id.to_s, dest_filename )
			info "\t\t\tCopy #{tmp_filename} to #{dest_filename}"
			FileUtils.rm(dest_filename, :force => true) if File.exists?(dest_filename)			
			FileUtils.cp(tmp_filename, dest_filename)
		end						
	end

	#------------------------------------------------------------------------------------------
	# submit the changelist
	#------------------------------------------------------------------------------------------
	def submit_changelist( change_id, enable_checkin, enable_revert_unchanged, error_count )
	
		files_pre_revert = @p4.run_opened( '-c', change_id.to_s )
		info "\t#{files_pre_revert.length} files before revert"
	
		if (enable_revert_unchanged)
			info "\tReverting unchanged files #{change_id}"
			@p4.run_revert( '-a', '-c', change_id.to_s, '//...')
		end
		
		files = @p4.run_opened( '-c', change_id.to_s )
		raise Exception if files.nil?
		
		num_reverted = files_pre_revert.length - files.length	
		info "\t#{num_reverted} files reverted"

		info_black "\tThere are #{files.size} files to submit."
		files.each { |file| info_black "\t\t#{file['depotFile']}"	}
		
		if ( enable_checkin )
			if ( files.size > 0 )		
				info_black "\tSubmitting file currently in #{change_id}"	
				submit_result = @p4.run_submit( '-c', change_id.to_s )				
			elsif ( 0 == files.size )
				info "\tDeleting #{change_id} no files changed."
				@p4.run_change('-d', change_id.to_s)
			end
		else
			info "\tCheckin is disabled the CL #{change_id} is pending."	
		end
	end

	#------------------------------------------------------------------------------------------
	# get the destination filename - based on the name of the file
	# some criteria is followed to compute an appropriate destination file destination.
	# *** IF THIS MAPPING CHANGES - A CLEAN UP OF PERFORCE METADATA FILES MAY BE REQUIRED! ***
	#------------------------------------------------------------------------------------------
	def get_dest_filename(env, tmp_filename, dst_folder )	

		filename = OS::Path.get_filename(tmp_filename)

		sub_folder 	= OS::Path.normalise(OS::Path.get_directory(tmp_filename))
		tmp_path 	= OS::Path.normalise( env.subst(TMP_FOLDER))				
		sub_folder.sub!(tmp_path,"")				
		
		ret = OS::Path::normalise( env.subst(OS::Path.combine(dst_folder, sub_folder, filename)))			
		info "\t#{filename} => #{ret}"
		ret
	end
	
end # class parcodegen_ci

#-----------------------------------------------------------------------------
# Application entry point
#-----------------------------------------------------------------------------
if __FILE__ == $0

	info "======================================================================================================="
	info_green "Running #{__FILE__} #{ARGV.join(" ")}"	
	info_green "https://devstar.rockstargames.com/wiki/index.php/Par_Code_Gen_Continuous_Integration"

	error_count     = 0 
	warning_count   = 0

	begin
		g_AppName = File::basename( __FILE__, '.rb' )
		g_ProjectName = ''
		g_BranchName = ''
		g_Project = nil
		g_Config = Pipeline::Config.instance()
			
		#---------------------------------------------------------------------
		# Parse Command Line.
		#---------------------------------------------------------------------
		opts, trailing = OS::Getopt.getopts( OPTIONS )
		
		if ( opts['help'] )
			puts OS::Getopt.usage( OPTIONS )
			puts ("Press Enter to continue...")
			$stdin.getc( )				
			Process.exit! 1
		end

		if ( ( 0 == trailing.size ) ) then
			error 'No trailing file arguments specified. Exiting.'
			puts OS::Getopt.usage( OPTIONS, { 'files' => TRAILING_DESC } )
			Process.exit( 2 )
		end				

		g_ProjectName = ( nil == opts['project'] ) ? '' : opts['project']
		project_exists = ( g_Config.projects.has_key?( g_ProjectName ) )
		if ( not project_exists ) then
			puts OS::Getopt.usage( OPTIONS )
			error "#{g_ProjectName} does not exist or its configuration is unreadable."
			Process.exit! 3
		end

		g_Project = g_Config.projects[ g_ProjectName ]
		g_Project.load_config( )
		if ( not g_Project.enabled ) then
			error "#{g_ProjectName} is not enabled on this machine.  Re-run installer."
			Process.exit! 4
		end

		g_BranchName = ( nil == opts['branch'] ) ? g_Project.default_branch : opts['branch']
		if ( not g_Project.branches.has_key?( g_BranchName ) ) then
			error "#{g_ProjectName} does not have branch #{g_BranchName} defined."
			Process.exit! 5
		end
		g_Branch = g_Project.branches[ g_BranchName ]

		g_DestFolder = ( nil == opts['dest_folder'] ) ? nil : opts['dest_folder']
		if ( g_DestFolder.nil? ) then
			error "no dest folder."
			Process.exit! 6
		end	

		g_enable_checkin 		= ( nil == opts['disablecheckin'] ) 			? true 	: false
		g_enable_revert_unchanged 	= ( nil == opts['disable_revert_unchanged'] ) 		? true 	: false
		g_pre_convert_script		= ( nil == opts['pre_convert_script'] ) 		? false : true
		
		# --- build environment ---
		env = Environment.new()
		g_Branch.fill_env( env ) unless ( g_Branch.nil? )
		g_Project.fill_env( env ) if ( g_Branch.nil? )


		info "g_pre_convert_script is #{g_pre_convert_script}"

		parcodegen_ci = ParCodeGenCi.new(g_pre_convert_script)

		#-------------------------------------------------------------------------
		# Parse the modifications xml file - get a list of LOCAL files to process
		# - some may not exist on the client 
		# - this is ok and needs handled.
		# - also recurse for wildcards in filesystem
		#-------------------------------------------------------------------------	
		
		files = []
		
		trailing.each do |wildcard|
			if (wildcard.downcase.include?(".xml"))
				info ""
				info_blue "*** Parsing modifications #{wildcard} \t#{Time.now}"		
				files += parcodegen_ci.parse_cc_modifications( wildcard, g_Project, g_BranchName )
				info_blue "*** Finished parsing #{files.length} modifications \t#{Time.now}"
			else
				info ""
				info_blue "*** Parsing wildcard #{wildcard}\t#{Time.now}"
				files_found = OS::FindEx::find_files_recurse( wildcard )	

				info "\tFiles wildcard: #{wildcard}, #{files_found.size} files found."
				files_found.each do |filename|
					info "\t\t#{filename}\n"
					files << filename
				end			
				info_blue "*** Finished parsing wildcard \t#{Time.now}"
			end
		end																			

		if ( files.length > 0 )

			#---------------------------------------------------------------------
			# Process files 
			#---------------------------------------------------------------------			
			info ""
			info_blue "*** Processing #{files.length} files \t#{Time.now}"
			
			copy_files = []
			convert_files = []
			
			files.each do |filename| 
				ext 		= OS::Path::get_extension(filename)
				copy_files << filename if COPY_EXTENSIONS.include?(ext)
				convert_files << filename if CONVERT_EXTENSIONS.include?(ext)
			end	
			
			copy_files.each do |cf|
				error "#{cf} is considered for copy AND convert, it cannot be BOTH" if (convert_files.include? cf)
			end
			
			#
			# DW: convert needs to happen before copy otherwise it may delete the copied file when it converts ( pcg quirk )
			#
			
			# --- blat tmp folder and ensure it exists. ---		
			dst_folder_root = OS::Path::normalise( env.subst(TMP_FOLDER) )
			FileUtils.rm_rf(dst_folder_root) if (File.exists?(dst_folder_root) )
			FileUtils::mkdir_p(dst_folder_root)													
			if not File.exists?dst_folder_root
				error_count += 1
				error "#{dst_folder_root} doesn't exist"
			else					
				info_blue "*** Started converting #{convert_files.length} files \t#{Time.now}"		
				processed_convert_files, error_count, abort = parcodegen_ci.process_files( env, convert_files, error_count, dst_folder_root )
				info_blue "*** Finished converting #{processed_convert_files.length} files \t#{Time.now}"		
			
				info_blue "*** Started copying #{copy_files.length} files \t#{Time.now}"	
				processed_copy_files, error_count, abort = parcodegen_ci.process_files( env, copy_files, error_count, dst_folder_root ) if (!abort)
				info_blue "*** Finished copy #{processed_copy_files.length} files \t#{Time.now}"						
			end
			
			processed_files = processed_convert_files + processed_copy_files
			processed_files = processed_files.uniq
			info_blue "*** Total #{processed_files.length} unique files \t#{Time.now}"						
			
			if (abort==false)			
				#---------------------------------------------------------------------
				# Build Changelist
				#---------------------------------------------------------------------					
				info ""
				info_blue "*** Build CL files \t#{Time.now}"			
				changelist = parcodegen_ci.build_changelist(env, processed_files, g_DestFolder, error_count )		
				info_blue "*** Build CL complete.\t#{Time.now}"

				#---------------------------------------------------------------------
				# Submit Changelist
				#---------------------------------------------------------------------			
				info ""
				info_blue "*** Submit CL \t#{Time.now}"
				parcodegen_ci.submit_changelist( changelist, g_enable_checkin, g_enable_revert_unchanged, error_count )		
				info_blue "*** Submit CL complete.\t#{Time.now}"
				info ""
			end
		else
			info_blue " Warning: no files where processed"
		end

	rescue Exception => ex
		error "Unhandled exception: #{ex.message}"
		error "Backtrace:"
		ex.backtrace.each { |m| error "\t#{m}" }
		Process.exit -1
	end

	ret_code = ( error_count > 0 ) ? -1 : 0

	info_blue "*** Script exiting with #{ret_code} since error_count is #{error_count} \t#{Time.now}"
	info "======================================================================================================="
	info ""
	info ""
	info ""
	Process.exit! ret_code

end #if __FILE__ == $0