gtav-src/tools_ng/ironlib/util/enum_maker.rb

#
# File:: enum_maker.rb
# Description:: Enum Maker - builds .sch file containing an enum of models found in a collection of .meta and .ide files.
#
#
# TODO : in .meta handle the @hideFrom="script"???
#
# Author:: Derek Ward <derek@rockstarnorth.com>
# Date:: 02 June 2011
#

#----------------------------------------------------------------------------
# Uses
#----------------------------------------------------------------------------

require 'RSG.Base.dll'
require 'RSG.Base.Configuration.dll'
require 'RSG.Base.Windows.dll'
require 'RSG.SourceControl.Perforce.dll'
include RSG::Base::Configuration
include RSG::Base::Logging
include RSG::Base::Logging::Universal
include RSG::Base::OS
include RSG::Base::Windows
include RSG::SourceControl::Perforce

require 'mscorlib'
require 'System'
require 'System.Core'
require 'System.Xml'
include System
include System::IO

require 'pipeline/os/options'
include Pipeline

#----------------------------------------------------------------------------
# Constants
#----------------------------------------------------------------------------

OPTIONS = [
	LongOption::new( 'help', 		LongOption::ArgType.None, 		'h', 	'display usage information.' ),
	LongOption::new( 'algorithm', 	LongOption::ArgType.Required,	'a', 	'set hashing algorithm (u32, u16, s16).' ),
	LongOption::new( 'scm', 		LongOption::ArgType.None, 		's',  	'build result file in the SCM' ),
	LongOption::new( 'submit', 		LongOption::ArgType.None, 		'sub', 	'submit result file to the SCM' ),
]

TRAILING = {
	'filenames' => 'output filename followed by any number of filelist, meta or ide filename(s)'
}

#
# Monkey patch integer class
class Integer
	def to_s32
		self >= (2**31) ? self - (2**32) : self
	end
end

#
# Enum maker class - reads input files (.txt, .ide, .meta, .xml ) and produces a .sch
class EnumMaker

	VERSION			= "1.1"
	VERBOSE 		= true

	XPATH_META 		= "//modelName | //Name"
	REGEX_IDE 		= /([\S]*),/
	REGEX_TXT		= /^([\S]*)$/
	REGEX_NAME		= /^\D[\w]+$/

	IDE_FILETYPE 		= "ide"
	META_FILETYPE 		= "meta"
	SCENEXML_FILETYPE 	= "xml"
	TXT_FILETYPE 		= "txt"	

	HARDCODED_ENTRY 	= "DUMMY_MODEL_FOR_SCRIPT=0,"
	COMMENT 		= "#"
	INFO_BLUE		= "[colourise=blue]INFO_MSG: "

	NUM_JUST		= 6
	L_JUST			= 100

	SCH_HARDCODED_ENTRIES 	= "#{HARDCODED_ENTRY.ljust(L_JUST,' ')}// hardcoded in script #{$0}\n"
	SCH_HEADER 		= "ENUM MODEL_NAMES\n"
	SCH_FOOTER 		= "ENDENUM\n"
	SCH_COMMENT 		= "//Built by enum_maker.rb version #{VERSION}\n"
	
	SCENE_XML_PARSER	= "$(toolsbin)/MapExport/SceneXmlEnumMaker.exe"
	
	EXCLUSIONS		= [ "NULL", "DEFAULT", "VEHICLE", "VEHICLE_TYPE_CAR", "PLAYER", "ROPE", "TOP", "SIGN", "FLAG" ]
	
	def self.log( )
		if ( @@log.nil? )
			LogFactory.Initialize()	
			LogFactory.CreateApplicationConsoleLogTarget( )
			@@log = LogFactory.ApplicationLog
		end
		@@log
	end	
	
	@@log = nil	
	
	def initialize( algorithm, scm = false, submit = false )
		@config = RSG::Base::Configuration::ConfigFactory::CreateConfig( )
		@project = @config.Project
		@algorithm = algorithm		
		@scm = @project.SCMConnect()
		@submit = submit
		scm_init() if scm
	end
	
	#
	# High level control
	def process( out_filename, input_files)	
		begin	
			if (@scm)
				change_list = @p4.CreatePendingChangelist( "Automatically Generated EnumMaker File" )
				@change_id = change_list.Number
				raise Exception if @change_id.nil?
				cc_info "Changelist #{@change_id} is created"

				cc_info "syncing #{out_filename} on #{@p4.port} #{@p4.client} #{@p4.user}" 
				@p4.Run('sync', true, out_filename)
			end
	
			if (not System::IO::File::Exists(out_filename ) )
				dir = System.IO.Path.GetDirectoryName( out_filename )
				FileUtils::mkdir_p( dir ) unless ( File::directory?( out_filename ) )
			end

			if (@scm)							
				cc_info "edit or add #{out_filename} on #{@p4.port} #{@p4.client} #{@p4.user}"
				
				@p4.ParseAndLogRecordSetForDepotFilename('edit', @p4.Run("edit", true, "-c", @change_id.to_s, out_filename))
				@p4.ParseAndLogRecordSetForDepotFilename('reopen', @p4.Run("reopen", true, "-c",@change_id.to_s, out_filename))
				@p4.ParseAndLogRecordSetForDepotFilename('reopen', @p4.Run("reopen", true, '-t', '+w', '-c', @change_id.to_s, out_filename))
			end

			names = read_files( input_files )
			cc_info "#{names.length} names have been read from all input files"
			write_outfile( out_filename, names) 
			
			if (@scm)
				cc_info "Reverting unchanged files #{@change_id}"
				@p4.ParseAndLogRecordSetForDepotFilename('revert', @p4.Run("revert", true, '-a', '-c', @change_id.to_s, '//...'))
				
				submit_changelist()
			end
				
		rescue Exception => ex
			error "Error: EnumMaker Process Exception: #{ex.message}"
			error "\tStacktrace: #{ex.backtrace.join('\n\r')}"	
		end	
	end	
	
	#---------------------------------------------------------------------
	# Private Methods
	#---------------------------------------------------------------------
		
	private

	#
	# Process all input files 
	def read_files( filenames )
			
		names = []
		filenames.each do |filename|
			result = read_file( filename )
			cc_info "#{filename} returned #{result.length} names..." 
			result.each do |res| 
				names << res if valid_name(res)
			end
		end	
		names
	end

	#
	# Write the outfile
	def write_outfile( outfile, names) 		
		cc_info "Writing to #{outfile}"

		File.open(outfile, "w+") do |o|
			o.write(SCH_COMMENT)
			o.write(SCH_HEADER)
			o.write(SCH_HARDCODED_ENTRIES)
			num1 = names.length
			names.uniq!
			names.sort! { |a,b| a["name"] <=> b["name"] }	
			names.each_with_index { |n,idx| names[idx-1]["name"] = nil if ( idx-1 >= 0 and ( names[idx-1]["name"] == n["name"]) ) }
			names.delete_if { |n| n["name"] == nil }
			
			num2 = names.length
			cc_info("#{num1-num2} duplicates removed resulting in #{num2} names")
					
			names.each do |name| 				
				formatted_name = name_format(name["name"], name == names[-1], name["filename"])
				
				debug formatted_name
				
				o.write("#{formatted_name}\n")				
			end
			o.write(SCH_FOOTER)
		end
	end
	
	#
	# Process an input file
	def read_file( filename )
		error("Error: file #{filename} doesn't exist") unless File.exist?(filename)
		
		filetype = Path::GetExtension(filename).Replace(".", "");
		case filetype.downcase
			when IDE_FILETYPE
				result = read_ide( filename )
				cc_info "\t#{filename} returned #{result.length} names" 
			when SCENEXML_FILETYPE
				result = read_scenexml( filename )
				cc_info "\t#{filename} returned #{result.length} names" 
			when META_FILETYPE
				result = read_meta( filename )
				cc_info "\t#{filename} returned #{result.length} names" 
			when TXT_FILETYPE
				result = read_txt( filename )	
				cc_info "\t#{filename} returned #{result.length} names" 
			else
				puts "Warning: File #{filename} is not a recognised filetype #{$0}, it must be either .#{IDE_FILETYPE} .#{META_FILETYPE} .#{SCENEXML_FILETYPE} or .#{IDE_FILETYPE}"
		end	
		return result
	end
	
	def normalize(file)	
		return System::IO::Path::GetFullPath(System::Uri.new(file).LocalPath).Replace("\\","/")
              .TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)
              .ToLowerInvariant();
	end
	
	#
	# Process the file list to read.
	def read_txt( filename )	

		error("Error: file #{filename} doesn't exist") unless File.exist?(filename)
		result = []
		File::open( filename ) do |fp|
			lines = fp.readlines()
			cc_info "...Parsing txt file       (#{num_just(lines.length)} lines) \t: #{filename}"
			lines.each do |line|
				line.strip!
				next if ( 0 == line.size or line.StartsWith( COMMENT ))				
				result += read_file(normalize(@config.Project.DefaultBranch.Environment.Subst($1))) if line =~ REGEX_TXT					
			end
		end
		cc_info "\t#{result.length} names derived from #{filename}"
		result
	end	
	
	#
	# Process a .xml scenexml file ( xml )
	def read_scenexml( filename )

		exists = File.exist?(filename)

		error("Error: file #{filename} doesn't exist") unless exists
		result = []

		if (exists)	
			sceneXmlParser = normalize(@config.Project.DefaultBranch.Environment.Subst(SCENE_XML_PARSER))
			args = "--nopopups #{filename}"
			command = "#{sceneXmlParser} #{args}"
	
			cc_info ">>> Running Command : #{command}"
			#status, stdout, stderr = systemu(command)
			
			process = System::Diagnostics::Process.new
			process.StartInfo.FileName = sceneXmlParser
			process.StartInfo.RedirectStandardOutput = true
			process.StartInfo.RedirectStandardError = true
			process.StartInfo.Arguments = args
			process.StartInfo.UseShellExecute = false
			process.Start()

			
			stdout = process.StandardOutput.ReadToEnd().Split(Environment.NewLine.ToCharArray())
			stderr = process.StandardError.ReadToEnd().Split(Environment.NewLine.ToCharArray())
			process.WaitForExit()
			
			cc_info "...Parsing scenexml file (#{num_just(stdout.length)} names) \t: #{filename}"

			cc_info "#{command} returned #{process.ExitCode}"

			stdout.each do |name| 
				if not name.nil?
				name = name.ToUpper() # DW : mmm I did have this commented out but can't remember why
				result << { "name" => name.gsub(/\n/,"").gsub(/\s/,""), "filename" => filename } if name
				end
			end

			stderr.each { |err| error err }
		end
		result
	end			
	
	#
	# Process a .meta file ( xml )
	def read_meta( filename )
	
		error("Error: file #{filename} doesn't exist") unless File.exist?(filename)
	
		result = []
		#doc = System::Xml::Linq::XDocument::Load(filename)
		#names = doc.root.XPathEvaluate( XPATH_META )
		
		doc = System::Xml::XmlDocument.new
		doc.load(filename)
		
		nav = doc.CreateNavigator();
		# Compile a standard XPath expression
		expr = nav.Compile(XPATH_META);
		names = nav.Select(expr);
		
		cc_info "...Parsing meta file      (#{num_just(names.Count)} nodes) \t: #{filename}"
			
		names.each do |name| 
			#info name.text.upcase if name and name.text 
			result << { "name" => name.Value.upcase, "filename" => filename } if name and name.Value 
		end
		
		result
	end		
	
	#
	# Process a .ide file ( text )
	# eg. V_Club_BarChair, V_Club_TXD, 100, 537001984, 0, -0.340655, -0.334494, -0.001928, 0.340654, 0.334494, 0.879548, -5.066395E-07, 0, 0.43881, 0.6497551, null
	def read_ide( filename )
	
		error("Error: file #{filename} doesn't exist") unless File.exist?(filename)
	
		result = []
		File::open( filename ) do |fp|	
			lines = fp.readlines()
			cc_info "...Parsing ide file       (#{num_just(lines.length)} lines) \t: #{filename}"
			lines.each do |line|
				line.strip!
				next if ( 0 == line.size or line.starts_with( COMMENT ))				
				result << { "name" => $1.upcase, "filename" => filename } if line =~ REGEX_IDE
			end
		end
		result
	end
	
	#
	# Present the name in a format for the 'csv' list in the output file.
	def name_format( name, last, filename = "" )
		sep = "," unless last
		str = "#{name.upcase}=#{string_hash(name)}#{sep}" 
		"#{str.ljust(L_JUST," ")}// #{filename}"
	end			

	#
	# Hash the string.
	def string_hash( str )
		case @algorithm
		when :u16
			RSG::Base::Security::Cryptography::Hash16U::ComputeHash( str ).to_s
		when :s16
			RSG::Base::Security::Cryptography::Hash16::ComputeHash( str ).to_s
		else
			RSG::Base::Security::Cryptography::OneAtATime::ComputeHash( str ).to_s32().to_s 
		end
	end
		
	#
	# Is the name valid / desired in the outfile?
	def valid_name( name )	
	
		puts (">>>>>>>>>>>>>>>>>Excluded #{name}") if name["name"].include?(" ")
		puts (">>>>>>>>>>>>>>>>>Excluded #{name}") if EXCLUSIONS.include?(name["name"])
		return false if name["name"].include?(" ")
		return false if EXCLUSIONS.include?(name["name"])
		name["name"] =~ REGEX_NAME
	end	
	
	#
	# Initialise SCM
	def scm_init()
		@p4 = @project.SCMConnect()	
	end

	#
	# submit changelist - also deletes empty changelist.
	def submit_changelist( )
		files = @p4.Run("opened", true, '-c', @change_id.to_s)
		raise Exception if files.nil?

		cc_info "There are #{files.Records.size} files to submit."
		files.each do |file|
			cc_info "**** #{file['depotFile']} has been updated and will be submitted. ****"			
		end

		if ( files.Records.size > 0 ) then
			if @submit
				cc_info "**** Submitting File currently in #{@change_id} ****"	
				submit_result = @p4.Run("submit", true, '-c', @change_id.to_s)
				puts submit_result.to_s
			else
				cc_info "**** CL #{@change_id} is waiting for you to submit. ****"	
			end
		elsif ( 0 == files.Records.size ) then
			cc_info "Deleting #{@change_id_dest} no files changed."
			@p4.Run("change", true, '-d', @change_id.to_s)
		end
	rescue Exception => ex
		error "Error: Unhandled exception in submit_changelist: #{ex.message}"
	end	
	
	def num_just( num )
		num.to_s.rjust(NUM_JUST," ")
	end
	
	def info( msg )
		EnumMaker::log().Message(msg)
	end

	def cc_info( msg )
		info("#{INFO_BLUE}#{msg}")
	end

	def debug( msg )		
		EnumMaker::log().Debug(msg)
	end
	
	def error( msg )
		$stderr.puts msg
		EnumMaker::log().Error(msg)
	end

	def warning( msg )
		puts msg
		EnumMaker::log().Warning(msg)
	end
		
end # class EnumMaker


#----------------------------------------------------------------------------
# Implementation
#----------------------------------------------------------------------------
if ( __FILE__ == $0 ) then

	begin
		g_Options = OS::Options::new( OPTIONS )
		
		if ( g_Options.is_enabled?('help') )
			puts g_Options.usage()
			exit( 1 )
		end
		
		algorithm = g_Options.get('algorithm').to_sym if ( g_Options.has_option?( 'algorithm' ) )
		algorithm = :u32 if ( algorithm.nil? )

		scm = g_Options.is_enabled?('scm')
		
		submit = g_Options.is_enabled?('submit')

		files = g_Options.trailing()		
		output_filename = files[0]
		input_filenames = files[1..-1]
		
		enum_maker = EnumMaker.new( algorithm, scm, submit )
		enum_maker.process( output_filename, input_filenames )
		LogFactory.FlushApplicationLog();
				
	rescue Exception => ex
	
		puts "Unhandled exception: #{ex.message}."
		puts ex.backtrace.join( "\n" )
	end
end

# data_mk_hash.rb