542 lines
20 KiB
Ruby
Executable File
542 lines
20 KiB
Ruby
Executable File
#
|
|
# File:: incredibuild_xge.rb
|
|
# Description:: Utility classes to help you build an XGE XML interface file.
|
|
#
|
|
# Author:: David Muir <david.muir@rockstarnorth.com>
|
|
# Date:: 1 April 2009
|
|
#
|
|
|
|
#----------------------------------------------------------------------------
|
|
# Uses
|
|
#----------------------------------------------------------------------------
|
|
require 'pipeline/config/project'
|
|
require 'pipeline/config/branch'
|
|
require 'pipeline/config/targets'
|
|
require 'pipeline/os/path'
|
|
require 'pipeline/util/incredibuild'
|
|
require 'pipeline/util/rage'
|
|
require 'rexml/document'
|
|
require 'systemu'
|
|
include REXML
|
|
|
|
#----------------------------------------------------------------------------
|
|
# Implementation
|
|
#----------------------------------------------------------------------------
|
|
module Pipeline
|
|
module Util
|
|
|
|
#
|
|
# == Description
|
|
# Define an XGE module to act as a container to our other helper classes.
|
|
#
|
|
module XGE
|
|
|
|
# Return temporary directory for project and branch for XGE files.
|
|
# Supports native Project and Branch objects as well as Strings.
|
|
def XGE::get_temp_dir( project = nil, branch = nil )
|
|
path = OS::Path::combine( Pipeline::Config::instance().temp, 'xge' )
|
|
path = OS::Path::combine( path, project.name ) if ( project.is_a?( Pipeline::Project ) )
|
|
path = OS::Path::combine( path, project ) if ( project.is_a?( String ) )
|
|
path = OS::Path::combine( path, branch.name ) if ( branch.is_a?( Pipeline::Branch ) )
|
|
path = OS::Path::combine( path, branch ) if ( branch.is_a?( String ) )
|
|
path
|
|
end
|
|
|
|
# Start the project via the 'xgconsole' command. Return true on
|
|
# command success; false otherwise.
|
|
def XGE::start( filename, logfile = nil, rebuild = false, showcmd = true, showtime = true, openmonitor = true, showmon = true )
|
|
|
|
monitor_file = (logfile and showmon) ? OS::Path.replace_ext(logfile,"ib_mon") : nil
|
|
|
|
xgconsole_dir = OS::Path::normalise( Incredibuild::installdirs().values.last )
|
|
xgconsole_path = OS::Path::combine( xgconsole_dir, 'xgconsole.exe' )
|
|
options = "#{filename} "
|
|
options += "/rebuild " if ( rebuild )
|
|
options += "/showcmd " if ( showcmd )
|
|
options += "/showtime " if ( showtime )
|
|
options += "/mon=#{monitor_file} " if ( monitor_file )
|
|
options += "/openmonitor " if ( openmonitor )
|
|
options += "/out=#{logfile}" unless ( logfile.nil? )
|
|
command_line = "#{xgconsole_path} #{options}"
|
|
|
|
XGE::log().info( "Starting #{xgconsole_path}... arguments #{options}..." )
|
|
|
|
# -- execute ---
|
|
begin
|
|
status, stdout, stderr = systemu(command_line)
|
|
rescue Exception => ex
|
|
puts "Exception: #{ex.message}"
|
|
puts "\tStacktrace: #{ex.backtrace.join('\n\r')}"
|
|
|
|
XGE::log().warn( "Systemu hit an execption, it will now run the command with System(). the exception was #{ex.message} the command was #{command_line}" )
|
|
# Gracefully handle exceptions in systemu itself ( NOT WHAT IT IS RUNNING - don't be confused. )
|
|
system( command_line )
|
|
status = 0 # force it to be happy, system doesn't return a code for the process that run anyway.
|
|
end
|
|
|
|
# -- check if failed ---
|
|
# http://www.xoreax.com/webhelp/pages/xgconsole.htm
|
|
# Return Codes
|
|
# xgConsole returns the following return codes:
|
|
# 0 - No errors were encountered
|
|
# 1 - Errors were encountered during the operation.
|
|
# 2 - A fatal IncrediBuild error was encountered (Invalid parameters, input file not found etc.).
|
|
# 3 - The operation was stopped before completing.
|
|
status = status.to_i
|
|
if ( status != 0 and ( not (1024 == ( status & 1024 ) ) ) ) # for some bizarre reason the status fo 1024 indicates a warning( is it a bit field the status ? )
|
|
|
|
XGE::log().error( "Error: XGE command completed with error return code: #{status}" )
|
|
|
|
if ( not File::exists?( xgconsole_path ) ) then
|
|
XGE::log().error( "xgconsole command failed. Is XGE installed?" )
|
|
XGE::log().error( "Xoreax Incredibuild install directory: #{xgconsole_dir}" )
|
|
end
|
|
false
|
|
else
|
|
XGE::log().info( "XGE command completed with return code: #{status}" )
|
|
true
|
|
end
|
|
end
|
|
|
|
#
|
|
# == Description
|
|
# Incredibuild XGE XML Interface helper class. This class and set of
|
|
# utility classes in the Pipeline::Util::XGE module help to construct
|
|
# the XML input file.
|
|
#
|
|
# Previously the Convert RAGE interface would create this manually. Erk!
|
|
#
|
|
# === References
|
|
# http://www.xoreax.com/webhelp/index.html?page=pages%2fxge_help_main.htm
|
|
#
|
|
# === Future Plans
|
|
#
|
|
#
|
|
# === Example Usage
|
|
# DHM TODO.
|
|
#
|
|
class Project
|
|
FORMAT_VERSION = 1
|
|
|
|
attr_reader :name # String name.
|
|
attr_reader :environment # XGE::Environment object.
|
|
attr_accessor :working_directory # String working directory.
|
|
attr_reader :tasks # Array of Task and TaskGroup objects.
|
|
|
|
def initialize( name, tasks = [], env = nil, work_dir = nil )
|
|
@name = name
|
|
@environment = env
|
|
@working_directory = work_dir
|
|
@tasks = tasks
|
|
end
|
|
|
|
#
|
|
# Add a TaskGroup or Task to this project.
|
|
#
|
|
def add_task( task )
|
|
throw ArgumentError.new( "Invalid Task or TaskGroup object (#{task.class})." ) \
|
|
unless ( task.is_a?( XGE::TaskGroup ) or task.is_a?( XGE::Task ) )
|
|
@tasks << task
|
|
end
|
|
|
|
#
|
|
# Return unique environments used by the Project.
|
|
#
|
|
def get_unique_environments( )
|
|
envs = []
|
|
envs << @environment if ( @environment.is_a?( XGE::Environment ) )
|
|
@tasks.each do |task|
|
|
next unless ( task.is_a?( XGE::TaskGroup ) )
|
|
envs += task.get_unique_environments()
|
|
end
|
|
# return only unique environments.
|
|
envs.uniq()
|
|
end
|
|
|
|
#
|
|
#
|
|
#
|
|
def write( filename )
|
|
XGE::log().info( "Writing XGE project XML to #{filename}." )
|
|
|
|
FileUtils::mkdir_p( OS::Path::get_directory( filename ) ) \
|
|
unless ( File::directory?( OS::Path::get_directory( filename ) ) )
|
|
File.open( filename, 'w' ) do |file|
|
|
outputXML = Document.new()
|
|
outputXML << XMLDecl.new()
|
|
outputXML << to_xml( )
|
|
|
|
fmt = REXML::Formatters::Pretty.new()
|
|
fmt.write( outputXML, file )
|
|
end
|
|
end
|
|
|
|
#
|
|
# Return representation of this Project in XML suitable for the
|
|
# XGE input file.
|
|
#
|
|
def to_xml( )
|
|
# Find all of our unique environments from Project and Task Groups.
|
|
environments = get_unique_environments( )
|
|
|
|
root_elem = Element.new( 'BuildSet' )
|
|
root_elem.add_attribute( 'FormatVersion', FORMAT_VERSION.to_s )
|
|
envs_elem = root_elem.add_element( 'Environments' )
|
|
environments.each do |env|
|
|
envs_elem.add_element( env.to_xml( ) )
|
|
end
|
|
proj_elem = root_elem.add_element( 'Project' )
|
|
|
|
# Project requires an Environment. If not explicitly set then,
|
|
# we will default to first environment.
|
|
if ( @environment.is_a?( XGE::Environment ) ) then
|
|
proj_elem.add_attribute( 'Env', @environment.name )
|
|
else
|
|
proj_elem.add_attribute( 'Env', environments.first.name )
|
|
end
|
|
|
|
proj_elem.add_attribute( 'Name', @name )
|
|
proj_elem.add_attribute( 'WorkingDir', @working_directory ) if ( @working_directory.is_a?( String ) )
|
|
@tasks.each do |task|
|
|
proj_elem.add_element( task.to_xml() )
|
|
end
|
|
root_elem
|
|
end
|
|
end
|
|
|
|
#
|
|
# == Description
|
|
# XGE environment abstraction. An XGE project is executed within the
|
|
# confines of an Environment.
|
|
#
|
|
# The environment defines the tools that the project can utilise and a
|
|
# set of project-specific variable/value pairs.
|
|
#
|
|
class Environment
|
|
attr_reader :name # String environment name (referenced by Project)
|
|
attr_reader :tools # Array of XGE::Tool objects.
|
|
attr_reader :variables # Hash of String => Object variables.
|
|
|
|
def initialize( name, tools = [], variables = {} )
|
|
@name = name
|
|
@tools = tools
|
|
@variables = variables
|
|
end
|
|
|
|
def add_tool( tool )
|
|
throw ArgumentError.new( "Invalid XGE::Tool specified (#{tool.class})." ) \
|
|
unless ( tool.is_a?( Pipeline::Util::XGE::Tool ) )
|
|
@tools << tool
|
|
end
|
|
|
|
def add_variable( varname, varvalue )
|
|
@variables[varname] = varvalue
|
|
end
|
|
|
|
#
|
|
# Return representation of this environment in XML suitable for
|
|
# the XGE input file.
|
|
#
|
|
def to_xml( )
|
|
env_elem = Element.new( 'Environment' )
|
|
env_elem.add_attribute( 'Name', @name )
|
|
tools_elem = Element.new( 'Tools' )
|
|
@tools.each do |tool|
|
|
tools_elem.add_element( tool.to_xml( ) )
|
|
end
|
|
env_elem.add_element( tools_elem )
|
|
variables_elem = Element.new( 'Variables' )
|
|
@variables.each_pair do |varname, varvalue|
|
|
var_elem = Element.new( 'Variable' )
|
|
var_elem.add_attribute( 'Name', varname )
|
|
var_elem.add_attribute( 'Value', varvalue.to_s )
|
|
variables_elem.add_element( var_elem )
|
|
end
|
|
env_elem.add_element( variables_elem )
|
|
env_elem
|
|
end
|
|
end
|
|
|
|
#
|
|
# == Description
|
|
# XGE tool abstraction.
|
|
#
|
|
# A common case for creating tools is for a project branch's
|
|
# Ragebuilder. As this is a common case a class method has been added
|
|
# to return this information.
|
|
#
|
|
class Tool
|
|
attr_reader :name # Tool name; used in Project Tasks.
|
|
attr_reader :output_prefix # Output prefix string printed during build.
|
|
attr_reader :group_prefix # Group prefix string printed during build.
|
|
attr_reader :masks # Output file masks created during build.
|
|
attr_reader :allow_remote # Allow remote tool execution.
|
|
attr_reader :path # Absolute path to tool (e.g. exe).
|
|
attr_reader :params # Tool command line parameters.
|
|
attr_reader :allow_restart_on_local # Allow restart on local agent
|
|
attr_reader :time_limit # Number of seconds for a XGE task.
|
|
|
|
def initialize( name, output_prefix, group_prefix, masks, remote, path, params, allow_restart = false, time_limit = nil )
|
|
@name = name
|
|
@output_prefix = output_prefix
|
|
@group_prefix = group_prefix
|
|
@output_mask = masks
|
|
@allow_remote = remote
|
|
@path = path
|
|
@params = params
|
|
@allow_restart_on_local = allow_restart
|
|
@time_limit = time_limit
|
|
end
|
|
|
|
#
|
|
# Return representation of this tool in XML suitable for the XGE
|
|
# input file.
|
|
#
|
|
def to_xml( )
|
|
toolelem = Element.new( 'Tool' )
|
|
toolelem.add_attribute( 'Name', @name )
|
|
toolelem.add_attribute( 'OutputPrefix', @output_prefix )
|
|
toolelem.add_attribute( 'OutputFileMasks', @output_mask )
|
|
toolelem.add_attribute( 'AllowRemote', @allow_remote.to_s )
|
|
toolelem.add_attribute( 'Path', @path )
|
|
toolelem.add_attribute( 'Params', @params )
|
|
toolelem.add_attribute( 'GroupPrefix', @group_prefix )
|
|
toolelem.add_attribute( 'AllowRestartOnLocal', @allow_restart_on_local )
|
|
toolelem.add_attribute( 'TimeLimit', @time_limit ) unless ( @time_limit.nil? )
|
|
toolelem
|
|
end
|
|
|
|
#----------------------------------------------------------------
|
|
# Class Methods
|
|
#----------------------------------------------------------------
|
|
|
|
#
|
|
# Utility class method which will construct an XGE::Tool object
|
|
# to execute a Ruby script.
|
|
#
|
|
def Tool::ruby( caption, script, options = '', remote = true, masks = '*.*', allow_restart = true )
|
|
|
|
# http://stackoverflow.com/questions/213368/how-can-i-reliably-discover-the-full-path-of-the-ruby-executable
|
|
rubypath = OS::Path::combine( ::Config::CONFIG['bindir'], ::Config::CONFIG['ruby_install_name']).sub(/.*\s.*/m, '"\&"') + ".exe"
|
|
options = " #{script} #{options}"
|
|
Tool.new( "Ruby", caption, caption, masks, remote, rubypath, options, allow_restart )
|
|
end
|
|
|
|
#
|
|
# Utility class method which will construct an XGE::Tool object
|
|
# from our standard Project, Branch and Target objects.
|
|
#
|
|
def Tool::from_project_ragebuilder( project, branch, target, rebuild = false )
|
|
throw ArgumentError.new( "Invalid Project object (#{project.class})." ) \
|
|
unless ( project.is_a?( Pipeline::Project ) )
|
|
throw ArgumentError.new( "Invalid Branch object (#{branch.class})." ) \
|
|
unless ( branch.is_a?( Pipeline::Branch ) )
|
|
throw ArgumentError.new( "Invalid Target object (#{target.class})." ) \
|
|
unless ( target.is_a?( TargetBase ) )
|
|
|
|
c = Pipeline::Config::instance()
|
|
projtools = RageConvertTool::parse( project, branch.name )
|
|
projtool = projtools[target.name]
|
|
|
|
options = " -nopopups -rebuild #{rebuild} -build #{branch.build}"
|
|
options += " -shader #{branch.shaders} -shaderdb #{OS::Path::combine(branch.shaders, 'db')}"
|
|
## options += " -temporary false"
|
|
## uniLogFile = LogSystem::instance().ulog_filename
|
|
## options += " -uLogFileName #{uniLogFile}" unless uniLogFile==nil
|
|
options += " #{projtool.options}"
|
|
options += " -output"
|
|
Tool.new( "Ragebuilder_#{target.name}", 'Convert file...', 'Convert remotely...', '*.*', true, projtool.path, options, false, 3600 )
|
|
end
|
|
|
|
#
|
|
# Utility class method which will construct an XGE::Tool object
|
|
# from our standard Project, Branch and Target objects and force
|
|
# it to run on the local machine.
|
|
#
|
|
def Tool::from_project_ragebuilder_local( project, branch, target, rebuild = false )
|
|
throw ArgumentError.new( "Invalid Project object (#{project.class})." ) \
|
|
unless ( project.is_a?( Pipeline::Project ) )
|
|
throw ArgumentError.new( "Invalid Branch object (#{branch.class})." ) \
|
|
unless ( branch.is_a?( Pipeline::Branch ) )
|
|
throw ArgumentError.new( "Invalid Target object (#{target.class})." ) \
|
|
unless ( target.is_a?( TargetBase ) )
|
|
|
|
c = Pipeline::Config::instance()
|
|
projtools = RageConvertTool::parse( project, branch.name )
|
|
projtool = projtools[target.name]
|
|
|
|
options = " -nopopups -rebuild #{rebuild} -build #{branch.build}"
|
|
options += " -shader #{branch.shaders} -shaderdb #{OS::Path::combine(branch.shaders, 'db')}"
|
|
options += " -temporary false"
|
|
uniLogFile = LogSystem::instance().ulog_filename
|
|
options += " -uLogFileName #{uniLogFile}" unless uniLogFile==nil
|
|
options += " #{projtool.options}"
|
|
options += " -output"
|
|
Tool.new( "RagebuilderLocal_#{target.name}", 'Convert file...', 'Convert locally...', '*.*', false, projtool.path, options, false, 3600 )
|
|
end
|
|
end
|
|
|
|
#
|
|
# == Description
|
|
# XGE TaskGroup abstraction. An XGE Project contains an Array of Task
|
|
# and TaskGroup objects. A TaskGroup contains an Array of Task and
|
|
# sub-TaskGroup objects.
|
|
#
|
|
class TaskGroup
|
|
attr_reader :name # String name of TaskGroup.
|
|
attr_reader :tool # XGE::Tool object (optional).
|
|
attr_reader :working_directory # Working path (optional).
|
|
attr_reader :environment # XGE::Environment Object (optional).
|
|
attr_reader :tasks # Array of Task and TaskGroup objects.
|
|
attr_reader :dependson # List of Task and TaskGroup names this group depends on for executing.
|
|
|
|
def initialize( name, tasks = [], env = nil, tool = nil, working_dir = nil )
|
|
throw ArgumentError.new( "Invalid tasks Array (#{tasks.class})." ) \
|
|
unless ( tasks.is_a?( Array ) )
|
|
@name = name
|
|
@tasks = tasks
|
|
@tool = tool
|
|
@environment = env
|
|
@working_directory = working_dir
|
|
@dependson = []
|
|
end
|
|
|
|
#
|
|
# Return unique environments used by the TaskGroup.
|
|
#
|
|
def get_unique_environments( )
|
|
|
|
envs = []
|
|
envs << @environment if ( @environment.is_a?( XGE::Environment ) )
|
|
@tasks.each do |task|
|
|
next unless ( task.is_a?( XGE::TaskGroup ) )
|
|
envs += task.get_unique_environments()
|
|
end
|
|
# return only unique environments.
|
|
envs.uniq()
|
|
|
|
envs
|
|
end
|
|
|
|
#
|
|
# Return representation of this TaskGroup in XML suitable for the
|
|
# XGE input file.
|
|
#
|
|
def to_xml( )
|
|
elem = Element.new( 'TaskGroup' )
|
|
elem.add_attribute( 'Env', @environment.name ) if ( @environment.is_a?( XGE::Environment ) )
|
|
elem.add_attribute( 'Name', @name )
|
|
elem.add_attribute( 'WorkingDir', @working_directory ) if ( @working_directory.is_a?( String ) )
|
|
elem.add_attribute( 'Tool', @tool.name ) if ( @tool.is_a?( XGE::Tool ) )
|
|
elem.add_attribute( 'DependsOn', @dependson.join(';') ) if ( @dependson.size > 0)
|
|
# Look through Task and TaskGroup elements serialising as we go.
|
|
@tasks.each do |task|
|
|
elem.add_element( task.to_xml() )
|
|
end
|
|
elem
|
|
end
|
|
end
|
|
|
|
#
|
|
# == Description
|
|
# XGE Task abstraction. A XGE Project currently contains an Array of
|
|
# these Task objects.
|
|
#
|
|
# === Future Plans
|
|
# Add input, output and depenedency filename support.
|
|
#
|
|
class Task
|
|
attr_reader :name # Task name.
|
|
attr_reader :source_file # Source file for task.
|
|
attr_accessor :caption # Caption as display in Build Monitor (optional)
|
|
attr_accessor :output_files # Array of output filenames.
|
|
attr_accessor :input_files # Array of additional input filenames.
|
|
attr_accessor :tool # Tool object.
|
|
attr_accessor :parameters # Parameters string.
|
|
attr_accessor :inherit_params # Bool, add $(inherited:params) to parameter list.
|
|
attr_accessor :dependson # List of Task and TaskGroup names this group depends on for executing.
|
|
|
|
def initialize( name, source )
|
|
throw ArgumentError.new( "Invalid source file specified. Does '#{source}' exist?" ) \
|
|
unless ( ( not source.nil? ) or ( File::exists?( source ) and File::readable?( source ) ) )
|
|
@name = name
|
|
@source_file = source
|
|
@caption = nil
|
|
@output_files = []
|
|
@input_files = []
|
|
@tool = nil
|
|
@parameters = nil
|
|
@inherit_params = true
|
|
@dependson = []
|
|
end
|
|
|
|
#
|
|
# Return representation of this Task in XML suitable for the XGE
|
|
# input file.
|
|
#
|
|
def to_xml( )
|
|
elem = Element.new( 'Task' )
|
|
elem.add_attribute( 'Name', name )
|
|
elem.add_attribute( 'SourceFile', @source_file )
|
|
elem.add_attribute( 'Caption', @caption ) if ( @caption.is_a?( String ) )
|
|
elem.add_attribute( 'OutputFiles', @output_files.join(';') ) if ( @output_files.size > 0 )
|
|
elem.add_attribute( 'InputFiles', @input_files.join(';') ) if ( @input_files.size > 0 )
|
|
elem.add_attribute( 'Tool', @tool.name ) if ( @tool.is_a?( XGE::Tool ) )
|
|
elem.add_attribute( 'Params', "#{@parameters} $(inherited:params)" ) if ( @inherit_params.is_a?( TrueClass ) )
|
|
elem.add_attribute( 'Params', "#{@parameters}" ) if ( @inherit_params.is_a?( FalseClass ) )
|
|
elem.add_attribute( 'DependsOn', @dependson.join(';') ) if ( @dependson.size > 0 )
|
|
elem
|
|
end
|
|
end
|
|
|
|
# Return the XGE-module-wide log object.
|
|
def XGE::log( )
|
|
@@log = Pipeline::Log::new( 'xge' ) if ( @@log.nil? )
|
|
@@log
|
|
end
|
|
|
|
#--------------------------------------------------------------------
|
|
# Module Private
|
|
#--------------------------------------------------------------------
|
|
private
|
|
@@log = nil
|
|
end # XGE module
|
|
|
|
end # Util module
|
|
end # Pipeline module
|
|
|
|
if ( __FILE__ == $0 ) then
|
|
require 'pipeline/config/projects'
|
|
include Pipeline::Util
|
|
|
|
|
|
c = Pipeline::Config::instance()
|
|
p = c.projects['jimmy']
|
|
p.load_config()
|
|
b = p.branches['dev']
|
|
t_360 = b.targets['xbox360']
|
|
t_ps3 = b.targets['ps3']
|
|
|
|
# Test conversion XML file...
|
|
env_360 = XGE::Environment.new( 'env_360' )
|
|
env_360.add_variable( 'BinVar', c.toolsroot )
|
|
env_ps3 = XGE::Environment.new( 'env_PS3' )
|
|
env_ps3.add_variable( 'BinVar', c.toolsroot )
|
|
env_360.add_tool( XGE::Tool::from_project_ragebuilder( p, b, t_360, true ) )
|
|
env_ps3.add_tool( XGE::Tool::from_project_ragebuilder( p, b, t_ps3, true ) )
|
|
|
|
grp_360 = XGE::TaskGroup.new( '360 Asset Group', [], env_360, env_360.tools[0] )
|
|
grp_360.tasks << XGE::Task.new( 'Map alpvil_n', 'x:/jimmy/build/dev/independent/levels/alpine/alp_cable.img' )
|
|
grp_ps3 = XGE::TaskGroup.new( 'PS3 Asset Group', [], env_ps3, env_ps3.tools[0] )
|
|
grp_ps3.tasks << XGE::Task.new( 'Map alpvil_n', 'x:/jimmy/build/dev/independent/levels/alpine/alp_cable.img' )
|
|
|
|
project = XGE::Project.new( "#{p.uiname} Asset Conversion" )
|
|
project.add_task( grp_360 )
|
|
project.add_task( grp_ps3 )
|
|
project.write( 'x:/test.xml' )
|
|
|
|
puts "Done."
|
|
end
|
|
|
|
# incredibuild_xge.rb
|