613 lines
18 KiB
Ruby
Executable File
613 lines
18 KiB
Ruby
Executable File
#!/usr/bin/env ruby
|
|
|
|
#VFX File Dependency Chain Validation Tool
|
|
#Check links between dependent files and assets for breaks and anomalies
|
|
#and report such incidents back to the user in a clear and concise form
|
|
|
|
#Notes:
|
|
#Final resource //depot/gta5/assets/export/data/effects/ptfx.zip
|
|
#• This file is made up of lots of zip files X:\gta5\assets\export\data\effects
|
|
#• Each one of these files is created from an fxlists //depot/gta5/art/VFX/rmptfx/fxlists/
|
|
|
|
#• Models are found in //depot/gta5/art/VFX/rmptfx/models/
|
|
#• Textures are found in //depot/gta5/art/VFX/rmptfx/textures/
|
|
#• Textures have to be referenced in //depot/gta5/art/VFX/rmptfx/textures/regionTextures.txt which holds the name of the texture and the frame count in rows / columns eg:
|
|
# ptfx_bush_twigs 4 1
|
|
# ptfx_explosion_fireball_rgba 6 6
|
|
|
|
#• All data for rules is held in .xml file format and each name / rule has a limit of 42 characters
|
|
# including the .effectrule, .emitrule and .ptxrule
|
|
|
|
#-----------------------------------------------------------------------------
|
|
# Uses
|
|
#-----------------------------------------------------------------------------
|
|
require 'rexml/document'
|
|
include REXML
|
|
require 'pipeline/config/project'
|
|
require 'pipeline/config/projects'
|
|
require 'pipeline/os/getopt'
|
|
require 'pipeline/os/path'
|
|
require 'pipeline/scm/perforce'
|
|
require 'pipeline/projectutil/data_zip'
|
|
require 'pipeline/projectutil/data_texture'
|
|
include Pipeline
|
|
include Pipeline::ProjectUtil
|
|
|
|
require 'logger'
|
|
|
|
#-----------------------------------------------------------------------------
|
|
# Globals
|
|
#-----------------------------------------------------------------------------
|
|
g_DEBUG = false
|
|
g_Config = Pipeline::Config.instance( )
|
|
g_Project = g_Config.projects[ 'gta5' ] #need a way to determine project
|
|
g_FXRoot = OS::Path::combine(g_Project.root, 'art/VFX/rmptfx')
|
|
$g_FXLists_Path = OS::Path::combine(g_FXRoot, 'fxlists')
|
|
$g_EffectRules_Path = OS::Path::combine(g_FXRoot, 'effectrules')
|
|
$g_EmitRules_Path = OS::Path::combine(g_FXRoot, 'emitrules')
|
|
$g_PtxRules_Path = OS::Path::combine(g_FXRoot, 'ptxrules')
|
|
$g_Models_Path = OS::Path::combine(g_FXRoot, 'models')
|
|
$g_Textures_Path = OS::Path::combine(g_FXRoot, 'textures')
|
|
#logfile = open('c:/temp/VFXValidator.log', File::WRONLY | File::APPEND | File::CREAT)
|
|
$g_LogPath = OS::Path::combine(g_Project.root, 'tools/logs/VFXValidator.log')
|
|
|
|
OPTIONS = [
|
|
[ '--fxlist', '-sfx', OS::Getopt, 'FXList to be processed.' ]
|
|
]
|
|
|
|
#-----------------------------------------------------------------------------
|
|
# Classes
|
|
#-----------------------------------------------------------------------------
|
|
|
|
#_______________________________________________________________________________
|
|
#Generic base node class for the dependency tree
|
|
#_______________________________________________________________________________
|
|
class VFXNode
|
|
|
|
#////////////////////////////////////////////////////////////////
|
|
def initialize()
|
|
#puts 'init Node'
|
|
end
|
|
|
|
#////////////////////////////////////////////////////////////////
|
|
#Check if the paths exists in p4
|
|
#return boolean
|
|
#////////////////////////////////////////////////////////////////
|
|
def p4Exists()
|
|
depotPath = File.join('//depot', (@path.split(':')[1]))
|
|
DEBUGPUTS("DepotPath: '#{depotPath}'")
|
|
inP4 = $g_perforce.exists?(depotPath)
|
|
DEBUGPUTS("inP4: '#{inP4}'")
|
|
|
|
return inP4
|
|
end
|
|
|
|
#////////////////////////////////////////////////////////////////
|
|
#Return a list of Rule names from a given XML
|
|
#////////////////////////////////////////////////////////////////
|
|
def get_RuleNames_From_XML(xmlpath, theXPath)
|
|
#Check the path is valid
|
|
|
|
#Process the EffectRule XML
|
|
begin
|
|
xmlDoc = Document.new File.new(xmlpath)
|
|
ruleNameElements = XPath.each(xmlDoc, theXPath){ |element| element.text }
|
|
ruleNames = []
|
|
for e in ruleNameElements do
|
|
ruleNames << e.text
|
|
end
|
|
|
|
#return
|
|
ruleNames
|
|
|
|
rescue REXML::ParseException => msg
|
|
DEBUGPUTS('XML ERROR')
|
|
$g_Log.error("XML Read error with file '#{xmlpath}'")
|
|
$g_Log.error(msg)
|
|
ruleNames = []
|
|
end
|
|
end
|
|
|
|
end
|
|
|
|
#_______________________________________________________________________________
|
|
#EffectRules
|
|
#//depot/gta5/art/VFX/rmptfx/effectrules/
|
|
#Parent Type: FXList
|
|
#Dependencies: emitrules, ptxrules
|
|
#signatures: <emitterRuleName>_test_clip</emitterRuleName>
|
|
# <particleRuleName>_test_clip</particleRuleName>
|
|
#_______________________________________________________________________________
|
|
class EffectRule < VFXNode
|
|
|
|
attr_accessor :name,:path,:emitRules,:ptxRules,:dependencies
|
|
@ext = '.effectrule'
|
|
|
|
#////////////////////////////
|
|
def initialize(name)
|
|
super()
|
|
|
|
@name = name
|
|
@emitRules = []
|
|
@ptxRules = []
|
|
@dependencies = []
|
|
@path = ''
|
|
|
|
nameToPath()
|
|
getDependencies()
|
|
end
|
|
|
|
#////////////////////////////
|
|
def nameToPath()
|
|
nameExt = @name + '.effectrule'
|
|
@path = File.join($g_EffectRules_Path, nameExt)
|
|
#puts absPath
|
|
end
|
|
|
|
#////////////////////////////
|
|
def getDependencies()
|
|
#parse the file
|
|
#emitRules = []
|
|
#ptxRules = []
|
|
#
|
|
###1 emitRules
|
|
emitRuleNames = get_RuleNames_From_XML(@path, "//emitterRuleName")
|
|
for item in emitRuleNames do
|
|
#Generate nodes as needed
|
|
if(item != nil) then emitNode = EmitRule.new(item, self) end
|
|
@emitRules << emitNode
|
|
@dependencies << emitNode
|
|
end
|
|
|
|
###2 pfxRules
|
|
ptxRuleNames = get_RuleNames_From_XML(@path, "//particleRuleName")
|
|
for item in ptxRuleNames do
|
|
#Generate nodes as needed
|
|
if(item != nil) then ptxNode = PtxRule.new(item, self) end
|
|
@ptxRules << ptxNode
|
|
@dependencies << ptxNode
|
|
end
|
|
|
|
#now add the Rules to the dependencies list
|
|
#@dependencies << rules
|
|
|
|
|
|
DEBUGPUTS('--Dependencies---')
|
|
DEBUGPUTS(self.dependencies)
|
|
DEBUGPUTS('-----------------')
|
|
|
|
end
|
|
|
|
#for each effect rule create an EffectRule
|
|
# for effect in effectRules do
|
|
# #check file exists locally
|
|
# effectRulePath = OS::Path::combine($g_EffectRules_Path, effect)
|
|
# if File.exist?(effectRulePath) == nil then
|
|
# puts "Missing effect file #{effect}"
|
|
# end
|
|
#
|
|
# effectName = OS::Path::get_basename(effectRulePath)
|
|
# puts effectName
|
|
# effectRule = EffectRule.new(effectName)
|
|
# #effectRule.nameToPath()
|
|
# puts effectRule.path
|
|
# puts (effectRule.p4Exists())
|
|
#
|
|
#
|
|
# end
|
|
|
|
end
|
|
|
|
#_______________________________________________________________________________
|
|
#EmitRules
|
|
#//depot/gta5/art/VFX/rmptfx/emitrules/
|
|
#Parent Type: EffectRule
|
|
#Dependencies:
|
|
#signatures:
|
|
#
|
|
#_______________________________________________________________________________
|
|
class EmitRule < VFXNode
|
|
attr_accessor :name,:parent,:path,:dependencies
|
|
def initialize(name, parent)
|
|
super()
|
|
|
|
@name = name
|
|
@parent = parent
|
|
@path = ''
|
|
@dependencies = []
|
|
|
|
nameToPath()
|
|
end
|
|
|
|
#////////////////////////////
|
|
def nameToPath()
|
|
nameExt = @name + '.emitrule'
|
|
@path = File.join($g_EmitRules_Path, nameExt)
|
|
#puts absPath
|
|
end
|
|
|
|
#////////////////////////////////////////////////////
|
|
#EmitRules have no child dependencies so just
|
|
#return an empty list
|
|
def getDependencies()
|
|
@dependencies = []
|
|
end
|
|
|
|
end
|
|
|
|
#_______________________________________________________________________________
|
|
#Particle Rules
|
|
#//depot/gta5/art/VFX/rmptfx/ptxrules/
|
|
#Parent Type: EffectRule
|
|
#Dependencies: Textures, Models
|
|
#Signatures: <textureName>_test_flat</textureName>
|
|
# <drawables><Item><name>ptfx_model_bin_coffee/ptfx_model_bin_coffee</name></Item></drawables>
|
|
#_______________________________________________________________________________
|
|
class PtxRule < VFXNode
|
|
|
|
attr_accessor :name,:parent,:path,:dependencies,:ptxTextures,:ptxModels
|
|
|
|
def initialize(name, parent)
|
|
super()
|
|
|
|
@name = name
|
|
@parent = parent
|
|
@path = ''
|
|
@dependencies = []
|
|
@ptxTextures = []
|
|
@ptxModels = []
|
|
|
|
nameToPath()
|
|
getDependencies()
|
|
end
|
|
|
|
#////////////////////////////
|
|
def nameToPath()
|
|
nameExt = @name + '.ptxrule'
|
|
@path = File.join($g_PtxRules_Path, nameExt)
|
|
#puts absPath
|
|
end
|
|
|
|
#////////////////////////////
|
|
def getDependencies()
|
|
###1 Textures
|
|
textureNames = get_RuleNames_From_XML(@path, "//textureName")
|
|
for item in textureNames do
|
|
if(item != nil) then
|
|
node = TextureNode.new(item, self)
|
|
@ptxTextures << node
|
|
@dependencies << node
|
|
end
|
|
end
|
|
|
|
|
|
###2 Models
|
|
modelNames = get_RuleNames_From_XML(@path, "//drawables/Item/name")
|
|
for item in modelNames do
|
|
if(item != nil) then
|
|
node = ModelNode.new(item, self)
|
|
@ptxModels << node
|
|
@dependencies << node
|
|
end
|
|
end
|
|
|
|
#@dependencies << @ptxTextures
|
|
#@dependencies << @ptxModels
|
|
|
|
DEBUGPUTS('--Ptx Dependencies---')
|
|
DEBUGPUTS(self.dependencies)
|
|
DEBUGPUTS('-----------------')
|
|
end
|
|
end
|
|
|
|
#_______________________________________________________________________________
|
|
#TextureNode
|
|
#Relates to a PtxRule node
|
|
#_______________________________________________________________________________
|
|
class TextureNode < VFXNode
|
|
attr_accessor :name,:parent,:path,:dependencies
|
|
|
|
def initialize(name, parent)
|
|
super()
|
|
|
|
@name = name
|
|
@parent = parent
|
|
@path
|
|
@dependencies = []
|
|
|
|
nameToPath()
|
|
end
|
|
|
|
#////////////////////////////
|
|
def nameToPath()
|
|
nameExt = @name + '.dds'
|
|
@path = File.join($g_Textures_Path, nameExt)
|
|
#puts absPath
|
|
end
|
|
end
|
|
|
|
#_______________________________________________________________________________
|
|
#ModelNode
|
|
#Relates to a PtxRule node
|
|
#_______________________________________________________________________________
|
|
class ModelNode < VFXNode
|
|
attr_accessor :name,:parent,:path,:dependencies
|
|
|
|
def initialize(name, parent)
|
|
super()
|
|
|
|
@name = name
|
|
@parent = parent
|
|
@path
|
|
@dependencies = []
|
|
|
|
nameToPath()
|
|
end
|
|
|
|
#////////////////////////////
|
|
def nameToPath()
|
|
nameExt = @name + '.mesh'
|
|
@path = File.join($g_Models_Path, nameExt)
|
|
#puts @path
|
|
end
|
|
end
|
|
|
|
#_______________________________________________________________________________
|
|
#EffectTree
|
|
#
|
|
#_______________________________________________________________________________
|
|
class EffectTree
|
|
|
|
attr_accessor :child
|
|
|
|
def initialize()
|
|
@child
|
|
end
|
|
end
|
|
|
|
|
|
#_______________________________________________________________________________
|
|
#FXLists
|
|
#//depot/gta5/art/VFX/rmptfx/fxlists/
|
|
#Parent Type: Effect Zip
|
|
#Dependencies: EffectRule
|
|
#Signatures: non XML, simple string, comment char == #
|
|
#_______________________________________________________________________________
|
|
class FXList < VFXNode
|
|
|
|
attr_accessor :dependencies,:parent,:path
|
|
@regexQuery = /^\w+/ #collect lines beginning with an alphanumeric character
|
|
|
|
def initialize(path, parent)
|
|
@path = path
|
|
@parent = parent
|
|
@dependencies = []
|
|
getDependencies()
|
|
end
|
|
|
|
#////////////////////////////
|
|
def getDependencies()
|
|
theFXListPathFile = File.open(@path, mode_string="r")
|
|
theFXList = File.readlines(@path).collect{ |line| (/^\w+/.match(line)).to_s() }
|
|
theFXList = theFXList.compact()
|
|
theFXListPathFile.close()
|
|
|
|
#collect up the effectrules that match
|
|
effectRules = []
|
|
effectRulesDirItems = read_list_from_dir( $g_EffectRules_Path )
|
|
effectRulesDirItems.each do |name|
|
|
nameNoExt = File.basename(name, '.effectrule')
|
|
|
|
processList = []
|
|
for item in theFXList do
|
|
if(nameNoExt == item) then processList.push(item) end
|
|
end
|
|
|
|
puts processList
|
|
|
|
processList.each_index do |x|
|
|
#puts "Processing #{x} of #{processList.length}"
|
|
@dependencies << EffectRule.new(processList[x])
|
|
end
|
|
# for item in theFXList do
|
|
# if(nameNoExt == item) then
|
|
# puts "Processing effectrule #{nameNoExt} in fxlist: #{self.path}"
|
|
# @dependencies << EffectRule.new(nameNoExt)
|
|
# end
|
|
# end
|
|
end
|
|
end
|
|
|
|
end
|
|
|
|
#-----------------------------------------------------------------------------
|
|
# Functions
|
|
#-----------------------------------------------------------------------------
|
|
|
|
#/////////////////////////////////////
|
|
#
|
|
#/////////////////////////////////////
|
|
def DEBUGPUTS(msg)
|
|
puts(msg) if ($g_DEBUG == true)
|
|
end
|
|
|
|
#/////////////////////////////////////
|
|
#
|
|
#/////////////////////////////////////
|
|
def read_list_from_file( filename )
|
|
list = []
|
|
File::open( filename, 'r' ) do |fp|
|
|
fp.each do |line|
|
|
list << line.chomp.strip unless ( line.starts_with( '#' ) )
|
|
end
|
|
end
|
|
list
|
|
end
|
|
|
|
#/////////////////////////////////////
|
|
#
|
|
#/////////////////////////////////////
|
|
def read_list_from_dir( path )
|
|
list = []
|
|
Dir::open( path ) do |dir|
|
|
dir.each do |filename|
|
|
next if ( '.' == filename )
|
|
next if ( '..' == filename )
|
|
list << filename
|
|
end
|
|
end
|
|
list
|
|
end
|
|
|
|
#/////////////////////////////////////
|
|
#
|
|
#/////////////////////////////////////
|
|
def buildTree(fxListPath, effectTreeList)
|
|
effectTreeNode = EffectTree.new()
|
|
fxListNode = FXList.new(fxListPath, effectTreeNode)
|
|
effectTreeNode.child = fxListNode
|
|
effectTreeList << effectTreeNode
|
|
end
|
|
|
|
#/////////////////////////////////////
|
|
#
|
|
#/////////////////////////////////////
|
|
def processDependencies(node, nodeList)
|
|
#puts node.dependencies
|
|
node.dependencies.each do |child|
|
|
#puts child.class
|
|
if child.p4Exists() == false then
|
|
nodeList << [child, false]
|
|
end
|
|
|
|
if child.dependencies.length != 0 then
|
|
processDependencies(child, nodeList)
|
|
end
|
|
end
|
|
end
|
|
|
|
#----------------------------------------------------------------------------
|
|
# Entry-Point
|
|
#----------------------------------------------------------------------------
|
|
if ( __FILE__ == $0 ) then
|
|
|
|
g_Opts = OS::Getopt::getopts( OPTIONS )
|
|
# Initialise log and console output
|
|
if File.exist?($g_LogPath) == true then
|
|
File.delete($g_LogPath)
|
|
end
|
|
$g_Log = Logger.new($g_LogPath)
|
|
$g_Log.info('created')
|
|
$g_Log.info('Starting Tree Building....')
|
|
|
|
puts 'Starting Tree Building....'
|
|
#puts Config::instance().project.scm
|
|
$g_perforce = Pipeline::SCM::Perforce.new()
|
|
$g_perforce.connect()
|
|
|
|
#Create an array to hold all the EffectTree chains created
|
|
effectTreeList = []
|
|
|
|
#Test FXList files based on input args
|
|
#g_Opts.get( 'FXList' )
|
|
puts "Args: #{g_Opts[1]}"
|
|
if g_Opts[1].to_s() == "all" then
|
|
fxListFiles = read_list_from_dir('x:/gta5/art/VFX/rmptfx/fxlists')
|
|
#puts fxListFiles
|
|
else
|
|
#check its a valid path first
|
|
if File.exists?(File.join($g_FXLists_Path, g_Opts[1])) then
|
|
fxListFiles = [g_Opts[1]]
|
|
else
|
|
$g_Log.error('Not a valid FXList file supplied')
|
|
exit()
|
|
end
|
|
end
|
|
|
|
processCount = fxListFiles.length
|
|
counter = 1
|
|
#puts fxListFiles
|
|
fxListFiles.each do |fxListName|
|
|
$g_Log.info("Working on '#{fxListName}'")
|
|
puts("Processing fxlists: '#{counter}' out of: '#{processCount}'")
|
|
DEBUGPUTS(fxListName)
|
|
#puts $g_FXLists_Path.class
|
|
fxListPath = File.join($g_FXLists_Path, fxListName)
|
|
#theFXListPath = 'x:/gta5/art/VFX/rmptfx/fxlists/cut_agencyheist.fxlist'
|
|
DEBUGPUTS('///////////////////////////////')
|
|
DEBUGPUTS("FXListPath: #{fxListPath}")
|
|
DEBUGPUTS('///////////////////////////////')
|
|
|
|
#Open the FXList file
|
|
if File.stat(fxListPath).size != nil
|
|
#puts 'got a file'
|
|
effectTreeNode = EffectTree.new()
|
|
fxListNode = FXList.new(fxListPath, effectTreeNode)
|
|
effectTreeNode.child = fxListNode
|
|
effectTreeList << effectTreeNode
|
|
end
|
|
counter += 1
|
|
end
|
|
|
|
puts effectTreeList
|
|
|
|
puts("Starting Dependency Validation")
|
|
nodeList = []
|
|
|
|
processCount = effectTreeList.length
|
|
counter = 1
|
|
effectTreeList.each do |tree|
|
|
puts("Processing: '#{counter}' out of: '#{processCount}'")
|
|
fxlist = tree.child
|
|
fxlist.dependencies.each do |effectRule|
|
|
if effectRule.p4Exists() == false then
|
|
nodeList << [effectRule, false]
|
|
end
|
|
processDependencies(effectRule, nodeList)
|
|
end
|
|
counter += 1
|
|
end
|
|
|
|
puts nodeList.length
|
|
|
|
#Report the results if theres anything amiss
|
|
if nodeList.length > 0 then
|
|
nodeList.each do |node|
|
|
if node[1] == false then
|
|
theNode = node[0]
|
|
$g_Log.error("Type:'#{theNode.class}' Name:'#{theNode.name}' Path:'#{theNode.path}' Not found in perforce")
|
|
end
|
|
puts("Type: '#{node[0].class}' Name: '#{node[0].name}' p4exists: '#{node[1]}'")
|
|
end
|
|
else
|
|
$g_Log.info("You are a winner!! All assets for this fxlist exist in perforce")
|
|
end
|
|
|
|
#close the log writer
|
|
$g_Log.close()
|
|
|
|
#open the log file
|
|
system("start #{$g_LogPath}")
|
|
|
|
puts 'END'
|
|
|
|
#read the effectrule defs
|
|
#is file in perforce?
|
|
|
|
#create an EffectTree instance
|
|
#effectTree = EffectTree()
|
|
|
|
|
|
|
|
#Create the EmitRule
|
|
#Verify the path exists
|
|
#Verify it is checked into perforce
|
|
#Set the parent to the EffectRule
|
|
|
|
#Create the PtxRule
|
|
#Verify the path exists
|
|
#Verify it is checked into perforce
|
|
#Set the parent to the EffectRule
|
|
#Look for texture or model dependencies
|
|
#verify the paths and check they are in perforce
|
|
end |