447 lines
17 KiB
Ruby
Executable File
447 lines
17 KiB
Ruby
Executable File
#
|
|
# File:: data_compare.rb
|
|
# Description:: Data comparison functions.
|
|
#
|
|
# Author:: David Muir <david.muir@rockstarnorth.com>
|
|
# Date:: 4 November 2008
|
|
#
|
|
|
|
#-----------------------------------------------------------------------------
|
|
# Uses
|
|
#-----------------------------------------------------------------------------
|
|
require 'pipeline/config/projects'
|
|
require 'pipeline/config/project'
|
|
require 'pipeline/fileformats/dds_raw'
|
|
require 'pipeline/os/file'
|
|
require 'pipeline/os/start'
|
|
require 'pipeline/projectutil/data_compare_idr'
|
|
require 'pipeline/projectutil/data_extract'
|
|
require 'pipeline/util/rage'
|
|
require 'pipeline/util/rage_image'
|
|
require 'pipeline/util/rage_pack'
|
|
|
|
#-----------------------------------------------------------------------------
|
|
# Globals HACKS************
|
|
#-----------------------------------------------------------------------------
|
|
$image = ''
|
|
$rpf = ''
|
|
|
|
#-----------------------------------------------------------------------------
|
|
# Implementation
|
|
#-----------------------------------------------------------------------------
|
|
module Pipeline
|
|
module ProjectUtil
|
|
|
|
#-------------------------------------------------------------------------
|
|
# Constants
|
|
#-------------------------------------------------------------------------
|
|
GNU_DIFF = '$(toolsbin)/bin/diff.exe'
|
|
MESH_DIFF = '$(toolsbin)/bin/meshdiff.exe'
|
|
|
|
DATA_COMPARE_EQUAL = 0
|
|
DATA_COMPARE_EQUIVALENT = 0
|
|
DATA_COMPARE_DIFFERENT = 1
|
|
|
|
DATA_COMPARE_MODE_TEXT = 0
|
|
DATA_COMPARE_MODE_BIN = 1
|
|
DATA_COMPARE_MODE_AUTO = 2
|
|
|
|
DATA_COMPARE_RPF_EXTENSIONS = [ 'rpf', 'ibd', 'ibn', 'idd', 'idr', 'itd' ]
|
|
|
|
#-------------------------------------------------------------------------
|
|
# Functions
|
|
#-------------------------------------------------------------------------
|
|
|
|
#
|
|
# == Description
|
|
# Return the full command line for a GNU diff command to compare file1
|
|
# against file2 using the specified comparison mode.
|
|
#
|
|
def ProjectUtil::data_compare_diff_command_line( file1, file2, mode = DATA_COMPARE_MODE_AUTO, app = GNU_DIFF, opts = '' )
|
|
diff_cmd = ''
|
|
case mode
|
|
when DATA_COMPARE_MODE_BIN
|
|
diff_cmd = Pipeline::Config::instance().envsubst( "#{app} --binary #{opts} #{file1} #{file2}" )
|
|
when DATA_COMPARE_MODE_TEXT
|
|
diff_cmd = Pipeline::Config::instance().envsubst( "#{app} --text #{opts} #{file1} #{file2}" )
|
|
else
|
|
# Default to auto
|
|
diff_cmd = Pipeline::Config::instance().envsubst( "#{app} #{opts} #{file1} #{file2}" )
|
|
end
|
|
|
|
diff_cmd
|
|
end #data_compare_diff_command_line
|
|
|
|
|
|
#
|
|
# == Description
|
|
# This function compares two files, returning a GNU diff output integer.
|
|
# -1: problems executing diff tool
|
|
# 0: files are identical
|
|
# 1: files are different
|
|
# 2: there was trouble comparing the files.
|
|
#
|
|
# The optional code block will execute if the files are different.
|
|
#
|
|
# === Example Usage
|
|
# DHM TODO
|
|
#
|
|
def ProjectUtil::data_compare_raw( file1, file2, mode = DATA_COMPARE_MODE_AUTO, app = GNU_DIFF, &block )
|
|
throw ArgumentError.new( "Invalid data comparison mode (#{mode})." ) \
|
|
unless ( DATA_COMPARE_MODE_TEXT <= mode and mode <= DATA_COMPARE_MODE_AUTO )
|
|
|
|
cmd = ProjectUtil::data_compare_diff_command_line( file1, file2, mode, app )
|
|
status, stdout, stderr = OS::start( cmd )
|
|
return ( -1 ) if ( not status.exited? )
|
|
|
|
case status.exitstatus
|
|
when 0:
|
|
when 1, 2:
|
|
if ( status != 0 ) then
|
|
#puts "Press any key to continue..."
|
|
#$stdin.gets()
|
|
ext = OS::Path::get_extension( file1 )
|
|
testdir = OS::Path::combine( "x:/compare_output/textures", $image, $rpf )
|
|
FileUtils::mkdir_p( testdir )
|
|
file1_dst = OS::Path::combine( testdir, "#{OS::Path::get_basename(file1)}__E1.#{ext}" )
|
|
file2_dst = OS::Path::combine( testdir, "#{OS::Path::get_basename(file1)}__GTA4.#{ext}" )
|
|
# Temp copy different textures for analysis.
|
|
FileUtils::cp( file1, file1_dst, :preserve => true ) unless ( ::File::exists?( file1_dst ) )
|
|
FileUtils::cp( file2, file2_dst, :preserve => true ) unless ( ::File::exists?( file2_dst ) )
|
|
end
|
|
|
|
yield( file1 ) if ( block_given? )
|
|
end
|
|
return ( status )
|
|
end #data_compare_raw
|
|
|
|
|
|
#
|
|
# == Description
|
|
# This function loads two DDS textures (using Pipeline::FileFormats::DDSFileRaw.
|
|
# We will yield if there is a change in the top-level texture mip.
|
|
#
|
|
# === Example Usage
|
|
# DHM TODO.
|
|
#
|
|
def ProjectUtil::data_compare_dds( file1, file2, &block )
|
|
testdir = OS::Path::combine( "x:/compare_output/textures", $image, $rpf )
|
|
|
|
dds1 = FileFormats::DDSFileRaw::from_filename( file1 )
|
|
dds2 = FileFormats::DDSFileRaw::from_filename( file2 )
|
|
puts "DDS COMPARE:"
|
|
puts "\t#{file1}"
|
|
puts "\t#{file2}"
|
|
if ( dds1.imagedata != dds2.imagedata ) then
|
|
puts "\tDIFFER"
|
|
|
|
FileUtils::mkdir_p( testdir )
|
|
file1_dst = OS::Path::combine( testdir, "#{OS::Path::get_basename(file1)}__E1.dds" )
|
|
file2_dst = OS::Path::combine( testdir, "#{OS::Path::get_basename(file1)}__GTA4.dds" )
|
|
# Temp copy different textures for analysis.
|
|
FileUtils::cp( file1, file1_dst, :preserve => true ) unless ( ::File::exists?( file1_dst ) )
|
|
FileUtils::cp( file2, file2_dst, :preserve => true ) unless ( ::File::exists?( file2_dst ) )
|
|
#puts "Press any key to continue..."
|
|
#$stdin.gets()
|
|
else
|
|
puts "\tSAME"
|
|
end
|
|
|
|
differ = ( dds1.imagedata != dds2.imagedata )
|
|
yield( file1 ) if ( differ )
|
|
|
|
return 1 if ( differ )
|
|
return 0 unless ( differ )
|
|
end #data_compare_dds
|
|
|
|
|
|
#
|
|
# == Description
|
|
# This function does a full context-aware file comparison, drilling down into RPF and
|
|
# image files, producing diffed equivalents invoking a user code block as required.
|
|
#
|
|
# === Example Usage
|
|
# DHM TODO
|
|
#
|
|
def ProjectUtil::data_compare_file( proj, branch, target, r, file1, file2, output_dir, rpf_extensions = DATA_COMPARE_RPF_EXTENSIONS, &block )
|
|
puts "Comparing #{file1} and #{file2}"
|
|
ext1 = OS::Path::get_extension( file1 )
|
|
ext2 = OS::Path::get_extension( file2 )
|
|
throw ArgumentError.new( "Files are not of the same type." ) \
|
|
unless ( 0 == ext1.casecmp( ext2 ) )
|
|
|
|
image_builder = Util::RagebuilderImage.new( proj, branch, branch.targets[target] )
|
|
rpf_builder = Util::RagebuilderPack.new( proj, branch, branch.targets[target] )
|
|
|
|
if ( rpf_extensions.include?( ext1 ) ) then
|
|
|
|
if ( 0 == 'idr'.casecmp( ext1 ) ) then
|
|
$rpf = OS::Path::get_filename( file1 )
|
|
data_compare_idr( proj, branch, target, r, file1, file2, output_dir, &block )
|
|
$rpf = ''
|
|
else
|
|
$rpf = OS::Path::get_filename( file1 )
|
|
data_compare_rpf( proj, branch, target, r, rpf_builder, file1, file2, output_dir, &block )
|
|
$rpf = ''
|
|
end
|
|
|
|
elsif ( 0 == 'type'.casecmp( ext1 ) ) then
|
|
|
|
status = data_compare_idr_type( file1, file2, &block )
|
|
puts "TYPE COMPARE: #{status}"
|
|
puts "\t#{file1}"
|
|
puts "\t#{file2}"
|
|
#puts "Press Enter to continue..."
|
|
#$stdin.gets()
|
|
|
|
if ( status != 0 ) then
|
|
#puts "Press any key to continue..."
|
|
#$stdin.gets()
|
|
testdir = OS::Path::combine( "x:/compare_output/textures", $image, $rpf )
|
|
FileUtils::mkdir_p( testdir )
|
|
file1_dst = OS::Path::combine( testdir, "#{OS::Path::get_basename(file1)}__E1.type" )
|
|
file2_dst = OS::Path::combine( testdir, "#{OS::Path::get_basename(file1)}__GTA4.type" )
|
|
# Temp copy different textures for analysis.
|
|
FileUtils::cp( file1, file1_dst, :preserve => true ) unless ( ::File::exists?( file1_dst ) )
|
|
FileUtils::cp( file2, file2_dst, :preserve => true ) unless ( ::File::exists?( file2_dst ) )
|
|
end
|
|
|
|
elsif ( 0 == 'mesh'.casecmp( ext1 ) ) then
|
|
|
|
status = data_compare_raw( file1, file2, DATA_COMPARE_MODE_AUTO, MESH_DIFF, &block )
|
|
puts "MESH COMPARE: #{status}"
|
|
puts "\t#{file1}"
|
|
puts "\t#{file2}"
|
|
#puts "Press Enter to continue..."
|
|
#$stdin.gets()
|
|
|
|
if ( status != 0 ) then
|
|
#puts "Press any key to continue..."
|
|
#$stdin.gets()
|
|
testdir = OS::Path::combine( "x:/compare_output/textures", $image, $rpf )
|
|
FileUtils::mkdir_p( testdir )
|
|
file1_dst = OS::Path::combine( testdir, "#{OS::Path::get_basename(file1)}__E1.mesh" )
|
|
file2_dst = OS::Path::combine( testdir, "#{OS::Path::get_basename(file1)}__GTA4.mesh" )
|
|
# Temp copy different textures for analysis.
|
|
FileUtils::cp( file1, file1_dst, :preserve => true ) unless ( ::File::exists?( file1_dst ) )
|
|
FileUtils::cp( file2, file2_dst, :preserve => true ) unless ( ::File::exists?( file2_dst ) )
|
|
end
|
|
|
|
elsif ( 0 == 'img'.casecmp( ext1 ) ) then
|
|
|
|
$image = OS::Path::get_basename( file1 )
|
|
data_compare_image( proj, branch, target, r, image_builder, file1, file2, output_dir, &block )
|
|
|
|
elsif ( 0 == 'dds'.casecmp( ext1 ) ) then
|
|
|
|
data_compare_dds( file1, file2, &block )
|
|
|
|
else
|
|
# Raw files that we cannot process anymore.
|
|
data_compare_raw( file1, file2, &block )
|
|
end
|
|
end #data_compare_file
|
|
|
|
|
|
#
|
|
# == Description
|
|
# This function compares two independent drawable (IDR) files. The IDR
|
|
# file is a special case because if any contained file is different we
|
|
# package up the entire contents and write it to our output directory.
|
|
#
|
|
# This is to ensure re-resourcing works correctly.
|
|
#
|
|
# == Example Usage
|
|
# DHM TODO
|
|
#
|
|
def ProjectUtil::data_compare_idr( proj, branch, target, r, pack1, pack2, output_dir, &block )
|
|
|
|
pack1_dest = OS::Path::combine( output_dir, 'pack1' )
|
|
pack2_dest = OS::Path::combine( output_dir, 'pack2' )
|
|
pack1_fullpath = []
|
|
pack2_fullpath = []
|
|
pack1_files = []
|
|
pack2_files = []
|
|
ProjectUtil::data_extract_rpf( r, pack1, pack1_dest ) do |extracted_filename|
|
|
pack1_fullpath << extracted_filename
|
|
pack1_files << OS::Path::get_filename( extracted_filename )
|
|
end
|
|
ProjectUtil::data_extract_rpf( r, pack2, pack2_dest ) do |extracted_filename|
|
|
pack2_fullpath << extracted_filename
|
|
pack2_files << OS::Path::get_filename( extracted_filename )
|
|
end
|
|
|
|
# Iterate over the file lists comparing filenames that match. As
|
|
# soon as we detect any difference we break and yield the pack1
|
|
# filename. If all files are identical then we do not yield.
|
|
differ = false
|
|
pack1_fullpath.each_with_index do |pack1_filename, index|
|
|
pack1_file = OS::Path::get_filename( pack1_filename )
|
|
unless ( pack2_files.include?( OS::Path::get_filename( pack1_file ) ) )
|
|
puts "IDR DIFFER: #{pack1_filename} is new."
|
|
differ = true
|
|
end
|
|
|
|
## break if ( differ )
|
|
if ( pack2_files.index( pack1_file ) ) then
|
|
pack2_filename = pack2_fullpath[pack2_files.index(pack1_file)]
|
|
ProjectUtil::data_compare_file( proj, branch, target, r, pack1_filename, pack2_filename, output_dir ) do
|
|
puts "IDR DIFFER: #{pack1_filename} differs from #{pack2_filename}"
|
|
differ = true
|
|
end
|
|
else
|
|
differ = true
|
|
end
|
|
|
|
## break if ( differ )
|
|
end #pack1_fullpath.each_with_index
|
|
|
|
# LPXO: We need to pick up files that have been removed as well
|
|
if differ == false then
|
|
pack2_fullpath.each_with_index do |pack2_filename, index|
|
|
pack2_file = OS::Path::get_filename( pack2_filename )
|
|
unless ( pack1_files.include?( OS::Path::get_filename( pack2_file ) ) )
|
|
puts "IDR DIFFER: #{pack2_filename} has been removed."
|
|
differ = true
|
|
end
|
|
end #pack2_fullpath.each_with_index
|
|
end
|
|
|
|
yield( pack1 ) if ( block_given? and differ )
|
|
|
|
# Clean up extracted files
|
|
::FileUtils::rm_r( pack1_dest ) if ( ::File::directory?( pack1_dest ) )
|
|
::FileUtils::rm_r( pack2_dest ) if ( ::File::directory?( pack2_dest ) )
|
|
end #data_compare_idr
|
|
|
|
|
|
#
|
|
# == Description
|
|
# This function compares two pack files.
|
|
#
|
|
# The code block will be invoked if the files are different or if it
|
|
# exists in pack1 and not pack2. Pack1 is considered the control pack
|
|
# which is used to drive the comparison.
|
|
#
|
|
# === Example Usage
|
|
# DHM TODO
|
|
#
|
|
def ProjectUtil::data_compare_rpf( proj, branch, target, r, builder, pack1, pack2, output_dir, &block )
|
|
|
|
pack1_dest = OS::Path::combine( output_dir, 'pack1' )
|
|
pack2_dest = OS::Path::combine( output_dir, 'pack2' )
|
|
pack1_fullpath = []
|
|
pack2_fullpath = []
|
|
pack2_files = []
|
|
ProjectUtil::data_extract_rpf( r, pack1, pack1_dest ) do |extracted_filename|
|
|
pack1_fullpath << extracted_filename
|
|
end
|
|
ProjectUtil::data_extract_rpf( r, pack2, pack2_dest ) do |extracted_filename|
|
|
pack2_fullpath << extracted_filename
|
|
pack2_files << extracted_filename.sub( pack2_dest, '' )
|
|
end
|
|
|
|
# Iterate over the file lists comparing filenames that match. As
|
|
# soon as we detect any difference we break and yield the pack1
|
|
# filename. If all files are identical then we do not yield.
|
|
differ = false
|
|
pack1_fullpath.each_with_index do |pack1_filename, index|
|
|
pack1_file = pack1_filename.sub( pack1_dest, '' )
|
|
unless ( pack2_files.include?( pack1_file ) )
|
|
puts "RPF DIFFER: #{pack1_filename} is new."
|
|
differ = true
|
|
end
|
|
|
|
if ( pack2_files.index( pack1_file ) ) then
|
|
pack2_filename = pack2_fullpath[pack2_files.index(pack1_file)]
|
|
ProjectUtil::data_compare_file( proj, branch, target, r, pack1_filename, pack2_filename, output_dir ) do
|
|
puts "RPF DIFFER: #{pack1_filename} differs from #{pack2_filename}"
|
|
differ = true
|
|
end
|
|
else
|
|
differ = true
|
|
end
|
|
end #pack1_fullpath.each_with_index
|
|
|
|
yield( pack1 ) if ( block_given? and differ )
|
|
if ( differ )
|
|
puts "RPF DIFFER (#{OS::Path::get_filename(pack1)})"
|
|
puts "\t#{pack1}"
|
|
puts "\t#{pack2}"
|
|
##$stdin.gets()
|
|
else
|
|
puts "RPF NO DIFFER (#{OS::Path::get_filename(pack1)})"
|
|
puts "\t#{pack1}"
|
|
puts "\t#{pack2}"
|
|
##$stdin.gets()
|
|
end
|
|
# Clean up extracted files
|
|
::FileUtils::rm_r( pack1_dest ) if ( ::File::directory?( pack1_dest ) )
|
|
::FileUtils::rm_r( pack2_dest ) if ( ::File::directory?( pack2_dest ) )
|
|
end #data_compare_rpf
|
|
|
|
|
|
#
|
|
# == Description
|
|
# This function compares two image files.
|
|
#
|
|
# The code block will execute if the files are different or if it exists
|
|
# in image1 and not image2. Image1 is considered the control image which
|
|
# is used to drive the comparison.
|
|
#
|
|
# === Example Usage
|
|
# DHM TODO
|
|
#
|
|
def ProjectUtil::data_compare_image( proj, branch, target, r, builder, image1, image2, output_dir, mode = DATA_COMPARE_MODE_AUTO, &block )
|
|
throw ArgumentError.new( "Invalid data comparison mode (#{mode})." ) \
|
|
unless ( DATA_COMPARE_MODE_TEXT <= mode and mode <= DATA_COMPARE_MODE_AUTO )
|
|
|
|
# Extract the contents of the two images ready for comparison.
|
|
image1_dest = OS::Path::combine( output_dir, 'image1' )
|
|
image2_dest = OS::Path::combine( output_dir, 'image2' )
|
|
image1_fullpath = []
|
|
image2_fullpath = []
|
|
image2_files = []
|
|
ProjectUtil::data_extract_image( r, image1, image1_dest ) do |extracted_filename|
|
|
image1_fullpath << extracted_filename
|
|
end
|
|
ProjectUtil::data_extract_image( r, image2, image2_dest ) do |extracted_filename|
|
|
image2_fullpath << extracted_filename
|
|
image2_files << OS::Path::get_filename( extracted_filename )
|
|
end
|
|
|
|
image_filename = OS::Path::combine( output_dir, OS::Path::get_filename( image1 ) )
|
|
builder.start( )
|
|
|
|
# Iterate over the file lists comparing filenames that match. Images
|
|
# have a flat directory structure so we don't need to worry about
|
|
# sub-directories.
|
|
image1_fullpath.each_with_index do |image1_filename, index|
|
|
image1_file = OS::Path::get_filename( image1_filename )
|
|
unless ( image2_files.include?( OS::Path::get_filename( image1_file ) ) )
|
|
# If the image1 file is not present in image2, then add to the new image
|
|
builder.add( image1_filename )
|
|
next
|
|
end
|
|
|
|
# Otherwise compare them
|
|
image2_filename = image2_fullpath[image2_files.index(image1_file)]
|
|
ProjectUtil::data_compare_file( proj, branch, target, r, image1_filename, image2_filename, output_dir ) do |file1, file2|
|
|
builder.add( file1 )
|
|
end
|
|
end
|
|
|
|
builder.save( image_filename )
|
|
builder.close( )
|
|
yield( image_filename ) if ( block_given? )
|
|
|
|
# Clean up extracted files
|
|
::FileUtils::rm_r( image1_dest ) if ( ::File::directory?( image1_dest ) )
|
|
::FileUtils::rm_r( image2_dest ) if ( ::File::directory?( image2_dest ) )
|
|
end #data_compare_image
|
|
|
|
end # ProjectUtil module
|
|
end # Pipeline module
|
|
|
|
# data_compare.rb
|