462 lines
15 KiB
Python
Executable File
462 lines
15 KiB
Python
Executable File
import datetime
|
|
import hashlib
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
from audio_logger import Logger, LogSeverity
|
|
|
|
|
|
# Runtime data
|
|
NOW_START = datetime.datetime.now()
|
|
NOW_START_STR = NOW_START.strftime('%y_%m_%d__%H_%M_%S')
|
|
|
|
RPF_ROOT_PATH = r'\\rockstar.t2.corp\Network\RSGEDI\Audio\Audio\RPF'
|
|
#RPF_ROOT_PATH = r'd:\rpfs'
|
|
|
|
LOG_DIR = os.path.join(r'C:\Temp\Logs\Audio RPFs', NOW_START_STR)
|
|
LOG_FILENAME = r'run_log_%s.txt'
|
|
|
|
# Perforce
|
|
PERFORCE_EXE_NAME = r'p4'
|
|
|
|
PLATFORMS = (
|
|
'PS4',
|
|
'XBOXONE',
|
|
'PC',
|
|
)
|
|
|
|
DEV_IGNORE_PACKS_LIST = (
|
|
#'^PS_(.*)',
|
|
'(.*)DLC(.*)',
|
|
)
|
|
|
|
RELEASE_IGNORE_PACKS_LIST = {
|
|
'PS4' : (
|
|
'(.*)DLC(.*)',
|
|
'TEST(.*)',
|
|
'^CUTSCENE$',
|
|
'^CUTSCENE_MASTERED_TRIMMED$',
|
|
),
|
|
'XBOXONE' : (
|
|
'^PS_(.*)',
|
|
'(.*)DLC(.*)',
|
|
'TEST(.*)',
|
|
'^CUTSCENE$',
|
|
'^CUTSCENE_MASTERED_TRIMMED$',
|
|
),
|
|
'PC' : (
|
|
'(.*)DLC(.*)',
|
|
'TEST(.*)',
|
|
'^CUTSCENE$',
|
|
'^CUTSCENE_MASTERED_TRIMMED$',
|
|
),
|
|
}
|
|
|
|
IGNORE_EXCEPTION_PACKS_LIST = (
|
|
'DLC_GTAO$',
|
|
'DLCTU$',
|
|
)
|
|
|
|
|
|
# Runtime args
|
|
audio_path = None
|
|
rage_builder_path = None
|
|
package_path_release = None
|
|
package_path_main = None
|
|
tools_root_path = None
|
|
|
|
packlist_path_main = None
|
|
packlist_path_release = None
|
|
input_path = None
|
|
audio_output_path = None
|
|
sfx_output_path = None
|
|
build_pack_path = None
|
|
working_path = None
|
|
rpf_md5_path = None
|
|
rpf_source_path = None
|
|
|
|
logger = None
|
|
md5_list = {}
|
|
|
|
|
|
def generate_audio_rpfs(platform, rebuild, branch):
|
|
global logger
|
|
|
|
logger = Logger('Audio RPFs', LOG_DIR, LOG_FILENAME % platform)
|
|
|
|
# Initialise paths to required dirs and files
|
|
if not init_paths(platform, branch):
|
|
logger.log("Failed to initialise paths", LogSeverity.ERROR)
|
|
failed()
|
|
return -1
|
|
|
|
# Load previous list of file MD5 values
|
|
load_md5()
|
|
|
|
# Generate RPF files
|
|
if not generate_rpfs(rebuild, platform):
|
|
failed()
|
|
return -1
|
|
|
|
#we dont need to build audio.rpf and audio_rel.rpf anymore for V
|
|
|
|
# Generate dev audio RPF file
|
|
#if not generate_audio_rpf(platform):
|
|
# failed()
|
|
# return -1
|
|
|
|
# Generate release audio RPF file
|
|
#if not generate_audio_rpf(platform, True):
|
|
# failed()
|
|
# return -1
|
|
|
|
# Save latest list of MD5 values
|
|
save_md5()
|
|
|
|
logger.save()
|
|
return 0
|
|
|
|
|
|
def init_paths(platform, branch):
|
|
global build_path
|
|
global audio_path
|
|
global rage_builder_path
|
|
global package_path_release
|
|
global package_path_main
|
|
global package_path_release_durango
|
|
global package_path_main_durango
|
|
global package_path_release_orbis
|
|
global package_path_main_orbis
|
|
global package_path_release_win64
|
|
global package_path_main_win64
|
|
|
|
global tools_root_path
|
|
|
|
global packlist_path_main
|
|
global packlist_path_release
|
|
global package_path_release
|
|
global input_path
|
|
global audio_output_path
|
|
global sfx_output_path
|
|
global build_pack_path
|
|
global rpf_md5_path
|
|
global rpf_source_path
|
|
global output_platform_name
|
|
|
|
#ragebuilder forces win32 if 'PC' is in output path so we will make sure to output to 'x64'
|
|
output_platform_name = platform
|
|
if platform.upper() == 'PC':
|
|
output_platform_name = 'X64'
|
|
|
|
# Init tools root path
|
|
tools_root_path = os.environ['RS_TOOLSROOT']
|
|
if not tools_root_path:
|
|
logger.log("'RS_TOOLSROOT' path not found in environment", LogSeverity.ERROR)
|
|
return False
|
|
|
|
ruby_path = os.environ['RUBYLIB']
|
|
if not ruby_path:
|
|
logger.log("'RUBYLIB' path not found in environment", LogSeverity.ERROR)
|
|
return False
|
|
|
|
# Init package paths
|
|
package_path_release_orbis = os.path.join(ruby_path, r'util\ragebuilder\package_audio_orbis_release.rbs')
|
|
package_path_main_orbis = os.path.join(ruby_path, r'util\ragebuilder\package_audio_orbis.rbs')
|
|
package_path_release_durango = os.path.join(ruby_path, r'util\ragebuilder\package_audio_durango_release.rbs')
|
|
package_path_main_durango = os.path.join(ruby_path, r'util\ragebuilder\package_audio_durango.rbs')
|
|
package_path_release_win64 = os.path.join(ruby_path, r'util\ragebuilder\package_audio_win64_release.rbs')
|
|
package_path_main_win64 = os.path.join(ruby_path, r'util\ragebuilder\package_audio_win64.rbs')
|
|
|
|
# Init rage builder path
|
|
rage_builder_path = os.path.join(tools_root_path, r'bin\ragebuilder_x64_0378.exe')
|
|
|
|
proj_root = os.environ['RS_PROJROOT']
|
|
if not proj_root:
|
|
logger.log("'RS_PROJROOT' path not found in environment", LogSeverity.ERROR)
|
|
return False
|
|
|
|
# Init audio path
|
|
audio_path = os.path.join(proj_root, 'audio', branch, 'staging')
|
|
|
|
# Init config dir
|
|
config_dir = os.path.join(audio_path, platform, 'config')
|
|
|
|
if not os.path.exists(config_dir):
|
|
logger.log("Creating path: %s" % config_dir, LogSeverity.INFO)
|
|
os.makedirs(config_dir)
|
|
|
|
# Init packlist paths (file may not exist locally)
|
|
packlist_path_main = os.path.join(config_dir, 'packlist.txt')
|
|
logger.log("Using main packlist path: %s" % packlist_path_main, LogSeverity.INFO)
|
|
packlist_path_release = os.path.join(config_dir, 'packlist_rel.txt')
|
|
logger.log("Using release packlist path: %s" % packlist_path_release, LogSeverity.INFO)
|
|
|
|
# Init input path
|
|
input_path = os.path.join(audio_path, platform, 'sfx')
|
|
if not os.path.exists(input_path):
|
|
logger.log("Path does not exist: %s" % input_path, LogSeverity.ERROR)
|
|
return False
|
|
logger.log("Using input path: %s" % input_path, LogSeverity.INFO)
|
|
|
|
if branch is None:
|
|
rpf_root_path = RPF_ROOT_PATH
|
|
else:
|
|
rpf_root_path = os.path.join(RPF_ROOT_PATH, branch)
|
|
|
|
rpf_md5_path = os.path.join(rpf_root_path, output_platform_name, 'md5_list.csv')
|
|
rpf_source_path = os.path.join(rpf_root_path, output_platform_name, 'audio')
|
|
|
|
# Init output path
|
|
if not os.path.exists(rpf_root_path):
|
|
logger.log("RPF output path does not exist: %s" % rpf_root_path, LogSeverity.ERROR)
|
|
return False
|
|
|
|
audio_output_path = os.path.join(rpf_root_path, output_platform_name, 'audio')
|
|
if not os.path.exists(audio_output_path):
|
|
logger.log("Path does not exist - creating directory: %s" % audio_output_path, LogSeverity.INFO)
|
|
try:
|
|
os.makedirs(audio_output_path)
|
|
except:
|
|
logger.log("Failed to create directory: %s" % audio_output_path, LogSeverity.ERROR)
|
|
return False
|
|
logger.log("Using audio output path: %s" % audio_output_path, LogSeverity.INFO)
|
|
|
|
sfx_output_path = os.path.join(audio_output_path, 'sfx')
|
|
if not os.path.exists(sfx_output_path):
|
|
logger.log("Path does not exist - creating directory: %s" % sfx_output_path, LogSeverity.INFO)
|
|
try:
|
|
os.makedirs(sfx_output_path)
|
|
except:
|
|
logger.log("Failed to create directory: %s" % sfx_output_path, LogSeverity.ERROR)
|
|
return False
|
|
logger.log("Using sfx output path: %s" % sfx_output_path, LogSeverity.INFO)
|
|
|
|
return True
|
|
|
|
|
|
def load_md5():
|
|
global md5_list
|
|
|
|
md5_list = {}
|
|
|
|
if os.path.exists(rpf_md5_path):
|
|
logger.log("Loading MD5 values: %s" % rpf_md5_path, LogSeverity.INFO)
|
|
|
|
md5_file = open(rpf_md5_path)
|
|
for line in md5_file:
|
|
if line:
|
|
file, md5 = line.split(',')
|
|
md5_list[file.upper()] = md5.replace('\n', '')
|
|
|
|
else:
|
|
logger.log("MD5 values file not found - will be created: %s" % rpf_md5_path, LogSeverity.INFO)
|
|
|
|
|
|
def save_md5():
|
|
global md5_list
|
|
|
|
logger.log("Saving MD5 values: %s" % rpf_md5_path, LogSeverity.INFO)
|
|
|
|
md5_file = open(rpf_md5_path, 'w+')
|
|
for file, md5 in md5_list.items():
|
|
md5_file.write("%s,%s\n" % (file, md5))
|
|
|
|
md5_file.close()
|
|
|
|
|
|
def generate_rpfs(rebuild, platform):
|
|
global input_path
|
|
global md5_list
|
|
global sfx_output_path
|
|
global package_path
|
|
|
|
if rebuild:
|
|
logger.log("Generating all audio RPF files..", LogSeverity.INFO)
|
|
else:
|
|
logger.log("Generating latest audio RPF files..", LogSeverity.INFO)
|
|
|
|
# Open pack list file for writing and overwrite previous data
|
|
pack_list_file_main = open(packlist_path_main, 'w+')
|
|
pack_list_file_release = open(packlist_path_release, 'w+')
|
|
|
|
# Generate RPFs for files in each pack (ie sub-dir) in input_path
|
|
for path, packs, files in os.walk(input_path):
|
|
for pack in packs:
|
|
|
|
# Add pack to dev file list (if required)
|
|
skip_for_dev = False
|
|
for except_pattern in IGNORE_EXCEPTION_PACKS_LIST:
|
|
if re.match(except_pattern, pack, re.IGNORECASE):
|
|
skip_for_dev = False
|
|
break
|
|
else:
|
|
for pattern in DEV_IGNORE_PACKS_LIST:
|
|
if re.match(pattern, pack, re.IGNORECASE):
|
|
logger.log("Skipping pack for dev pack list: %s" % pack, LogSeverity.INFO)
|
|
skip_for_dev = True
|
|
break
|
|
|
|
if not skip_for_dev:
|
|
pack_list_file_main.write("%s\n" % pack)
|
|
|
|
# Add pack to release file list (if required)
|
|
skip_for_rel = False
|
|
for except_pattern in IGNORE_EXCEPTION_PACKS_LIST:
|
|
if re.match(except_pattern, pack, re.IGNORECASE):
|
|
skip_for_rel = False
|
|
break
|
|
else:
|
|
for pattern in RELEASE_IGNORE_PACKS_LIST[platform.upper()]:
|
|
if re.match(pattern, pack, re.IGNORECASE):
|
|
logger.log("Skipping pack for release pack list: %s" % pack, LogSeverity.INFO)
|
|
skip_for_rel = True
|
|
break
|
|
|
|
if not skip_for_rel:
|
|
pack_list_file_release.write("%s\n" % pack)
|
|
|
|
if not rebuild and not pack_requires_build(os.path.join(path, pack), pack):
|
|
# Skip current pack if build not required and
|
|
# process not marked for full rebuild
|
|
logger.log("Skipping pack - build not required: %s" % pack, LogSeverity.INFO)
|
|
continue
|
|
|
|
# Init build params
|
|
logger.log("Building pack %s..." % pack, LogSeverity.INFO)
|
|
input_pack_path = os.path.join(input_path, pack) + '\\'
|
|
output_file = os.path.join(sfx_output_path, pack) + '.rpf'
|
|
|
|
if platform.upper() == 'PS4':
|
|
package_path = package_path_main_orbis
|
|
if platform.upper() == 'XBOXONE':
|
|
package_path = package_path_main_durango
|
|
if platform.upper() == 'PC':
|
|
package_path = package_path_main_win64
|
|
|
|
# Run build command
|
|
#cmd = r'%s %s -pack %s -rootpath %s -extignore .rpf -nameheapshift=1' % (rage_builder_path, package_path, output_file, input_pack_path)
|
|
cmd = r'%s %s -pack %s -rootpath %s -extignore .rpf' % (rage_builder_path, package_path, output_file, input_pack_path)
|
|
logger.log("Running cmd: %s" % cmd, LogSeverity.INFO)
|
|
if os.system(cmd) != 0:
|
|
logger.log("Failed to successfully run command: %s" % cmd, LogSeverity.ERROR)
|
|
return False
|
|
|
|
# Calculate MD5 of new file
|
|
if os.path.exists(output_file):
|
|
md5 = md5_for_file(output_file)
|
|
md5_list[output_file.upper()] = md5
|
|
logger.log("MD5 calculated for %s: %s" % (output_file, md5), LogSeverity.INFO)
|
|
|
|
# We save after each file in case process fails to complete but
|
|
# the file has still been generated
|
|
save_md5()
|
|
|
|
pack_list_file_main.close()
|
|
pack_list_file_release.close()
|
|
|
|
return True
|
|
|
|
|
|
def pack_requires_build(pack_path, pack_name):
|
|
global sfx_output_path
|
|
|
|
rpf_path = '%s.rpf' % os.path.join(sfx_output_path, pack_name)
|
|
|
|
# If output file doesn't exist, it needs to be built
|
|
if not os.path.exists(rpf_path):
|
|
return True
|
|
|
|
for file in os.listdir(pack_path):
|
|
# Only interested in AWC files
|
|
if file.lower().endswith('.awc'):
|
|
awc_path = os.path.join(pack_path, file)
|
|
# If AWC file is newer than RPF file, it needs to be built
|
|
if os.stat(awc_path).st_mtime > os.stat(rpf_path).st_mtime:
|
|
return True
|
|
|
|
return False
|
|
|
|
|
|
def generate_audio_rpf(platform, release=False):
|
|
global audio_path
|
|
global audio_output_path
|
|
global rage_builder_path
|
|
global package_path
|
|
|
|
root_path = os.path.join(audio_path, platform)
|
|
|
|
if release:
|
|
output_file = 'audio_rel.rpf'
|
|
if platform.upper() == 'PS4':
|
|
package_path = package_path_release_orbis
|
|
if platform.upper() == 'XBOXONE':
|
|
package_path = package_path_release_durango
|
|
if platform.upper() == 'PC':
|
|
package_path = package_path_release_win64
|
|
else:
|
|
output_file = 'audio.rpf'
|
|
if platform.upper() == 'PS4':
|
|
package_path = package_path_main_orbis
|
|
if platform.upper() == 'XBOXONE':
|
|
package_path = package_path_main_durango
|
|
if platform.upper() == 'PC':
|
|
package_path = package_path_main_win64
|
|
|
|
pack = os.path.join(audio_output_path, output_file)
|
|
|
|
cmd = r'%s %s -pack %s -rootpath %s\ -extignore .awc' % (rage_builder_path, package_path, pack, root_path)
|
|
#cmd = r'%s %s -pack %s -rootpath %s\ -extignore .awc -nameheapshift=1' % (rage_builder_path, package_path, pack, root_path)
|
|
|
|
logger.log("Running cmd: %s" % cmd, LogSeverity.INFO)
|
|
if os.system(cmd) != 0:
|
|
logger.log("Failed to successfully run command: %s" % cmd, LogSeverity.ERROR)
|
|
return False
|
|
|
|
if os.path.exists(pack):
|
|
md5 = md5_for_file(pack)
|
|
md5_list[pack.upper()] = md5
|
|
logger.log("MD5 calculated for %s: %s" % (pack, md5), LogSeverity.INFO)
|
|
save_md5()
|
|
|
|
return True
|
|
|
|
|
|
def set_working_path(path):
|
|
global working_path
|
|
|
|
# Change working path if required
|
|
if working_path != path:
|
|
working_path = path
|
|
msg = "Setting working path as %s" % path
|
|
logger.log(msg, LogSeverity.INFO)
|
|
os.chdir(path)
|
|
|
|
|
|
def run_p4_cmd(action, args):
|
|
cmd = '%s %s %s' % (PERFORCE_EXE_NAME, action, args)
|
|
logger.log("Running Perforce cmd: %s" % cmd, LogSeverity.INFO)
|
|
|
|
if os.system(cmd) != 0:
|
|
logger.log("Failed to successfully run P4 command: %s" % cmd, LogSeverity.ERROR)
|
|
return False
|
|
|
|
return True
|
|
|
|
|
|
def md5_for_file(path, block_size=2**20):
|
|
logger.log("Generating MD5 for file: %s" % path, LogSeverity.INFO)
|
|
|
|
file = open(path)
|
|
md5 = hashlib.md5()
|
|
while True:
|
|
data = file.read(block_size)
|
|
if not data:
|
|
break
|
|
md5.update(data)
|
|
return md5.hexdigest()
|
|
|
|
|
|
def failed():
|
|
logger.save()
|