import datetime import hashlib import os import re import sys from audio_logger import Logger, LogSeverity # Runtime data NOW_START = datetime.datetime.now() NOW_START_STR = NOW_START.strftime('%y_%m_%d__%H_%M_%S') RPF_ROOT_PATH = r'\\rockstar.t2.corp\Network\RSGEDI\Audio\Audio\RPF' #RPF_ROOT_PATH = r'd:\rpfs' LOG_DIR = os.path.join(r'C:\Temp\Logs\Audio RPFs', NOW_START_STR) LOG_FILENAME = r'run_log_%s.txt' # Perforce PERFORCE_EXE_NAME = r'p4' PLATFORMS = ( 'PS4', 'XBOXONE', 'PC', ) DEV_IGNORE_PACKS_LIST = ( #'^PS_(.*)', '(.*)DLC(.*)', ) RELEASE_IGNORE_PACKS_LIST = { 'PS4' : ( '(.*)DLC(.*)', 'TEST(.*)', '^CUTSCENE$', '^CUTSCENE_MASTERED_TRIMMED$', ), 'XBOXONE' : ( '^PS_(.*)', '(.*)DLC(.*)', 'TEST(.*)', '^CUTSCENE$', '^CUTSCENE_MASTERED_TRIMMED$', ), 'PC' : ( '(.*)DLC(.*)', 'TEST(.*)', '^CUTSCENE$', '^CUTSCENE_MASTERED_TRIMMED$', ), } IGNORE_EXCEPTION_PACKS_LIST = ( 'DLC_GTAO$', 'DLCTU$', ) # Runtime args audio_path = None rage_builder_path = None package_path_release = None package_path_main = None tools_root_path = None packlist_path_main = None packlist_path_release = None input_path = None audio_output_path = None sfx_output_path = None build_pack_path = None working_path = None rpf_md5_path = None rpf_source_path = None logger = None md5_list = {} def generate_audio_rpfs(platform, rebuild, branch): global logger logger = Logger('Audio RPFs', LOG_DIR, LOG_FILENAME % platform) # Initialise paths to required dirs and files if not init_paths(platform, branch): logger.log("Failed to initialise paths", LogSeverity.ERROR) failed() return -1 # Load previous list of file MD5 values load_md5() # Generate RPF files if not generate_rpfs(rebuild, platform): failed() return -1 #we dont need to build audio.rpf and audio_rel.rpf anymore for V # Generate dev audio RPF file #if not generate_audio_rpf(platform): # failed() # return -1 # Generate release audio RPF file #if not generate_audio_rpf(platform, True): # failed() # return -1 # Save latest list of MD5 values save_md5() logger.save() return 0 def init_paths(platform, branch): global build_path global audio_path global rage_builder_path global package_path_release global package_path_main global package_path_release_durango global package_path_main_durango global package_path_release_orbis global package_path_main_orbis global package_path_release_win64 global package_path_main_win64 global tools_root_path global packlist_path_main global packlist_path_release global package_path_release global input_path global audio_output_path global sfx_output_path global build_pack_path global rpf_md5_path global rpf_source_path global output_platform_name #ragebuilder forces win32 if 'PC' is in output path so we will make sure to output to 'x64' output_platform_name = platform if platform.upper() == 'PC': output_platform_name = 'X64' # Init tools root path tools_root_path = os.environ['RS_TOOLSROOT'] if not tools_root_path: logger.log("'RS_TOOLSROOT' path not found in environment", LogSeverity.ERROR) return False ruby_path = os.environ['RUBYLIB'] if not ruby_path: logger.log("'RUBYLIB' path not found in environment", LogSeverity.ERROR) return False # Init package paths package_path_release_orbis = os.path.join(ruby_path, r'util\ragebuilder\package_audio_orbis_release.rbs') package_path_main_orbis = os.path.join(ruby_path, r'util\ragebuilder\package_audio_orbis.rbs') package_path_release_durango = os.path.join(ruby_path, r'util\ragebuilder\package_audio_durango_release.rbs') package_path_main_durango = os.path.join(ruby_path, r'util\ragebuilder\package_audio_durango.rbs') package_path_release_win64 = os.path.join(ruby_path, r'util\ragebuilder\package_audio_win64_release.rbs') package_path_main_win64 = os.path.join(ruby_path, r'util\ragebuilder\package_audio_win64.rbs') # Init rage builder path rage_builder_path = os.path.join(tools_root_path, r'bin\ragebuilder_x64_0378.exe') proj_root = os.environ['RS_PROJROOT'] if not proj_root: logger.log("'RS_PROJROOT' path not found in environment", LogSeverity.ERROR) return False # Init audio path audio_path = os.path.join(proj_root, 'audio', branch, 'staging') # Init config dir config_dir = os.path.join(audio_path, platform, 'config') if not os.path.exists(config_dir): logger.log("Creating path: %s" % config_dir, LogSeverity.INFO) os.makedirs(config_dir) # Init packlist paths (file may not exist locally) packlist_path_main = os.path.join(config_dir, 'packlist.txt') logger.log("Using main packlist path: %s" % packlist_path_main, LogSeverity.INFO) packlist_path_release = os.path.join(config_dir, 'packlist_rel.txt') logger.log("Using release packlist path: %s" % packlist_path_release, LogSeverity.INFO) # Init input path input_path = os.path.join(audio_path, platform, 'sfx') if not os.path.exists(input_path): logger.log("Path does not exist: %s" % input_path, LogSeverity.ERROR) return False logger.log("Using input path: %s" % input_path, LogSeverity.INFO) if branch is None: rpf_root_path = RPF_ROOT_PATH else: rpf_root_path = os.path.join(RPF_ROOT_PATH, branch) rpf_md5_path = os.path.join(rpf_root_path, output_platform_name, 'md5_list.csv') rpf_source_path = os.path.join(rpf_root_path, output_platform_name, 'audio') # Init output path if not os.path.exists(rpf_root_path): logger.log("RPF output path does not exist: %s" % rpf_root_path, LogSeverity.ERROR) return False audio_output_path = os.path.join(rpf_root_path, output_platform_name, 'audio') if not os.path.exists(audio_output_path): logger.log("Path does not exist - creating directory: %s" % audio_output_path, LogSeverity.INFO) try: os.makedirs(audio_output_path) except: logger.log("Failed to create directory: %s" % audio_output_path, LogSeverity.ERROR) return False logger.log("Using audio output path: %s" % audio_output_path, LogSeverity.INFO) sfx_output_path = os.path.join(audio_output_path, 'sfx') if not os.path.exists(sfx_output_path): logger.log("Path does not exist - creating directory: %s" % sfx_output_path, LogSeverity.INFO) try: os.makedirs(sfx_output_path) except: logger.log("Failed to create directory: %s" % sfx_output_path, LogSeverity.ERROR) return False logger.log("Using sfx output path: %s" % sfx_output_path, LogSeverity.INFO) return True def load_md5(): global md5_list md5_list = {} if os.path.exists(rpf_md5_path): logger.log("Loading MD5 values: %s" % rpf_md5_path, LogSeverity.INFO) md5_file = open(rpf_md5_path) for line in md5_file: if line: file, md5 = line.split(',') md5_list[file.upper()] = md5.replace('\n', '') else: logger.log("MD5 values file not found - will be created: %s" % rpf_md5_path, LogSeverity.INFO) def save_md5(): global md5_list logger.log("Saving MD5 values: %s" % rpf_md5_path, LogSeverity.INFO) md5_file = open(rpf_md5_path, 'w+') for file, md5 in md5_list.items(): md5_file.write("%s,%s\n" % (file, md5)) md5_file.close() def generate_rpfs(rebuild, platform): global input_path global md5_list global sfx_output_path global package_path if rebuild: logger.log("Generating all audio RPF files..", LogSeverity.INFO) else: logger.log("Generating latest audio RPF files..", LogSeverity.INFO) # Open pack list file for writing and overwrite previous data pack_list_file_main = open(packlist_path_main, 'w+') pack_list_file_release = open(packlist_path_release, 'w+') # Generate RPFs for files in each pack (ie sub-dir) in input_path for path, packs, files in os.walk(input_path): for pack in packs: # Add pack to dev file list (if required) skip_for_dev = False for except_pattern in IGNORE_EXCEPTION_PACKS_LIST: if re.match(except_pattern, pack, re.IGNORECASE): skip_for_dev = False break else: for pattern in DEV_IGNORE_PACKS_LIST: if re.match(pattern, pack, re.IGNORECASE): logger.log("Skipping pack for dev pack list: %s" % pack, LogSeverity.INFO) skip_for_dev = True break if not skip_for_dev: pack_list_file_main.write("%s\n" % pack) # Add pack to release file list (if required) skip_for_rel = False for except_pattern in IGNORE_EXCEPTION_PACKS_LIST: if re.match(except_pattern, pack, re.IGNORECASE): skip_for_rel = False break else: for pattern in RELEASE_IGNORE_PACKS_LIST[platform.upper()]: if re.match(pattern, pack, re.IGNORECASE): logger.log("Skipping pack for release pack list: %s" % pack, LogSeverity.INFO) skip_for_rel = True break if not skip_for_rel: pack_list_file_release.write("%s\n" % pack) if not rebuild and not pack_requires_build(os.path.join(path, pack), pack): # Skip current pack if build not required and # process not marked for full rebuild logger.log("Skipping pack - build not required: %s" % pack, LogSeverity.INFO) continue # Init build params logger.log("Building pack %s..." % pack, LogSeverity.INFO) input_pack_path = os.path.join(input_path, pack) + '\\' output_file = os.path.join(sfx_output_path, pack) + '.rpf' if platform.upper() == 'PS4': package_path = package_path_main_orbis if platform.upper() == 'XBOXONE': package_path = package_path_main_durango if platform.upper() == 'PC': package_path = package_path_main_win64 # Run build command #cmd = r'%s %s -pack %s -rootpath %s -extignore .rpf -nameheapshift=1' % (rage_builder_path, package_path, output_file, input_pack_path) cmd = r'%s %s -pack %s -rootpath %s -extignore .rpf' % (rage_builder_path, package_path, output_file, input_pack_path) logger.log("Running cmd: %s" % cmd, LogSeverity.INFO) if os.system(cmd) != 0: logger.log("Failed to successfully run command: %s" % cmd, LogSeverity.ERROR) return False # Calculate MD5 of new file if os.path.exists(output_file): md5 = md5_for_file(output_file) md5_list[output_file.upper()] = md5 logger.log("MD5 calculated for %s: %s" % (output_file, md5), LogSeverity.INFO) # We save after each file in case process fails to complete but # the file has still been generated save_md5() pack_list_file_main.close() pack_list_file_release.close() return True def pack_requires_build(pack_path, pack_name): global sfx_output_path rpf_path = '%s.rpf' % os.path.join(sfx_output_path, pack_name) # If output file doesn't exist, it needs to be built if not os.path.exists(rpf_path): return True for file in os.listdir(pack_path): # Only interested in AWC files if file.lower().endswith('.awc'): awc_path = os.path.join(pack_path, file) # If AWC file is newer than RPF file, it needs to be built if os.stat(awc_path).st_mtime > os.stat(rpf_path).st_mtime: return True return False def generate_audio_rpf(platform, release=False): global audio_path global audio_output_path global rage_builder_path global package_path root_path = os.path.join(audio_path, platform) if release: output_file = 'audio_rel.rpf' if platform.upper() == 'PS4': package_path = package_path_release_orbis if platform.upper() == 'XBOXONE': package_path = package_path_release_durango if platform.upper() == 'PC': package_path = package_path_release_win64 else: output_file = 'audio.rpf' if platform.upper() == 'PS4': package_path = package_path_main_orbis if platform.upper() == 'XBOXONE': package_path = package_path_main_durango if platform.upper() == 'PC': package_path = package_path_main_win64 pack = os.path.join(audio_output_path, output_file) cmd = r'%s %s -pack %s -rootpath %s\ -extignore .awc' % (rage_builder_path, package_path, pack, root_path) #cmd = r'%s %s -pack %s -rootpath %s\ -extignore .awc -nameheapshift=1' % (rage_builder_path, package_path, pack, root_path) logger.log("Running cmd: %s" % cmd, LogSeverity.INFO) if os.system(cmd) != 0: logger.log("Failed to successfully run command: %s" % cmd, LogSeverity.ERROR) return False if os.path.exists(pack): md5 = md5_for_file(pack) md5_list[pack.upper()] = md5 logger.log("MD5 calculated for %s: %s" % (pack, md5), LogSeverity.INFO) save_md5() return True def set_working_path(path): global working_path # Change working path if required if working_path != path: working_path = path msg = "Setting working path as %s" % path logger.log(msg, LogSeverity.INFO) os.chdir(path) def run_p4_cmd(action, args): cmd = '%s %s %s' % (PERFORCE_EXE_NAME, action, args) logger.log("Running Perforce cmd: %s" % cmd, LogSeverity.INFO) if os.system(cmd) != 0: logger.log("Failed to successfully run P4 command: %s" % cmd, LogSeverity.ERROR) return False return True def md5_for_file(path, block_size=2**20): logger.log("Generating MD5 for file: %s" % path, LogSeverity.INFO) file = open(path) md5 = hashlib.md5() while True: data = file.read(block_size) if not data: break md5.update(data) return md5.hexdigest() def failed(): logger.save()