Files
gtav-src/tools_ng/bin/audio/scripts/Build/daily_sync_ng.py
T
2025-09-29 00:52:08 +02:00

634 lines
20 KiB
Python
Executable File

'''
This process performs the steps involved in the daily audio sync process:
1. Get latest data
2. Build and submit SCRIPTED_SPEECH pack
3. Build and submit placeholder speech waves
4. Get latest data again
5. Build and submit all audio packs
6. Integrate data into staging
7. Build audio RPF files and place in a public directory ready for testing.
'''
import os
import subprocess
import sys
import time
from datetime import datetime
from audio_logger import Logger, LogSeverity
from audio_rpfs_ng import generate_audio_rpfs
from placeholder_speech import generate_placeholder_speech
from P4 import P4
# Wait for user input between each stage of the sync process (for testing)
WAIT_FOR_INPUT = True
# Earliest time (HH:MM) to start phase two of sync
WAIT_HOUR = 4
WAIT_MINUTE = 0
NOW_START = datetime.now()
NOW_START_STR = NOW_START.strftime('%y_%m_%d__%H_%M_%S')
PERFORCE_SYNC_PATHS = (
# Path, Allow force sync, Sync to change list (or get latest)
(r'//depot/gta5/tools_ng/bin/audio/AudioDataBuilder/...', True, False),
(r'//depot/gta5/tools_ng/bin/audio/metadatacompiler/...', True, False),
(r'//depot/gta5/tools_ng/bin/audio/PerforceLibTools/...', True, False),
(r'//depot/gta5/tools_ng/bin/audio/pipeline/...', True, False),
(r'//depot/gta5/tools_ng/bin/audio/PlaceholderSpeechGenerator/...', True, False),
(r'//depot/gta5/audio/{branch}/build/...', True, False),
(r'//depot/gta5/audio/{branch}/runtime/...', True, False),
(r'//depot/gta5/audio/{branch}/staging/...', True, False),
(r'//depot/gta5/audio/{branch}/stagingrpf/...', True, False),
(r'//depot/gta5/audio/{branch}/projectSettings.xml', True, False),
(r'//depot/gta5/audio/{branch}/assets/LipsyncAnims/...', True, False),
(r'//depot/gta5/audio/{branch}/assets/LipSyncTextFiles/...', True, False),
(r'//depot/gta5/audio/{branch}/assets/Objects/...', False, False),
(r'//depot/gta5/audio/{branch}/assets/Waves/...', False, False),
)
DEFAULT_BRANCH = 'dev_ng'
PERFORCE_EXE_NAME = r'p4'
P4SERVER = r'rsgperforce'
P4PORT = r'1666'
P4CLIENT = r'dailySync-GTAVdev_ng'
P4USER = r'svcrsgaudauto'
P4PASSWD = r'few-xSz5bxN'
P4DEPOT = r'//depot'
P4_ARGS = (
r'-p4host '+ P4SERVER + ':' + P4PORT,
r'-p4client '+P4CLIENT,
r'-p4user '+P4USER,
r'-p4passwd '+P4PASSWD,
r'-p4depotroot '+P4DEPOT,
)
# Audio build settings
AUDIO_BUILD_WORKING_PATH = r'X:\gta5\tools_ng\bin\audio\AudioDataBuilder'
AUDIO_BUILD_EXE_NAME = r'audioDataBuilder.exe'
AUDIO_BUILD_ARGS_BASE = (
r'-project \gta5\audio\{branch}\projectSettings.xml',
r'-temppath D:\Temp', # Required on Ivona server due to disk space shortage on C:\
)
# P4 integration settings
INTEGRATE_TOOL_WORKING_PATH = r'X:\gta5\tools_ng\bin\audio\PerforceLibTools'
INTEGRATE_TOOL_EXE_NAME = r'perforceIntegrate.exe'
INTEGRATE_TOOL_ARGS_BASE = (
r'-resolve accepttheirs',
)
PLATFORMS = (
'PC',
'PS4',
'XBOXONE',
'PS5',
'XBOXSERIES'
)
ALL_PLATFORMS = 'all_platforms'
LOG_DIR = os.path.join(r'C:\Temp\Logs\Daily Sync', NOW_START_STR)
# SMTP email notification settings
SMTP_FROM = 'Audio Sync <audio-sync@rockstarnorth.com>'
SMTP_TO = (
'Audio Sync GTA5 NG <Audio_Sync_GTA5_NG@rockstarnorth.com>',
)
change_list_number = None
branch = None
run_placeholder_incremental = False
run_placeholder_full = False
run_audio_build = False
integrate = False
run_rpf_build = False
run_rpf_rebuild = False
no_email = False
email_addresses = None
use_p4_env = False
platform = None
triggered_context = None
wait_after_placeholder = False
logger = None
def main():
global logger
global platform
global triggered_context
logger = Logger('[GTA5 NG] Daily Sync', LOG_DIR)
logger.setPassword(P4PASSWD)
logger.log_spacer()
logger.log("Running daily sync process", LogSeverity.INFO)
if parse_args() != 0:
return -1
if triggered_context is not None:
logger.log_spacer()
logger.log("Triggered Context: %s" % triggered_context, LogSeverity.INFO)
logger.log_spacer()
# Validate and set platform if required
if platform is None:
platform = ALL_PLATFORMS
else:
if platform.upper() not in PLATFORMS:
logger.log("Invalid platform specified: %s" % platform, LogSeverity.ERROR)
return -1
# initialise p4 environment
if initialise_p4environment() != 0:
error()
# Step 1. Get latest data
if get_latest_data() != 0:
error()
if run_placeholder_incremental or run_placeholder_full:
# Step 2. Build placeholder speech packs & Build and submit placeholder speech waves
if build_placeholder_speech() != 0:
#error()
# It's okay to continue if placeholder speech build fails, it shouldn't affect the sync
pass
# Step 3. Get latest data again
if get_latest_data() != 0:
error()
if(wait_after_placeholder):
wait()
# We need to wait until a specified time of day to be able to
# run the next steps
if (run_audio_build or integrate or run_rpf_build or run_rpf_rebuild):
# Get latest data again - do this once for all platforms
# so they all use the same source asset change list
if get_latest_data() != 0:
error()
if platform == ALL_PLATFORMS:
# Run remaining steps per platform
for curr_platform in PLATFORMS:
process_platform(curr_platform)
else:
process_platform(platform)
# Log change list number that audio data was synced at
if run_audio_build and change_list_number is not None:
logger.log_spacer()
logger.log("Audio data synced at change list %s" % change_list_number, LogSeverity.INFO)
# Mail the results
if not no_email:
if email_addresses is not None:
logger.mail_results(SMTP_FROM, email_addresses)
else:
logger.mail_results(SMTP_FROM, SMTP_TO)
logger.save()
return 0
def process_platform(platform):
# Step 5. Build and submit all audio packs
if run_audio_build:
if build_packs(platform, None, change_list_number, True) != 0:
error()
# Step 6. Integrate data into staging
if integrate:
if integrate_into_staging(platform) != 0:
error()
# Step 7. Build audio RPF files and place in a public directory ready for testing.
if run_rpf_build:
if build_rpfs(platform, False) != 0:
error()
elif run_rpf_rebuild:
if build_rpfs(platform, True) != 0:
error()
def initialise_p4environment():
logger.log("Setting P4 environment", LogSeverity.INFO)
cmd = '%s set P4PORT=%s:%s' % (PERFORCE_EXE_NAME, P4SERVER, P4PORT)
logger.log(" %s" % cmd, LogSeverity.INFO)
if os.system(cmd) != 0:
logger.log("Command failed: %s" % cmd, LogSeverity.ERROR)
return -1
cmd = '%s set P4CLIENT=%s' % (PERFORCE_EXE_NAME, P4CLIENT)
logger.log(" %s" % cmd, LogSeverity.INFO)
if os.system(cmd) != 0:
logger.log("Command failed: %s" % cmd, LogSeverity.ERROR)
return -1
cmd = '%s set P4USER=%s' % (PERFORCE_EXE_NAME, P4USER)
logger.log(" %s" % cmd, LogSeverity.INFO)
if os.system(cmd) != 0:
logger.log("Command failed: %s" % cmd, LogSeverity.ERROR)
return -1
cmd = '%s set P4PASSWD=%s' % (PERFORCE_EXE_NAME, P4PASSWD)
logger.log(" %s" % cmd, LogSeverity.INFO)
if os.system(cmd) != 0:
logger.log("Command failed: %s" % cmd, LogSeverity.ERROR)
return -1
return 0
def parse_args():
# Set run modes based on input args
global run_placeholder_full
global run_placeholder_incremental
global run_audio_build
global integrate
global run_rpf_build
global run_rpf_rebuild
global no_email
global email_addresses
global use_p4_env
global platform
global change_list_number
global branch
global triggered_context
global wait_after_placeholder
logger.log_spacer()
logger.log("Parsing input args...", LogSeverity.INFO)
args = sys.argv
counter = 1
for arg in args[1:]:
if arg == '-pf':
run_placeholder_full = True
elif arg == '-pi':
run_placeholder_incremental = True
elif arg == '-b':
run_audio_build = True
elif arg == '-i':
integrate = True
elif arg == '-r':
run_rpf_build = True
elif arg == '-ar':
run_rpf_rebuild = True
elif arg == '-noemail':
no_email = True
elif arg == '-email':
if email_addresses is None:
email_addresses = []
counter += 1
while counter < len(args) and not args[counter].startswith('-'):
address = args[counter].upper()
if address not in email_addresses:
email_addresses.append(address)
counter += 1
counter -= 1
elif arg == '-usep4env':
use_p4_env = True
elif arg == '-platform':
platform = args[counter+1]
counter += 1
elif arg == '-cl':
change_list_number = args[counter+1]
counter += 1
elif arg == '-br':
branch = args[counter+1]
counter += 1
elif arg == '-tc':
triggered_context = args[counter+1]
counter += 1
elif arg == '-wait':
wait_after_placeholder = True
elif arg[0] != '-':
continue
else:
logger.log("Invalid arg: %s" % arg, LogSeverity.ERROR)
help()
return -1
counter += 1
if run_placeholder_full and run_placeholder_incremental:
logger.log("Cannot choose both full and incremental placeholder speech generation modes", LogSeverity.ERROR)
help()
return -1
if run_rpf_build and run_rpf_rebuild:
logger.log("Cannot choose both full and incremental RPF build modes", LogSeverity.ERROR)
help()
return -1
# Log the settings used
logger.log("Full placeholder speech: %s" % run_placeholder_full, LogSeverity.INFO)
logger.log("Incremental placeholder speech: %s" % run_placeholder_incremental, LogSeverity.INFO)
logger.log("Audio build: %s" % run_audio_build, LogSeverity.INFO)
logger.log("Integrate into staging: %s" % integrate, LogSeverity.INFO)
logger.log("RPF build: %s" % run_rpf_build, LogSeverity.INFO)
logger.log("Full RPF rebuild: %s" % run_rpf_rebuild, LogSeverity.INFO)
logger.log("No email: %s" % no_email, LogSeverity.INFO)
logger.log("Email alerts to: %s" % (', '.join(email_addresses) if email_addresses is not None else 'Default List'), LogSeverity.INFO)
logger.log("Platform: %s" % (platform if platform is not None else ', '.join(PLATFORMS)), LogSeverity.INFO)
logger.log("Change list sync at: %s" % (change_list_number if change_list_number is not None else 'Latest'), LogSeverity.INFO)
logger.log("Use P4 environment variables: %s" % use_p4_env, LogSeverity.INFO)
placeholderInfo=''
if run_placeholder_full or run_placeholder_incremental:
placeholderInfo=' (Placeholder generation)'
if branch is None:
logger.updateName("[GTA5 %s] DailySync%s" % (DEFAULT_BRANCH, placeholderInfo))
else:
logger.updateName("[GTA5 %s] DailySync%s" % (branch, placeholderInfo))
return 0
def get_latest_data():
get_latest_change_list_number()
logger.log_spacer()
logger.log("Getting latest data from Perforce...", LogSeverity.INFO)
# Run Perforce sync on all required paths
for path, can_force, sync_to_cl in PERFORCE_SYNC_PATHS:
path = replace_branch(path)
# Want to sync all paths at the same change list number
if change_list_number is not None and sync_to_cl:
path = path + '@' + change_list_number
cmd = '%s sync -q %s' % (PERFORCE_EXE_NAME, path)
logger.log("Running Perforce sync: %s" % cmd, LogSeverity.INFO)
if os.system(cmd) != 0:
# Try a force sync if failure occurred
if can_force:
logger.log("Failed to run normal sync - trying forced sync: %s" % path, LogSeverity.WARNING)
cmd = '%s sync -q -f %s' % (PERFORCE_EXE_NAME, path)
logger.log("Running Perforce forced sync: %s" % cmd, LogSeverity.INFO)
# Do not continue if force sync failed
if os.system(cmd) != 0:
logger.log("Failed to get latest data: %s" % path, LogSeverity.ERROR)
return -1
else:
logger.log("Failed to get latest: %s" % path, LogSeverity.ERROR)
return -1
return 0
def get_latest_change_list_number():
global change_list_number
logger.log_spacer()
logger.log("Getting latest change list number affecting audio files...", LogSeverity.INFO)
audioRootPath = replace_branch('//depot/gta5/audio/{branch}/...')
proc = subprocess.Popen([PERFORCE_EXE_NAME, 'changes', '-ssubmitted', '-m1', audioRootPath], stdout=subprocess.PIPE, shell=True)
(out, err) = proc.communicate()
if err is not None:
logger.log("Failed to get latest change list number: %s" % err, LogSeverity.ERROR)
return -1
split = out.split(" ")
if len(split) < 2:
logger.log("Failed to parse P4 response: %s" % out, LogSeverity.ERROR)
return -1
change_list_number = split[1]
logger.log("Set latest change list number as: %s" % change_list_number, LogSeverity.INFO)
return 0
def build_packs(platform, pack_list, sync_change_list_number, build_metadata):
global use_p4_env
logger.log_spacer()
set_working_path(AUDIO_BUILD_WORKING_PATH)
result = 0
args = []
for arg in AUDIO_BUILD_ARGS_BASE:
args += [
replace_branch(arg),
]
if build_metadata:
args += [
r'-buildmetadata',
]
if use_p4_env:
args += get_p4_env_vars()
else:
args += P4_ARGS
if platform != ALL_PLATFORMS:
args += [
r'-platform %s' % platform,
]
# Add change list number if set to build from same data set
if sync_change_list_number is not None:
args += [
r'-changelistnumber %s' % sync_change_list_number,
]
if pack_list is None:
# Build all packs
logger.log("Running audio build for all packs...", LogSeverity.INFO)
result = build_audio_data(args)
return result
# Else, build specified packs
for pack in pack_list:
print args
logger.log("Running audio build for pack %s..." % pack, LogSeverity.INFO)
inner_args = args + [
r'-pack %s' % pack,
]
# Want to try building all packs but keep track of any failure
inner_result = build_audio_data(inner_args)
if inner_result != 0:
logger.log("Failed to build pack: %s" % pack, LogSeverity.ERROR)
if result == 0:
result = inner_result
return result
def build_audio_data(args):
args_str = ' '.join(args)
logger.log_spacer()
# Execute audio builder process
cmd = '%s %s' % (AUDIO_BUILD_EXE_NAME, args_str)
logger.log("Executing cmd: %s" % cmd, LogSeverity.INFO)
result = os.system(cmd)
# Check for errors
if result == 0:
logger.log("Pack built successfully", LogSeverity.INFO)
else:
msg = '%s failed with error code %d\n' % (AUDIO_BUILD_EXE_NAME, result)
logger.log(msg, LogSeverity.ERROR)
return result
def build_placeholder_speech():
global no_email
global use_p4_env
result = 0
if not run_placeholder_incremental and not run_placeholder_full:
return result
logger.log_spacer()
if run_placeholder_incremental:
logger.log("Running incremental placeholder speech process...", LogSeverity.INFO)
result = generate_placeholder_speech(False, use_p4_env, no_email, branch)
elif run_placeholder_full:
logger.log("Running full placeholder speech process...", LogSeverity.INFO)
result = generate_placeholder_speech(True, use_p4_env, no_email, branch)
if result == 0:
logger.log("Placeholder speech process completed successfully", LogSeverity.INFO)
else:
logger.log("Placeholder speech process failed", LogSeverity.ERROR)
return result
def integrate_into_staging(platform):
global use_p4_env
if integrate:
set_working_path(INTEGRATE_TOOL_WORKING_PATH)
logger.log_spacer()
logger.log("Integrating files into staging...", LogSeverity.INFO)
args = INTEGRATE_TOOL_ARGS_BASE + (
r'-branch gta5_audio_to_staging_%s' % platform,
)
if not use_p4_env:
args += get_p4_env_vars()
else:
args += P4_ARGS
args_str = ' '.join(args)
cmd = '%s %s' % (INTEGRATE_TOOL_EXE_NAME, args_str)
logger.log("Running Perforce integration: %s" % cmd, LogSeverity.INFO)
if os.system(cmd) != 0:
logger.log("Perforce integration failed: %s" % cmd, LogSeverity.ERROR)
return -1
return 0
def build_rpfs(platform, rebuild):
logger.log_spacer()
return generate_audio_rpfs(platform, rebuild, branch or DEFAULT_BRANCH)
def get_p4_env_vars():
p4 = P4()
return (
r'-p4host %s' % p4.port,
r'-p4client %s' % p4.client,
r'-p4user %s' % p4.user,
r'-p4depotroot //depot',
)
def set_working_path(path):
logger.log("Setting working path as %s" % path, LogSeverity.INFO)
os.chdir(path)
def replace_branch(path):
return path.replace('{branch}', branch or DEFAULT_BRANCH)
def wait():
# Pause the process until the specified time of day has been reached.
today = datetime.today()
future = datetime(today.year, today.month, today.day, WAIT_HOUR, WAIT_MINUTE)
duration = (future-today).total_seconds()
if duration > 0:
logger.log("Pausing process for %d second(s)" % duration, LogSeverity.INFO)
time.sleep(duration)
return True
else:
logger.log("Wait duration has already passed; no need to wait", LogSeverity.INFO)
return False
def error():
logger.log("Process failed to complete", LogSeverity.ERROR)
if not no_email:
if email_addresses is not None:
logger.mail_results(SMTP_FROM, email_addresses)
else:
logger.mail_results(SMTP_FROM, SMTP_TO)
logger.save()
exit()
def help():
print "\nCommand to run daily sync: daily_sync.py [-pf/-pi -b -i -r -noemail -platform <platform> -cl <change list>]\n"
print "-pf Generate placeholder speech (full rebuild)"
print "-pi Generate placeholder speech (incremental build)"
print "-b Run full audio build"
print "-i Integrate into staging"
print "-r Generate RPF files (incremental)"
print "-ar Generate all RPF files (rebuild)"
print "-noemail Skips the email notification step at end of process"
print "-email List of email addresses to send notifications to (space separated value, uses default mail list if not specified)"
print "-platform <platform> Specify a particular platform to sync"
print "-cl <change list> Specify a particular change list to sync at"
print "-br <branch name> Sync to a branch instead of Live"
print "-tc <triggered context name> Description indicating where the script was executed from"
print "-wait Wait after placeholder speech generation"
if __name__ == "__main__":
main()