import os import shutil import subprocess import logging as log import timeit import glob import argparse import multiprocessing rpf_ls = "X:/gta5/src/dev_gen9_sga/rage/framework/tools/src/cli/RageInterop/bin/x64/Release/RPF-ls.exe" magpie = "X:/gta5/src/dev_gen9_sga/rage/framework/tools/src/cli/RageInterop/bin/x64/Release/Magpie.exe" sp_args = "{magpie} -out {plat}/patchBvh00/sp.rpf -referenced -flatten -path_from dev_gen9_bvh_rebuild -dlc_patch X:/gta5/titleupdate/dev_gen9_bvh_rebuild/dlc_patch/ -log_adds -logfile {plat}/patchBvh00/sp.txt -keep_loose -ignore ignore.txt -patterns *.{letter}bvh,*.{letter}bvhd @sp_paths.txt -X:/gta5/build/dev_gen9_bvh_rebuild/{ignore_plat}/* -X:/gta5/titleupdate/dev_gen9_bvh_rebuild/{ignore_plat}/*" sp_content_args = "python content.py -mp dlc_bvhPatch00:/ -p {plat}/patchBvh00/_sp -cs CCS_BVHPATCH00_STREAMING -o {plat}/patchBvh00/" sp_patch_args = "{magpie} -out {plat}/patchBvh00/dlc.rpf -platform {letter} -log_adds -logfile {plat}/patchBvh00/dlc.txt -adds {plat}/patchBvh00/content.xml,{plat}/patchBvh00/setup2.xml,{plat}/patchBvh00/_sp" base_mp_args = "{magpie} -out {plat}/mpBvh00/base_mp.rpf -rpf_postfixes _update,_bvh -path_from dev_gen9_bvh_rebuild -log_adds -logfile {plat}/mpBvh00/base_mp.txt -keep_loose -ignore ignore.txt -patterns *.{letter}bvh,*.{letter}bvhd X:/gta5/build/dev_gen9_bvh_rebuild/{plat}/dlcPacks/mp*" update_mp_args = "{magpie} -out {plat}/mpBvh00/update_mp.rpf -rpf_postfixes _update,_bvh -path_from dev_gen9_bvh_rebuild -log_adds -logfile {plat}/mpBvh00/update_mp.txt -keep_loose -ignore ignore.txt -patterns *.{letter}bvh,*.{letter}bvhd X:/gta5/titleupdate/dev_gen9_bvh_rebuild/{plat}/dlcPacks/mp*" dlc_patch_mp_args = "{magpie} -out {plat}/mpBvh00/dlc_patch.rpf -rpf_postfixes _bvh -path_from dev_gen9_bvh_rebuild -log_adds -logfile {plat}/mpBvh00/dlc_patch.txt -keep_loose -ignore ignore.txt -patterns *.{letter}bvh,*.{letter}bvhd X:/gta5/titleupdate/dev_gen9_bvh_rebuild/dlc_patch/mp* -X:/gta5/titleupdate/dev_gen9_bvh_rebuild/dlc_patch/mp*/{ignore_plat}/*" mp_content_args = "{magpie} -out {plat}/mpBvh00/content.rpf -path_from dev_gen9_bvh_rebuild -log_adds -logfile {plat}/mpBvh00/content.txt -patterns */content.xml -extract_rpfs -ignore ignore.txt -keep_loose X:/gta5/titleupdate/dev_gen9_bvh_rebuild/{plat}/dlcPacks/mp* X:/gta5/build/dev_gen9_bvh_rebuild/{plat}/dlcPacks/mp* -X:/gta5/build/dev_gen9_bvh_rebuild/{ignore_plat}/* -X:/gta5/titleupdate/dev_gen9_bvh_rebuild/{ignore_plat}/*" bvh_packs = ["_base_mp", "_update_mp"] pack_map = {"_base_mp": "build", "_update_mp": "titleupdate"} letter_map = {"ps5": "p", "xbsx": "z"} ignore_map = {"ps5": "xbsx", "xbsx": "ps5"} mp_dlc_src = "{plat}\\mpBvh00\\{pack}\\{plat}\\dlcPacks\\" dlc_patch_src = "X:\\gta5\\titleupdate\\dev_gen9_sga\\dlc_patch\\" dlc_dest = "X:\\gta5\\{pack_map}\\dev_gen9_sga\\{plat}\\dlcPacks\\{pack}\\" def multiprocess_log(log_name): logger = multiprocessing.get_logger() if not len(logger.handlers): logger.setLevel(log.INFO) formatter = log.Formatter(\ '[%(asctime)s| %(levelname)s| %(processName)s] %(message)s') handler = log.FileHandler(log_name, mode="w") handler.setFormatter(formatter) logger.addHandler(handler) return logger def SinglePlayer(plats, log_name, changelist=None): log = multiprocess_log(log_name) wait_for = [] # Gnerate sp log.info("Generating SP") for plat in plats: # Handle patchBvh00 log.info(sp_args.format(magpie=magpie, plat=plat, letter=letter_map[plat], ignore_plat=ignore_map[plat])) wait_for.append(subprocess.Popen(sp_args.format(magpie=magpie, plat=plat, letter=letter_map[plat], ignore_plat=ignore_map[plat]), shell=True, creationflags=subprocess.DETACHED_PROCESS)) while wait_for: wait_for.pop().wait() # Generate content.xml log.info("Generating content.xml") for plat in plats: log.info(sp_content_args.format(plat=plat)) wait_for.append(subprocess.Popen(sp_content_args.format(plat=plat), shell=True, creationflags=subprocess.DETACHED_PROCESS)) while wait_for: wait_for.pop().wait() log.info("Generating patchBvh00/dlc.rpf") for plat in plats: log.info(sp_patch_args.format(magpie=magpie, plat=plat, letter=letter_map[plat])) wait_for.append(subprocess.Popen(sp_patch_args.format(magpie=magpie, plat=plat, letter=letter_map[plat]), shell=True, creationflags=subprocess.DETACHED_PROCESS)) while wait_for: wait_for.pop().wait() log.info("Moving patchBvh00/dlc.rpf") for plat in plats: for i in range(0, 16): patch_dest = dlc_dest.format(pack_map="titleupdate", plat=plat, pack="patchBVH00") patch_src = f"{plat}/patchBvh00/dlc.rpf" if i: patch_dest = os.path.join(patch_dest, f"dlc{i}.rpf") patch_src = f"{plat}/patchBvh00/dlc{i}.rpf" else: patch_dest = os.path.join(patch_dest, "dlc.rpf") if os.path.exists(patch_src): log.info(f"moving {patch_src} -> {patch_dest}") try: if changelist and os.path.exists(patch_dest): log.info(subprocess.check_output(f"p4 edit -c {changelist} {patch_dest}")) shutil.move(patch_src, patch_dest) if changelist: log.info(subprocess.check_output(f"p4 add -c {changelist} {patch_dest}")) except Exception as e: log.error(e) print(e) else: break def MultiPlayer(plats, log_name, changelist=None, no_xml=False): log = multiprocess_log(log_name) wait_for = [] log.info("Generating MP dlc_patch") for plat in plats: # Handle _dlc_patch log.info(dlc_patch_mp_args.format(magpie=magpie, plat=plat, letter=letter_map[plat], ignore_plat=ignore_map[plat])) wait_for.append(subprocess.Popen(dlc_patch_mp_args.format(magpie=magpie, plat=plat, letter=letter_map[plat], ignore_plat=ignore_map[plat]), shell=True, creationflags=subprocess.DETACHED_PROCESS)) while wait_for: wait_for.pop().wait() log.info("Moving dlc_patch _bvh.rpfs") for plat in plats: log.info("finding "+f"{plat}/mpBvh00/_dlc_patch/**/*.rpf") rpfs = glob.glob(f"{plat}/mpBvh00/_dlc_patch/**/*.rpf", recursive=True) for rpf in rpfs: stripped_rpf = rpf.replace(f"{plat}/mpBvh00/_dlc_patch\\dlc_patch\\", "") move_dest = os.path.join(dlc_patch_src, stripped_rpf) log.info(f"moving {rpf} -> {move_dest}") if changelist and os.path.exists(move_dest): log.info(subprocess.check_output(f"p4 edit -c {changelist} {move_dest}")) shutil.move(rpf, move_dest) if changelist: log.info(subprocess.check_output(f"p4 add -c {changelist} {move_dest}")) log.info("Generating MP base, update, and content") for plat in plats: log.info(base_mp_args.format(magpie=magpie, plat=plat, letter=letter_map[plat])) log.info(update_mp_args.format(magpie=magpie, plat=plat, letter=letter_map[plat])) wait_for.append(subprocess.Popen(base_mp_args.format(magpie=magpie, plat=plat, letter=letter_map[plat], ignore_plat=ignore_map[plat]), shell=True, creationflags=subprocess.DETACHED_PROCESS)) wait_for.append(subprocess.Popen(update_mp_args.format(magpie=magpie, plat=plat, letter=letter_map[plat], ignore_plat=ignore_map[plat]), shell=True, creationflags=subprocess.DETACHED_PROCESS)) if not no_xml: log.info(mp_content_args.format(magpie=magpie, plat=plat, ignore_plat=ignore_map[plat])) wait_for.append(subprocess.Popen(mp_content_args.format(magpie=magpie, plat=plat, ignore_plat=ignore_map[plat]), shell=True, creationflags=subprocess.DETACHED_PROCESS)) while wait_for: wait_for.pop().wait() log.info("Updating content.xml and moving dlc_update/_bvh rpfs") for plat in plats: for pack in bvh_packs: dlc_src = mp_dlc_src.format(pack=pack, plat=plat) if os.path.exists(dlc_src): log.info(f"Cleaning {plat} subpacks.") dlc_packs = os.listdir(dlc_src) for dlc_name in dlc_packs: clean_path = os.path.join(dlc_src, dlc_name, plat) if os.path.exists(clean_path): log.info("Removing: "+clean_path) try: shutil.rmtree(clean_path) except Exception as E: pass rpf = os.path.join(dlc_src, dlc_name, "dlc_update.rpf") content_xml = rpf.replace(pack, "_content").replace("dlc_update.rpf", "content.xml") dupe_xml = os.path.join(dlc_patch_src, content_xml.split("dlcPacks\\")[1]) if os.path.exists(dupe_xml): content_xml = dupe_xml for i in range(0, 16): if i: rpf = os.path.join(dlc_src, dlc_name, f"dlc{i}_update.rpf") if os.path.exists(rpf): if not no_xml: log.info("Updating content.xml for "+rpf) log.info(content_xml+" -> "+dupe_xml) lines = subprocess.check_output(rpf_ls+" "+rpf).decode().splitlines() lines = [l.strip() for l in lines if l.endswith(".rpf")] lines = [l.replace(plat, "%PLATFORM%") for l in lines] copy_insert = ",".join([",".join([l.replace("_bvh.rpf", ".rpf"),l]) for l in lines]) try: if changelist and os.path.exists(dupe_xml): log.info(subprocess.check_output(f"p4 edit -c {changelist} {dupe_xml}")) log.info(f"python content_dup.py -i {content_xml} -o {dupe_xml} -c {copy_insert}") if changelist: log.info(subprocess.check_output(f"p4 add -c {changelist} {dupe_xml}")) out = subprocess.check_output(f"python content_dup.py -i {content_xml} -o {dupe_xml} -c {copy_insert}").decode() log.info("=========================================================================\n") log.info(content_xml+"\n") log.info("=========================================================================\n") log.info(out.replace("\r\n", "\n")) except subprocess.CalledProcessError as e: log.info(e) breakpoint() break # move_dest = dlc_dest.format(pack_map=pack_map[pack], plat=plat, pack=dlc_name) move_dest = dlc_dest.format(pack_map="titleupdate", plat=plat, pack=dlc_name) os.makedirs(os.path.dirname(move_dest), exist_ok=True) if i: move_dest = os.path.join(move_dest, f"dlc{i}_update.rpf") else: move_dest = os.path.join(move_dest, "dlc_update.rpf") log.info(f"Moving: {rpf} -> {move_dest}") try: if changelist and os.path.exists(move_dest): log.info(subprocess.check_output(f"p4 edit -c {changelist} {move_dest}")) shutil.move(rpf, move_dest) if changelist: log.info(subprocess.check_output(f"p4 add -c {changelist} {move_dest}")) except Exception as e: log.error(e) else: log.info(rpf+ " doesn't exist") break pass else: log.info(dlc_src +" doesn't exist.") def main(): global plats parser = argparse.ArgumentParser(prog="BVH Pack Builder") parser.add_argument("-sp", "--singleplayer", action="store_true", help="Build the single player patch") parser.add_argument("-mp", "--multiplayer", action="store_true", help="Build the multiplayer patch") parser.add_argument("-nxml", "--no_xml", action="store_true", help="Don't change dlc_patch content.xml files") parser.add_argument("-p", "--platform", choices=["xbsx", "ps5"], help="Build a specific platform") parser.add_argument("-l", "--logfile", help="log file name") parser.add_argument("-c", "--changelist", help="files will be added to the given changelist.") args = parser.parse_args() cur_dir = os.path.dirname(__file__) os.chdir(cur_dir) log_name = "log.txt" if args.logfile: log_name = args.logfile log.basicConfig(level=log.DEBUG, format="%(message)s") start = timeit.default_timer() log.info("".ljust(64, "-")) log.info("Starting BVH Builder") log.info("Logging to: "+os.path.abspath(log_name)) log.info("Settings: ") for k, v in vars(args).items(): log.info("\t"+k+": "+str(v)) wait_for = [] plats = ["ps5", "xbsx"] log.info("Waiting for processes to complete.") # Do work if args.platform: plats = [args.platform] if args.singleplayer: wait_for.append(multiprocessing.Process(target=SinglePlayer, name="SinglePlayer", args=[plats, log_name, args.changelist])) if args.multiplayer: wait_for.append(multiprocessing.Process(target=MultiPlayer, name="Multiplayer", args=[plats, log_name, args.changelist, args.no_xml])) for w in wait_for: w.start() for w in wait_for: w.join() end = timeit.default_timer() # time formating via format specifiers # https://docs.python.org/3.8/library/string.html#formatspec s = end-start total_time = f"{(s/60)/60:0>2.0f}:{s/60:0>2.0f}:{s%60:0>2.2f}" log.info("".ljust(64, "-")) log.info("Took "+total_time) log.info("Work complete have a nice day. ") if __name__ == "__main__": main()