using P4API; using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Reflection; using System.Reflection.Emit; using System.Security.Cryptography; using System.Text; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using System.Windows.Media; using System.Xml.Serialization; using System.Xml.Linq; using System.Xml.XPath; using System.Runtime.CompilerServices; using System.Diagnostics; namespace OozyBuild { /****************************************************************************************************************** ** ******************************************************************************************************************/ public static class ProjectExtension { public static void AddLog( this OozyBuild.Project project, string log, Brush colour = null ) { if( project.log.Add != null ) { project.log.Add( log ); } } public static void AddWarning( this OozyBuild.Project project, string log ) { if( project.log.AddWarning != null ) { project.log.AddWarning( log ); } } public static void AddError( this OozyBuild.Project project, string log ) { if( project.log.AddError != null ) { project.log.AddError( log ); } } public static void AddSuccess( this OozyBuild.Project project, string log ) { if( project.log.AddSuccess != null ) { project.log.AddSuccess( log ); } } public static void RewindLog( this OozyBuild.Project project, int count ) { if( project.log.RewindLog != null ) { project.log.RewindLog.Invoke( count ); } } } /****************************************************************************************************************** ** ******************************************************************************************************************/ public class BranchStatus { public int lastChangelist; public int syncedChangelist; public List current = new List(); public List sinceNightbuild = new List(); } /****************************************************************************************************************** ** ******************************************************************************************************************/ public class BranchSyncInfo { public bool storeChanges; public List syncPaths = new List(); } /****************************************************************************************************************** ** ******************************************************************************************************************/ public class GTA5Extension { public Project project; public P4 currentP4; public string p4root; public bool nightbuild; public bool package; protected virtual Dictionary branchPaths { get; set; } /****************************************************************************************************************** ** ******************************************************************************************************************/ public GTA5Extension() { } /****************************************************************************************************************** ** Sets up an instance of p4 ******************************************************************************************************************/ protected P4 SetupP4([CallerFilePath] string scriptFile = @"RSG.Leigh.Bird.Oozy.unknown" ) { P4 p4 = currentP4; if( p4 != null ) { if( p4.rep != null && p4.rep.Connection != null && p4.rep.Connection.connectionEstablished() ) { try { p4.rep.Connection.getP4Server().RunCommand( "help", 0, false, null, 0 ); return p4; } catch( Exception ) { } try { currentP4 = null; p4.Disconnect(); } catch( Exception ) { } } } string ini = project.p4IniFile; if( !File.Exists( ini ) ) { project.AddLog( "p4ini \"" + ini + "\" not found, aborting SetupP4" ); return null; } p4 = new P4( ini ); if( !string.IsNullOrEmpty( scriptFile ) ) { p4.ProgramName = @"RSG.Leigh.Bird.Oozy." + Path.GetFileNameWithoutExtension( scriptFile ); p4.ProgramVersion = "1.0.0"; } project.AddLog( "Connecting to " + p4.Port ); p4.OnConnected += ( P4 p ) => { project.AddLog( "Connected" ); project.SetupSCCOutput( p4 ); }; if( !p4.Connect() ) { project.AddError( "P4: Failed to connect host: " + p4.Port + ", user: " + p4.User + ", workspace: " + p4.Workspace ); return null; } p4root = p4.Root(); if( string.IsNullOrEmpty( p4root ) ) { Console.WriteLine( "p4 not connected propery, will try again later" ); return null; } currentP4 = p4; return p4; } /****************************************************************************************************************** ** ******************************************************************************************************************/ protected BuildChange GetBuildInfo( Perforce.P4.Changelist changelist, List buildInfo, List users ) { string user = changelist.OwnerName; BuildInfo build = buildInfo.FirstOrDefault( b => string.Compare( b.userName, user, true ) == 0 ); if( build == null ) { build = new BuildInfo(); build.userName = user; buildInfo.Add( build ); users.Add( user ); } BuildChange bc = new BuildChange(); bc.id = changelist.Id; bc.description = changelist.Description; build.changes.Add( bc ); return bc; } /****************************************************************************************************************** ** ******************************************************************************************************************/ protected bool SetupUserInfo( P4 p4, List buildInfo, List users ) { if( users.Count > 0 ) { var p4Users = p4.Users( users ); foreach( var user in p4Users ) { BuildInfo info = buildInfo.FirstOrDefault( a => a.userName == user.Id ); if( info != null ) { info.email = user.EmailAddress; info.fullName = user.FullName; } } } return true; } /****************************************************************************************************************** ** ******************************************************************************************************************/ protected void RemoveProjGen( P4 p4, bool submit ) { Perforce.P4.Changelist pgChangelist = p4.FindChangelist( "ProjectGenerator v3.2.0.0" ); if( pgChangelist != null ) { p4.RevertUnchangedFiles( pgChangelist.Id ); List clFiles = p4.GetFilesInChangelist( pgChangelist.Id ); if( clFiles.Count > 0 ) { if( submit ) { p4.Submit( pgChangelist ); } else { p4.Revert( clFiles.ToFileSpecs() ); } } } } /****************************************************************************************************************** ** Given (depot) files, lastChangelist and currentChangelist, generate a list of local paths and return a list of user/changelist/file information ** Much simpler to go from last CL to latest CL, but with auto builders constantly checking in on multiple projects that becomes really bad ** to managed. ** So go through each synced file and get its history between last and current cl and work out changelists from there ******************************************************************************************************************/ protected List GetBuildInfoFromFiles( P4 p4, List buildPaths, IList files, int lastChangelist, int currentChangelist ) { List buildInfo = new List(); List users = new List(); project.AddLog( "Synced:" ); foreach( Perforce.P4.FileSpec file in files ) { project.AddLog( "\t" + file.DepotPath.Path ); } Perforce.P4.VersionSpec changelistVersions = new ChangelistRange( lastChangelist + 1, currentChangelist ); buildPaths.AddRange( files.Select( a => a.LocalPath.Path ) ); Dictionary> checkedIn = new Dictionary>(); Dictionary> histories = p4.FileLog( files, changelistVersions ); Dictionary foundClsFor = new Dictionary(); foreach( var pair in histories ) { foundClsFor[pair.Key.DepotPath.Path] = true; foreach( var history in pair.Value ) { //should never be false due to the changelist range if( (history.ChangelistId > lastChangelist && history.ChangelistId <= currentChangelist) ) { List clFiles; if( !checkedIn.TryGetValue( history.ChangelistId, out clFiles ) ) { clFiles = new List(); checkedIn[history.ChangelistId] = clFiles; } clFiles.Add( new Perforce.P4.FileSpec( new Perforce.P4.DepotPath( pair.Key.DepotPath.Path ), changelistVersions ) ); } } } List extraFiles = new List(); foreach( var fs in files ) { bool done; if( !foundClsFor.TryGetValue( fs.DepotPath.Path, out done ) ) { foundClsFor[fs.DepotPath.Path] = true; extraFiles.Add( fs ); } } if( extraFiles.Count > 0 ) { IList metas = p4.FStat( extraFiles ); if( metas.Count > 0 ) { foreach( var meta in metas ) { List clFiles; if( !checkedIn.TryGetValue( meta.HeadChange, out clFiles ) ) { clFiles = new List(); checkedIn[meta.HeadChange] = clFiles; } clFiles.Add( new Perforce.P4.FileSpec( new Perforce.P4.DepotPath( Perforce.P4.PathSpec.EscapePath( meta.DepotPath.Path ) ), new Perforce.P4.ChangelistIdVersion( currentChangelist ) ) ); } } } if( checkedIn.Count > 0 ) { //Same file can exist in multiple checkins, so remove duplicates IList clsToGet = checkedIn.Select( a => a.Value[0] ).Distinct().ToList(); IList changes = p4.GetChangelistsAtRevision( clsToGet, 0 ).GroupBy( a => a.Id ).Select( b => b.First() ).ToList(); foreach( var change in changes ) { List missedFiles = new List(); BuildChange bc = GetBuildInfo( change, buildInfo, users ); List clFiles = p4.GetFilesInChangelist( change.Id ); foreach( Perforce.P4.FileMetaData spec in clFiles ) { Perforce.P4.FileSpec file = files.FirstOrDefault( a => a.DepotPath.Path == spec.DepotPath.Path ); if( file != null ) { bc.files.Add( file.LocalPath.Path ); } else { Console.WriteLine( spec.DepotPath.Path + " in CL " + change.Id + " but not synced" ); missedFiles.Add( new Perforce.P4.FileSpec( spec.DepotPath, new Perforce.P4.Revision( spec.HeadRev ) ) ); } } if( missedFiles.Count > 0 ) { IList missed = p4.Where( missedFiles ); if( !missed.Empty() ) { foreach( var spec in missed ) { bc.files.Add( spec.LocalPath.Path ); files.Add( spec ); } } } } } if( !SetupUserInfo( p4, buildInfo, users ) ) { return null; } return buildInfo; } /****************************************************************************************************************** ** Get all the paths to sync for all branches, or a specific branch ******************************************************************************************************************/ protected List GetBranchPaths( string singleBranch = "" ) { List syncPaths = new List(); if( !string.IsNullOrEmpty( singleBranch ) ) { BranchSyncInfo branch; if( !branchPaths.TryGetValue( singleBranch, out branch ) ) { project.AddError( "Failed to find paths for branch " + singleBranch + ", it doesn't appear to be setup" ); project.PostBuildError( "Failed to find paths branch " + singleBranch + ", it doesn't appear to be setup" ); return null; } syncPaths = branch.syncPaths; } else { foreach( var dirs in branchPaths.Select( a => a.Value.syncPaths ) ) { syncPaths.AddRange( dirs ); } } //a branch might reference the same paths as another branch, so remove duplicates syncPaths = syncPaths.Distinct( StringComparer.OrdinalIgnoreCase ).ToList(); return syncPaths; } /****************************************************************************************************************** ** Set the synced changelist number for all or a specified branch ******************************************************************************************************************/ void SetBranchVersion( int cl, string singleBranch = "" ) { SerializableDictionary branchStatuses = (SerializableDictionary)project.customSettings["branchStatus"]; if( branchStatuses != null ) { if( !string.IsNullOrEmpty( singleBranch ) ) { BranchStatus branch; if( branchStatuses.TryGetValue( singleBranch, out branch ) ) { branch.lastChangelist = branch.syncedChangelist; branch.syncedChangelist = cl; } } else { foreach( var pair in branchStatuses ) { pair.Value.lastChangelist = pair.Value.syncedChangelist; pair.Value.syncedChangelist = cl; } } } } /****************************************************************************************************************** ** Sync relevant paths for all or a single branch. Specify a changelist or it'll just use latest ******************************************************************************************************************/ protected IList SyncBranches( P4 p4, string singleBranch = "", int currentChangelist = -1, bool autoResolve = true, string labelName = "" ) { if( p4 == null ) { p4 = SetupP4(); if( p4 == null ) { return null; } } List syncPaths = GetBranchPaths( singleBranch ); if( syncPaths == null ) { return null; } Perforce.P4.VersionSpec version = null; if( currentChangelist == -1 ) { IList latest = p4.GetChangelists( 1, Perforce.P4.ChangeListStatus.Submitted ); if( latest.Count > 0 ) { currentChangelist = latest[0].Id; } } if( currentChangelist != -1 ) { version = new Perforce.P4.ChangelistIdVersion( currentChangelist ); SetBranchVersion( currentChangelist, singleBranch ); } else { //Something has to have failed by here and it really shouldn't happen with the previous p4 checks happening but just in case, the show must go on version = new P4API.HeadChangelist(); } IList syncFiles = new List(); IList tagFiles = new List(); foreach( string path in syncPaths ) { Perforce.P4.FileSpec root = p4.GetFileSpecFromPath( Path.Combine( p4root, path ) ); if( root == null ) { project.AddError( "P4: Problem finding root paths for " + path ); project.PostBuildError( "P4: Problem finding root paths for " + path ); return null; } if( !string.IsNullOrEmpty( labelName ) ) { Perforce.P4.FileSpec tagSpec = new Perforce.P4.FileSpec( root ); if( root.Version == null ) { tagSpec.Version = version; } else { tagSpec.Version = root.Version; root.Version = new Perforce.P4.LabelNameVersion( labelName ); } tagFiles.Add( tagSpec ); } if( root.Version == null ) { root.Version = version; } syncFiles.Add( root ); } bool preview = project.builder.test; if( !string.IsNullOrEmpty( labelName ) ) { p4.Tag( labelName, tagFiles, preview: preview ); } IList files = new List(); if( nightbuild || package ) { project.PostBuildMessage( "Syncing..." ); } #if true files = p4.Sync( syncFiles, preview: preview, autoResolve: autoResolve, clobberWritable: true ); #else { //handy test code string tstFile = @"//depot/gta5/src/dev_gen9_sga/game/script/commands_debug.cpp"; files.Add( p4.Where( tstFile ) ); files[0].Version = new Perforce.P4.Revision( 8 ); } #endif return files; } /****************************************************************************************************************** ** ******************************************************************************************************************/ public int StartProcess( string fullPathToExe ) { project.AddLog( "Starting Process: " + fullPathToExe ); string exeName = Path.GetFileNameWithoutExtension( fullPathToExe ); project.AddLog( "Checking if any process with name '" + exeName + "' is running..." ); bool isRunning = Process.GetProcessesByName( exeName ).Any(); if( isRunning ) { project.AddLog( "Already running!" ); return 0; } project.AddLog( "Not currently running - Starting " + fullPathToExe + "..." ); Process.Start( fullPathToExe ); return 0; } /****************************************************************************************************************** ** Called from .nom, parses the output from a batch file sync, hopefully deprecated but useful as a ref. ******************************************************************************************************************/ protected List buildPaths; protected List buildInfo; public static bool HasProperty( dynamic settings, string name ) { return settings.GetType().GetProperty( name ) != null || settings.GetType().GetField( name ) != null; } /****************************************************************************************************************** ** ******************************************************************************************************************/ public int ParseCLInfo( string branchName ) { dynamic settings = project.customSettings; if( !HasProperty( settings, "branchStatus" ) ) { Console.WriteLine( "Settings " + project.customSettings.GetType().Name + " doesn't have a property 'SerializableDictionary branchStatus { get; set; }'" ); return 1; } SerializableDictionary branchStatus = settings.branchStatus; BranchStatus status; if( !branchStatus.TryGetValue( branchName, out status ) ) { Console.WriteLine( "Failed to find branch \"" + branchName + "\"" ); return 1; } P4 p4 = SetupP4(); if( p4 == null ) { return 1; } Console.WriteLine( "Looking for BuildEntrty for \"Sync Latest Data for " + branchName + "\"" ); BuildEntry entry = project.builder.allBuilds.FirstOrDefault( a => a.displayName == "Sync Latest Data for " + branchName ); if( entry == null ) { Console.WriteLine( "Failed" ); return 1; } List log = entry.log.ToList(); IList files = new List( log.Count ); int clId = 0; int currentChangelist = -1; foreach( string file in log ) { if( clId == 0 ) { Match cl = Regex.Match( file, @"Syncing Latest Data to\s*(?[\d]+)", RegexOptions.IgnoreCase ); if( cl.Success ) { if( int.TryParse( cl.Groups["cl"].Value, out clId ) && clId > 0 ) { currentChangelist = branchStatus[branchName].syncedChangelist = clId; Console.WriteLine( "Parsed currentChangelist=" + currentChangelist ); } } } Match match = Regex.Match( file, @"^(?[\s\S]+)#(?\d+)\s*-\s*\w+\s*(?[\s\S]+)" ); if( match.Success ) { string localPath = match.Groups["localPath"].Value; if( localPath.StartsWith( p4root, StringComparison.OrdinalIgnoreCase ) ) { Perforce.P4.FileSpec fileSpec = new Perforce.P4.FileSpec(); fileSpec.DepotPath = new Perforce.P4.DepotPath( match.Groups["depotPath"].Value ); fileSpec.LocalPath = new Perforce.P4.LocalPath( localPath ); files.Add( fileSpec ); } } } if( currentChangelist == -1 ) { Console.WriteLine( "Failed to determine the changelist from the log" ); return 1; } status.syncedChangelist = currentChangelist; if( !files.Empty() ) { buildPaths = new List(); buildInfo = GetBuildInfoFromFiles( p4, buildPaths, files, status.lastChangelist, currentChangelist ); AddBuildInfo( buildInfo ); } return 0; } /****************************************************************************************************************** ** ******************************************************************************************************************/ public void AddBuildInfo( StringBuilder sb, List changes, List excludes = null ) { foreach( BuildInfo info in changes ) { foreach( BuildChange change in info.changes ) { if( excludes != null ) { if( excludes.Exists( a => a.changes.Exists( b => b.id == change.id ) ) ) { continue; } } string[] descriptionLines = change.description.Replace( "\r\n", "\n" ).Split( '\n' ); string description = ""; foreach( string line in descriptionLines ) { description += "\t" + line + "\n"; } sb.Append( "**************************************************\n" + change.id.ToString() + " " + info.fullName + "\n" + description + "\n" ); } } } /****************************************************************************************************************** ** ******************************************************************************************************************/ public void AddBugstarInfo( StringBuilder sb, List changes, List excludes = null ) { foreach( BuildInfo info in changes ) { foreach( BuildChange change in info.changes ) { if( excludes != null ) { if( excludes.Exists( a => a.changes.Exists( b => b.id == change.id ) ) ) { continue; } } foreach( Match m in Regex.Matches( change.description, @"bugstar\s*:\s*(?\s*\d+)", RegexOptions.IgnoreCase ) ) { if( m.Success ) { sb.Append( "url:bugstar:" + m.Groups["bugstar"].Value + " - " + info.fullName + "\n" ); } } } } } /****************************************************************************************************************** ** ******************************************************************************************************************/ public void AddBuildInfo( List current ) { dynamic settings = project.customSettings; if( !HasProperty( settings, "branchStatus" ) ) { Console.WriteLine( "Settings " + project.customSettings.GetType().Name + " doesn't have a property 'SerializableDictionary branchStatus { get; set; }'" ); } SerializableDictionary branchStatus = settings.branchStatus; //go through known branches and their sync paths foreach( var pair in branchPaths ) { string branchName = pair.Key; if( !pair.Value.storeChanges ) { //flag may have changed, ensure old information is cleared branchStatus[branchName].current.Clear(); branchStatus[branchName].sinceNightbuild.Clear(); continue; } BranchStatus status = branchStatus[branchName]; List currentInfos = status.current; List nightbuildInfos = status.sinceNightbuild; foreach( string p4Path in pair.Value.syncPaths ) { string path = Path.Combine( p4root, p4Path ).Replace( "...", "*" ).Replace( Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar ); path = Program.WildCardToRegular( path ); //go through the changelist info and check each related file to see if is relevant to this branch foreach( BuildInfo buildInfo in current ) { BuildInfo currentInfo = currentInfos.FirstOrDefault( a => a.userName == buildInfo.userName ); BuildInfo nightbuildInfo = nightbuildInfos.FirstOrDefault( a => a.userName == buildInfo.userName ); foreach( BuildChange buildChange in buildInfo.changes ) { foreach( string file in buildChange.files ) { if( Regex.IsMatch( file, path, RegexOptions.IgnoreCase ) ) { //Have a match //add a BuildInfo if it doesn't already exist if( currentInfo == null ) { currentInfo = new BuildInfo() { userName = buildInfo.userName, email = buildInfo.email, fullName = buildInfo.fullName }; currentInfos.Add( currentInfo ); } if( nightbuildInfo == null ) { nightbuildInfo = new BuildInfo() { userName = buildInfo.userName, email = buildInfo.email, fullName = buildInfo.fullName }; nightbuildInfos.Add( nightbuildInfo ); } //As we are going through files, it's possible to get the same CL more than once //Create a new version of BuildChange containing minimal info, don't need the list of files BuildChange storedBuildChange = new BuildChange() { id = buildChange.id, description = buildChange.description }; if( !currentInfo.changes.Exists( a => a.id == storedBuildChange.id ) ) { currentInfo.changes.Add( storedBuildChange ); } if( !nightbuildInfo.changes.Exists( a => a.id == storedBuildChange.id ) ) { nightbuildInfo.changes.Add( storedBuildChange ); } } } } } } } } /****************************************************************************************************************** ** Writes out bugs.log and changes.log to show changes since the last build ******************************************************************************************************************/ public int WriteCLInfo( string branchName, string outDir ) { //Generate bugs.log and changes.log dynamic settings = project.customSettings; if( !HasProperty( settings, "branchStatus" ) ) { Console.WriteLine( "Settings " + project.customSettings.GetType().Name + " doesn't have a property 'SerializableDictionary branchStatus { get; set; }'" ); } SerializableDictionary branchStatus = settings.branchStatus; StringBuilder sb = new StringBuilder(); StringBuilder changes = new StringBuilder(); StringBuilder bugs = new StringBuilder(); outDir = Path.Combine( project.enginePath, outDir ); if( nightbuild ) { project.PostBuildMessage( branchName + "has " + branchStatus[branchName].sinceNightbuild.Count + " changelists since last nightbuild" ); changes.Append( "Changelist since last nightbuild:\n" ); AddBuildInfo( changes, branchStatus[branchName].sinceNightbuild ); bugs.Append( "Bugs since last nightbuild:\n" ); AddBugstarInfo( bugs, branchStatus[branchName].sinceNightbuild ); sb.Append( bugs ); sb.Append( changes ); } else if( package ) { project.PostBuildMessage( branchName + "has " + branchStatus[branchName].current.Count + " changelists since last package, and there are " + branchStatus[branchName].sinceNightbuild.Count + " changelists since last nightbuild" ); changes.Append( "Changelist since last package:\n" ); AddBuildInfo( changes, branchStatus[branchName].current ); changes.Append( "\nChangelist since last nightbuild:\n" ); AddBuildInfo( changes, branchStatus[branchName].sinceNightbuild, branchStatus[branchName].current ); bugs.Append( "Bugs since last package:\n" ); AddBugstarInfo( bugs, branchStatus[branchName].current ); bugs.Append( "\nBugs since last nightbuild:\n" ); AddBugstarInfo( bugs, branchStatus[branchName].sinceNightbuild, branchStatus[branchName].current ); sb.Append( bugs ); sb.Append( changes ); } else { sb.Append( "No idea what's gone on here. Hopefully I'll never see this message" ); } string bugsFile = Path.Combine( outDir, "bugs.log" ); Util.MakeDirectory( Path.GetDirectoryName( bugsFile ) ); try { File.WriteAllText( bugsFile, bugs.ToString() ); } catch( Exception ex ) { sb.Append( "Failed to write to " + bugsFile + "\n" + App.LogException( ex ) ); } string changesFile = Path.Combine( outDir, "changes.log" ); Util.MakeDirectory( Path.GetDirectoryName( changesFile ) ); try { File.WriteAllText( changesFile, changes.ToString() ); } catch( Exception ex ) { sb.Append( "Failed to write to " + changesFile + "\n" + App.LogException( ex ) ); } project.AddLog( sb.ToString() ); return 0; } /****************************************************************************************************************** ** OUTPUT ERROR CAPTURING ******************************************************************************************************************/ public string GetProjgenErrors( string log, int maxErrors ) { string[] lines = log.Split( new[] { "\r\n", "\r", "\n" }, StringSplitOptions.RemoveEmptyEntries ); int errors = 0; int endLine = -1; int startLine = -1; for( int i = lines.Length - 1; i >= 0; --i ) { string line = lines[i]; if( endLine == -1 ) { if( line.Contains( "[error]" ) ) { endLine = i; continue; } } else if( startLine == -1 ) { if( !line.Contains( "[error]" ) ) { startLine = i + 1; break; } } } StringBuilder sb = new StringBuilder( 1024 ); for( int i = startLine; i < endLine && i < lines.Length && (maxErrors <= 0 || errors < maxErrors); ++i, ++errors ) { sb.Append( lines[i] ); sb.Append( "\r\n" ); } return sb.ToString(); } public string GetSlnErrors( string log, int maxErrors ) { return NomProject.CaptureSlnErrors( log.Split( new[] { "\r\n", "\r", "\n" }, StringSplitOptions.RemoveEmptyEntries ), maxErrors ); } } }