# # File:: unity_build.rb # # Description:: Supports a unity build side by side with project builder. # - Loads .unity config file. # - Creates new cpp files ( unity files ) # - 'Excludes' files from build inside the project format - this is required to be set for all platforms and build configs. # - Creates CL for submission to p4 of new unity files. # - Tidies p4 of old unity files as it goes. # - Monkey patches into project builder classes. # - Tests by importing a non unity VS2008 vcproj. # - handles unity prologue and epilogue about each included file # # Author:: Derek Ward # Date:: 31 January 2011 # # #----------------------------------------------------------------------------- # Uses #----------------------------------------------------------------------------- require "fileutils" require 'pipeline/os/path' require 'pipeline/os/getopt' require 'pipeline/log/log' require "pipeline/coding/projbuild/generators/internal" require "pipeline/coding/projbuild/generators/vs2008" require "pipeline/coding/projbuild/generators/rageprojbuilder" require "pipeline/coding/projbuild" #----------------------------------------------------------------------------- # Implementation #----------------------------------------------------------------------------- module Pipeline module ProjBuild ##################################################### # A unity file # represents the definition of a new file added to # the project to compile larger units of compilation. # - serialisable. # - for cpp files class UnityFile FOLDER_TOKEN = "_folder_" EMPTY_REGEXP = "^(((\.*|\w:)\/)*)(#{FOLDER_TOKEN})\/(.*)((\\.cpp)|(\\.c))$" #"^(\.*|\w:)\/(#{FOLDER_TOKEN})\/(((?!exclusions_can\.|go_here\.).)*)(\.)(cpp)$" EMPTY_REGEXP_OPTIONS = "i" UNITY_BUILD_DEFINE = "__UNITYBUILD" UNITY_PREFIX = "_unity_" attr_reader :name # unique identifier. attr_reader :unity_folder # the folder where the unity file resides. attr_reader :project_folder # the project folder the unity file is being built for. attr_reader :project # the project the unity file is being built for. attr_reader :regexp # regexp filter for files. attr_reader :files # a list of files for unity inclusion. def initialize( name, unity_folder, project, project_folder ) @name = name @unity_folder = unity_folder @project = project @project_folder = project_folder @files = [] @regexps = [] end # relative paths are used to make the project portable. def relative_path( path ) rel_path = "../" # because the unity files are in their own folder, this takes us up to the folder of the project file - the unity folder is always relative to the project. # now calculate where the source files would be located relative to this path # this is the project folder minus the full path proj_folders = @project_folder.split("/") path_folders = path.split("/") same = true remaining_path,common_path = "","" proj_folders.each_with_index do |proj_folder, idx| if (same and (proj_folder == path_folders[idx]) ) common_path += "#{proj_folder}/" else same = false remaining_path += "#{proj_folder}/" end end # count the folders left in path, add ../ to relative path for each. remaining_path.scan("/").length.times { rel_path += "../" } # remove the common path and add the relative path. path.sub( common_path, rel_path ) end def include( path ) @files << path end def filename( ) OS::Path.set_downcase_on_normalise(false) OS::Path::combine( @unity_folder, "#{UNITY_PREFIX}#{@project}_#{@name}.cpp" ) end def write( comment, folder, logues ) File.open(filename(), 'w') do |f| f.write(to_cpp( comment, folder, logues )) end end def prologue_cpp( ) #"\n\#pragma message( \"Compiling \" __FILE__ \" \" __TIME__)" "" end def epilogue_cpp( ) #"\n\#pragma message( \"Compiled \" __FILE__ \" \" __TIME__)" "" end def prologue_each_cpp( folder ) file = UnityBuild::PROLOGUE_FILENAME "\n\#include \"#{file}\"" end def epilogue_each_cpp( folder ) file = UnityBuild::EPILOGUE_FILENAME "\n\#include \"#{file}\"" end def to_cpp( comment, folder, logues ) define = "\n\#define #{UNITY_BUILD_DEFINE}" includes = "" @files.each do |file| includes += "#{prologue_each_cpp(folder) }\n\#include \"#{relative_path(file)}\"#{epilogue_each_cpp(folder)}" if logues includes += "\n#include \"#{relative_path(file)}\"" unless logues end prologue_cpp + define + includes + epilogue_cpp + "\n" end def add_regexp( regexp ) #throw ArgumentError.new( "Invalid regexp object specified (#{regexp.class})." ) unless ( regexp.is_a?( Regexp ) ) @regexps << regexp end # returns true on first match with regexps def match( path ) #puts "#{path} matching with #{@regexps.length} regexps" @regexps.each do |regexp| regexp = Regexp.new(EMPTY_REGEXP.sub(FOLDER_TOKEN, @name), EMPTY_REGEXP_OPTIONS) if regexp.nil? #puts "match #{path} #{regexp.to_s}" if path =~ regexp #puts "no match #{path} #{regexp.to_s}" unless path =~ regexp #puts "Regexp= #{regexp.to_s} path #{path} #{path =~ regexp}" return true if path =~ regexp end return false end end # class UnityFile ################################################### # The Unity Build System. # class UnityBuild VERSION = "1.0" REGEXP_COMMENT = /^(\s*)(REM|\#|\/\/)/i REGEXP_UNITY_FILE = /^(\S*)\s+(\S*)\s+(\S*)/i REGEXP_EMPTY_UNITY_FILE = /^(\w*)/i REGEXP_COMMAND = /^(\s*)(set)\s+(\w+)\s*(.*)/i REGEXP_SPLIT_DELIMITERS = /[\s|\,]/i UNITY_FOLDER = "_Unity" LOG_FILE = 'unitybuild' BUDDY = "\nBuddy: n/a" COMMAND_ENABLED = "enabled" COMMAND_EXE = "exe" COMMAND_EXCLUDE = "exclude" PROLOGUE_FILENAME = "forceinclude/_unity_prologue.h" EPILOGUE_FILENAME = "forceinclude/_unity_epilogue.h" LOGUE_COMMENT = "//Created by #{__FILE__} v#{VERSION} : This file can be edited as required for its project requirements. A default definition is only ever automatically added if it doesn't exist.\n" attr_accessor :p4 # p4 object - handy since it determines if rage p4 or otherwise! attr_reader :unity_files # instances of UnityFile class. attr_reader :other_files # other files in the unity build eg. support files ( _unity_epilogue.h & _unity_prologue.h ) attr_reader :num_excluded # number of excluded files attr_accessor :logues # logues on or off? attr_reader :exe # true if the target assembly is an exe. @@log = nil def UnityBuild.log @@log = Log.new( LOG_FILE ) if @@log == nil @@log end def enabled() # unity_files.length > 0 and @active @active end def initialize( config_file ) begin @project = OS::Path.get_basename(config_file) @project_folder = OS::Path.get_directory(config_file) @unity_folder = OS::Path.combine(@project_folder,UNITY_FOLDER) @unity_files = [] @other_files = [] @exclusions = [] @num_excluded = 0 @config = Pipeline::Config.instance @active = true @logues = true @exe = false OS::Path.set_downcase_on_normalise(false) @config_file = OS::Path.normalise( config_file ) ret = load_config( @config_file ) if ( ret == false ) UnityBuild::log.info " No unity build will be processed due to config file." @active = false return self end UnityBuild::log.debug "" UnityBuild::log.debug "Unity Build - Version #{VERSION}" UnityBuild::log.debug "==========================" UnityBuild::log.debug " Project folder #{@project_folder}" UnityBuild::log.debug " Unity folder #{@unity_folder}" FileUtils.mkdir_p(@unity_folder) unless File.directory?(@unity_folder) scm_init( ) rescue Exception => ex puts "Error: Unhandled exception: #{ex.message}" puts ex.backtrace.join( "\n" ) UnityBuild::log.error( ex, 'Error: Unhandled exception in unitybuild' ) end end # load up the unity configuration file def load_config( path ) begin OS::Path.set_downcase_on_normalise(false) @config_file = OS::Path.normalise(path) if not File.exist? @config_file UnityBuild::log.warn("The config file #{@config_file} does not exist") return false end UnityBuild::log.debug "0. Parsing unity config file : #{path}..." count = 1 ok = false enabled = false File.open(@config_file, 'r') do |f| while (line = f.gets) UnityBuild::log.debug("#{count}: #{line}") count += 1 line.strip! is_comment = line =~ REGEXP_COMMENT if ( (not is_comment) and line.length > 0 ) if (line =~ REGEXP_COMMAND) command = $3.strip.downcase params = $4.strip.downcase UnityBuild::log.debug( " Processing command #{command} #{params}" ) case command when COMMAND_ENABLED enabled = true ok = true UnityBuild::log.debug( " Enabled" ) when COMMAND_EXE @exe = true UnityBuild::log.debug( " Executable" ) when COMMAND_EXCLUDE @exclusions.concat(params.split(REGEXP_SPLIT_DELIMITERS)) UnityBuild::log.debug( " Exclusions added" ) end next end if ( line !~ REGEXP_UNITY_FILE and line !~ REGEXP_EMPTY_UNITY_FILE) UnityBuild::log.error("Error: Invalid line in config file #{count}: #{line}") next end name = $1.strip regexp_str = $2 regexp_options = $3 regexp = nil if (regexp_str and regexp_options) regexp = Regexp.new( regexp_str, regexp_options ) regexp_str.strip! regexp_options.strip! end # add to existing unity_file? added = false @unity_files.each do |unity_file| if ( unity_file.name == name ) UnityBuild::log.debug( "Adding another regexp #{name}" ) unity_file.add_regexp( regexp ) added = true ok = true break end end if not added UnityBuild::log.debug( " Adding unity file #{name} #{regexp_str}" ) unity_file = UnityFile.new( name, @unity_folder, @project, @project_folder ) unity_file.add_regexp( regexp ) @unity_files << unity_file ok = true end end end end UnityBuild::log.warn("Unity Build is enabled but not OK") if enabled and not ok ok and enabled rescue Exception => ex puts "Error: Unhandled exception: #{ex.message}" puts ex.backtrace.join( "\n" ) UnityBuild::log.error( ex, 'Error: Unhandled exception in unitybuild' ) end end # load_config # filename is registered with the unity build system # - returns false if the file is not included in the unity build # - returns true if the file is included in the unity build def add_file( path ) begin OS::Path.set_downcase_on_normalise(false) OS::Path.normalise( path ) filename = OS::Path.get_filename(path).downcase # check for specific exclusion. @exclusions.each do |exclude| if exclude.downcase==filename UnityBuild::log.info(" Excluded #{path.downcase}.") @num_excluded += 1 return false end end matches = 0 @unity_files.each do |unity_file| if ( unity_file.match( path ) ) unity_file.include( path ) matches += 1 break end end matches==1 rescue Exception => ex puts "Error: Unhandled exception: #{ex.message}" puts ex.backtrace.join( "\n" ) UnityBuild::log.error( ex, 'Error: Unhandled exception in unitybuild' ) end end # a comment for scm & files. def comment() " Automatically generated by unity_build.rb v#{VERSION}\n// Using config in #{@config_file} for project #{@project}\n" end # the content of the prologue file ( default ) def prologue_content( ) "#{LOGUE_COMMENT}\#pragma warning(push)" end # the content of the epilogue file ( default ) def epilogue_content( ) "#{LOGUE_COMMENT}\#pragma warning(pop)" end # create the prologue or epilogue file and update in p4 # the file is only created if it doesn't exist otherwise # it is left as it is - since there may be custom settings per project. def create_logue(file, type, content) filename = OS::Path.combine(@unity_folder, file) other_files << filename fstat = @p4.run_fstat( filename ).shift if (fstat.nil? or fstat['headAction'].include?("delete")) UnityBuild::log.debug(" Adding a new #{type} #{filename}") @p4.run_add( filename ) File.open(filename, 'w') do |f| f.write( content ) end else UnityBuild::log.debug(" #{type} #{filename} already exists - it will now be created.") end end # Create unity epilogue and epilogue # - for want of a better name! - what the f*ck are they collectively? def create_unity_logues( ) create_logue(PROLOGUE_FILENAME, "prologue", prologue_content) create_logue(EPILOGUE_FILENAME, "epilogue", epilogue_content) end # Writes out new unity files. def create_unity_files( ) scm_open() UnityBuild::log.debug(" #{@unity_files.length} unity files.") total_includes = 0 @unity_files.each do |unity_file| UnityBuild::log.debug(" Writing (#{unity_file.files.length} includes) in #{unity_file.filename()}" ) unity_file.write( comment(), UNITY_FOLDER, @logues ) total_includes += unity_file.files.length end UnityBuild::log.debug(" Creating unity logues ( prologue & epilogue for each file )") #create_unity_logues() if @logues UnityBuild::log.debug(" Reverting any files that didn't change") reverted = [] @unity_files.each do |unity_file| reverted = @p4.run_revert('-a', unity_file.filename) # revert any files that didn't change end UnityBuild::log.debug(" Reverted #{reverted.length} files") if reverted UnityBuild::log.debug(" #{total_includes} includes written in #{unity_files.length} files ( mean = #{total_includes/(unity_files.length)} includes )" ) if unity_files.length > 0 end # Create p4 object def scm_init( ) @p4 = SCM::Perforce::new @p4.connect() end # checkout unity files def scm_open( ) begin @p4.connect() raise Exception if not @p4.connected? @p4.run_sync( "#{@unity_folder}/*.cpp" ) @unity_files.each do |unity_file| filename = unity_file.filename() fstat = @p4.run_fstat( filename ).shift if (fstat.nil?) UnityBuild::log.debug " #{filename} added to default CL" @p4.run_add( filename ) fstat = @p4.run_fstat( filename ).shift basetype, modifiers = fstat['headType'].split( '+' ) if (fstat.key?('headType')) # perforce drives me dippy. basetype, modifiers = fstat['type'].split( '+' ) if (fstat.key?('type')) @p4.run_reopen( '-t', "#{basetype}+w", filename ) else UnityBuild::log.debug " #{filename} edited in default CL" basetype, modifiers = fstat['headType'].split( '+' ) if (fstat.key?('headType')) # perforce drives me dippy. basetype, modifiers = fstat['type'].split( '+' ) if (fstat.key?('type')) @p4.run_revert( filename ) @p4.run_reopen( '-t', "#{basetype}+w", filename ) @p4.run_sync( filename ) @p4.run_edit( filename ) end end rescue Exception => ex puts "Error: Unhandled exception: #{ex.message}" puts ex.backtrace.join( "\n" ) UnityBuild::log.error( ex, 'Error: Unhandled exception in unitybuild' ) end end end # class UnityBuild ###################################################################################################### # MONKEY PATCHES ##################################################################################### # monkey patch the file class of the projbuilder class Info::File attr_accessor :_fullpath_ end # monkey patch the ProjBuilder class class ProjBuilder attr_accessor :projects # recurse into filters and return array of files def get_filter_files( filter, filter_path ) files = [] UnityBuild::log.debug " adding #{filter.files.length} files in filter #{filter.path}" filter.files.each do |file| UnityBuild::log.debug "filter_path, file.path = #{filter_path}, #{file.path}" file._fullpath_ = filter_path+file.path files << file end filter.filters.each do |f| files.concat( get_filter_files( f, filter_path+f.path ) ) end files end # for a project discover all the files within it. def discover_files( project ) files = [] project.files.each do |file| UnityBuild::log.debug " adding root file #{file.path}" file._fullpath_ = file.path files << file end project.filters.each do |filter| UnityBuild::log.debug " adding root filter #{filter.path}" filter_files = get_filter_files( filter, filter.path ) filter_files.each do |file| UnityBuild::log.debug "file discovered : #{file._fullpath_}" files << file end end return files end # Process project builder internal format so it represents a unity build. def unity_process( unity_config_file, project ) ub = UnityBuild.new(unity_config_file) if ub.exe UnityBuild::log.info "Unity build not enabled for executables" return true if ub.exe # we don't support unity builds for exes yet end if not ub.enabled UnityBuild::log.info "The unity build is not enabled and will not process" return nil end #ub.p4.run_revert(vcproj) UnityBuild::log.debug "-------------------------------------" UnityBuild::log.debug "1. Get a list of all the files in the project, record if they have custom build steps or if they are excluded from the build." files = discover_files( project ) UnityBuild::log.debug " #{files.length} files discovered." files.each do |file| UnityBuild::log.debug file.has_custom_build_steps ? " discovered #{file.class} #{file._fullpath_} (custom build steps)" : "discovered #{file.class} #{file._fullpath_}" file.excluded_from_build = false end UnityBuild::log.debug "-------------------------------------" UnityBuild::log.debug "2. For the list of files in the project add them to the unitybuild." UnityBuild::log.debug " - unless they have custom build steps." UnityBuild::log.debug " - unless they are are already excluded from the build." files_accepted, files_rejected, files_unavailable = [], [], [] files.each do |file| if ( (not file.has_custom_build_steps) and (not file.excluded_from_build) ) UnityBuild::log.debug "file sent to unity build : #{file._fullpath_}" accepted = ub.add_file( file.path ) files_accepted << file if accepted files_rejected << file unless accepted else files_unavailable << file end end UnityBuild::log.debug " #{ub.num_excluded} files excluded from unity build" UnityBuild::log.debug " #{files_unavailable.length} unavailable to unity build" files_unavailable.each { |file| UnityBuild::log.debug " file unavailable to unity build : #{file._fullpath_}" } UnityBuild::log.debug " #{files_rejected.length} rejected by unity build" files_rejected.each { |file| UnityBuild::log.debug " file rejected by unity build : #{file._fullpath_}" } UnityBuild::log.debug " #{files_accepted.length} accepted by unity build" files_accepted.each { |file| UnityBuild::log.debug " file accepted by unity build : #{file._fullpath_}" } UnityBuild::log.info " * Unity build summary : unavailable=#{files_unavailable.length} rejected=#{files_rejected.length} accepted=#{files_accepted.length}." UnityBuild::log.debug "-------------------------------------" UnityBuild::log.debug "3. For all files sucessfully added to the unity build now exclude them from the build - for all platforms and build configurations." files_accepted.each do |file| UnityBuild::log.debug " file accepted by unity build now excluded from build : #{file._fullpath_}" file.excluded_from_build = true end files = discover_files( project ) excluded = 0 files.each do |file| if (file.excluded_from_build) UnityBuild::log.debug " #{file.class} #{file._fullpath_} (verified as excluded from build)" excluded += 1 end end UnityBuild::log.debug " #{excluded} files verified as excluded from build." UnityBuild::log.debug "-------------------------------------" UnityBuild::log.debug "4. Create the unity files." ub.create_unity_files() UnityBuild::log.debug "-------------------------------------" UnityBuild::log.debug "5. Add unity files to the project." if (ub.unity_files.length > 0 and files_accepted.length > 0) UnityBuild::log.debug " 5.1 create a filter" filter = Info::Filter.new(UnityBuild::UNITY_FOLDER) filter.files = [] UnityBuild::log.debug " 5.2 add unity files to the filter" ub.unity_files.each do |unity_file| filepath = unity_file.filename file = Info::File.new( filepath ) filter.files << file end #if (ub.logues) # UnityBuild::log.debug " 5.3 add epilogue and prologue files to the filter" # ub.other_files.each do |filename| # file = Info::File.new( filename ) # filter.files << file # end #end end project.filters << filter UnityBuild::log.debug "Unity Build Processed OK" return true end end ###################################################################################################### # ENTRY POINT ######################################################################################## # Test - allows the unity build to be tested in _relative isolation_ from the projectbuilder system. # well it is using the internal format / importer / exporter to test it, but is not injected into the # project builder. # # NB. you should feed it a non unity build project! - unity conversion is not a reverable process - it can't ununitise itself. if ( __FILE__ == $0 ) g_AppName = File::basename( __FILE__, '.rb' ) OPTIONS = [ [ '--help', '-h', OS::Getopt::BOOLEAN ], [ '--config', '-c', OS::Getopt::REQUIRED ], [ '--vcproj', '-p', OS::Getopt::REQUIRED ], [ '--pretest_projbuilder', '-t', OS::Getopt::BOOLEAN ], ] opts, trailing = OS::Getopt.getopts( OPTIONS ) if ( opts['help'] ) puts OS::Getopt.usage( OPTIONS ) puts ("Press Enter to continue...") Process.exit!( 1 ) end if ( not opts['config']) $stderr.puts("config not specified in commandline to #{g_AppName}") puts OS::Getopt.usage( OPTIONS ) Process.exit!( 1 ) end if ( not opts['vcproj']) $stderr.puts("vcproj not specified in commandline to #{g_AppName}") puts OS::Getopt.usage( OPTIONS ) Process.exit!( 1 ) end pretest_projbuilder = opts['pretest_projbuilder'] vcproj = opts['vcproj'] config_file = opts['config'] format = :vs2005 if vcproj.include?("2005") format = :vs2008 if vcproj.include?("2008") format = :vs2010 if vcproj.include?("2010") UnityBuild::log.debug "unity config #{config_file}" ub = UnityBuild.new(config_file) UnityBuild::log.debug "Revert VCproj file" ub.p4.run_revert(vcproj) projbuild = ProjBuilder.new( format == :vs2010 ) # Test import / export first - cos I'm about to dick around in the internal representation. # So I can be sure I haven't broken anything - I need to verify the import/export works first. if (pretest_projbuilder) UnityBuild::log.debug "Pre-testing projectbuilder import/export" if (projbuild.import(vcproj, format)) if ( projbuild.export(vcproj, format) ) UnityBuild::log.debug " project #{vcproj} exported ok" else UnityBuild::log.debug " error occurred exporting project before unity build could start! - i.e. not my fault #{vcproj}" end end end if (projbuild.import(vcproj, format)) projbuild.unity_process(config_file, vcproj.projects[0] ) UnityBuild::log.debug "-------------------------------------" UnityBuild::log.debug "Export the project." if ( projbuild.export(vcproj, format) ) UnityBuild::log.debug " #{vcproj} Exported OK." else UnityBuild::log.error " error occurred exporting project #{vcproj}" end else UnityBuild::log.error "error occurred importing project #{vcproj}" end UnityBuild::log.debug "" UnityBuild::log.debug "#{__FILE__} Completed OK." end # if ( __FILE__ == $0 ) end # module ProjBuild end # module Pipeline