982 lines
32 KiB
Ruby
Executable File
982 lines
32 KiB
Ruby
Executable File
#
|
|
# File:: codebuilder_stats_analysis.rb
|
|
# Description:: Post process script that analyses the stats ( performance, memory and otherwise ) written out from game.
|
|
#
|
|
# - game code team ultimately maintain this script since it will be involved with memory and perf budgets.
|
|
#
|
|
# - if this script returns a non zero return code the error will bubble up to Cruise control to indicate a breakage of build and email those involved in breakage.
|
|
#
|
|
# - Forthcoming features in collaboration with runtime team...
|
|
# - reads the modifications.xml file to determine the breakers and the scope of the change.
|
|
# - analyse last entry in database and email upon bad deltas ( an opportunity for a custom email with very specific details to be sent )
|
|
# - produce graphs.
|
|
# - makes a nice cup of tea.
|
|
#
|
|
# Author:: Klaas Schilstra <klaas.schilstra@rockstarnorth.com>
|
|
# Author:: Derek Ward <derek.ward@rockstarnorth.com>
|
|
# Date:: 29th August 2011
|
|
#
|
|
# Passed in :- see OPTIONS ...
|
|
# Passed out :- stderr contains all errors
|
|
# stdout for all other output.
|
|
# Returns :- returns non zero upon detecting any errors
|
|
|
|
#-----------------------------------------------------------------------------
|
|
# Uses / Requires
|
|
#-----------------------------------------------------------------------------
|
|
require 'pipeline/config/projects'
|
|
require 'pipeline/os/getopt'
|
|
require 'pipeline/os/file'
|
|
require 'pipeline/util/email'
|
|
require 'systemu'
|
|
require 'rexml/document'
|
|
require 'fileutils'
|
|
require 'sqlite3'
|
|
|
|
include Pipeline
|
|
|
|
#-----------------------------------------------------------------------------
|
|
# Constants
|
|
#-----------------------------------------------------------------------------
|
|
OPTIONS = [
|
|
[ "--help", "-h", OS::Getopt::BOOLEAN, "display usage information." ],
|
|
[ '--publish_folder_src', '-ps', OS::Getopt::REQUIRED, 'the folder name of published artifacts triggering this build' ],
|
|
[ '--publish_folder_dst', '-pd', OS::Getopt::REQUIRED, 'the folder name of the published database' ],
|
|
[ '--stats_db', '-s', OS::Getopt::REQUIRED, 'the stats database filename ( this should be in the build directory )' ],
|
|
[ '--stats_capture', '-cap', OS::Getopt::REQUIRED, 'the stats capture filename' ],
|
|
[ '--orig_stats_db_filename', '-osd', OS::Getopt::REQUIRED, 'the stats database filename where it persists in p4 ( this should be in the publish_folder_dst, this is a filesystem path )' ],
|
|
[ '--enable_checkin', '-c', OS::Getopt::BOOLEAN, 'enable checkin - default is off' ],
|
|
|
|
]
|
|
|
|
# -- output prefixes ---
|
|
INFO = "[colourise=black]INFO_MSG: "
|
|
INFO_BLUE = "[colourise=blue]INFO_MSG: "
|
|
MSG_PREFIX_EMAIL = "[colourise=blue]INFO_EMA: "
|
|
MSG_PREFIX_WEB = "[colourise=blue]INFO_WEB: "
|
|
INFO_GREEN = "[colourise=green]INFO_MSG: "
|
|
INFO_ORANGE = "[colourise=orange]INFO_MSG: "
|
|
INFO_GREY = "[colourise=grey]INFO_MSG: "
|
|
INFO_RED = "[colourise=red]INFO_MSG: "
|
|
MSG_PREFIX = "#{INFO_BLUE} Codebuilder_stats_analysis:"
|
|
MSG_PREFIX_PERSIST = "#{INFO_BLUE} Codebuilder_stats_analysis:"
|
|
MSG_PREFIX_GREY = "#{INFO_GREY} Codebuilder_stats_analysis:"
|
|
MSG_PREFIX_RED = "#{INFO_RED} Codebuilder_stats_analysis:"
|
|
|
|
# --- email settings ---
|
|
EMAIL_ADDRESSES = [ "derek@rockstarnorth.com", "klaas.schilstra@rockstarnorth.com" ] # build masters email addresses
|
|
EMAILER_ENABLED = true # all emailing disabled ( except Cruise control )
|
|
EMAILER_EMAILS_USERS = false # users in tested changelists emailed?
|
|
EMAIL_SUBJECT = "Codebuilder Stats Analysis"
|
|
EMAIL_REPLY = "do_not_reply"
|
|
EMAIL_URL = true
|
|
BGCOL_DEFAULT = "#999"
|
|
FGCOL_DEFAULT = "#FFF"
|
|
BGCOL_ERROR = "#F00"
|
|
BGCOL_WARNING = "#F60"
|
|
|
|
# --- misc ---
|
|
AGGREGATE_MODIFICATIONS_FILE = "modifications.xml"
|
|
STATS_FILE = "stats.xml"
|
|
MODIFICATIONS_XPATH = "//ArrayOfModification/Modification"
|
|
|
|
#=================================================================================================================
|
|
#
|
|
# CodebuilderStatsAnalyser : a class that analyses a database of code stats gathered from a smoketest of the game.
|
|
#
|
|
class CodebuilderStatsAnalyser
|
|
|
|
attr_accessor :database
|
|
|
|
#************************************ CONSTRUCTOR & CONTROL ******************************************
|
|
|
|
#---------------------------------------------------------------------------------------------------------------------------
|
|
#
|
|
# Constructor
|
|
#
|
|
def initialize(project_name)
|
|
@project_name = project_name
|
|
end
|
|
|
|
@@log = nil
|
|
def CodebuilderStatsAnalyser.log
|
|
|
|
@@log = Log.new( 'codebuilder_stats_analyser' ) if @@log == nil
|
|
@@log
|
|
end
|
|
|
|
#---------------------------------------------------------------------------------------------------------------------------
|
|
#
|
|
# Control logic
|
|
#
|
|
def process(config, publish_folder_src, publish_folder_dst, stats_capture, stats_db, enable_checkin, orig_stats_db_filename)
|
|
@publish_folder_src = publish_folder_src
|
|
@publish_folder_dst = publish_folder_dst
|
|
@stats_capture_filename = stats_capture
|
|
@stats_db_filename = stats_db
|
|
@enable_checkin = enable_checkin
|
|
@orig_stats_db_filename = orig_stats_db_filename
|
|
|
|
# ---- 1. create a p4 connection ----
|
|
create_p4(config)
|
|
|
|
# ---- 2. read modifications ----
|
|
read_modifications()
|
|
|
|
# ---- 3. get the latest db ----
|
|
get_stats()
|
|
get_db()
|
|
|
|
# ---- 4. evaluate stats ----
|
|
evaluation = stats_evaluation()
|
|
|
|
# ---- 5. send evaluation ----
|
|
send_evaluation(evaluation)
|
|
|
|
# ---- 6. update the db locally ----
|
|
update_db()
|
|
|
|
CodebuilderStatsAnalyser.log.info("#{MSG_PREFIX} FIN!!")
|
|
|
|
if evaluation.success?
|
|
return 1
|
|
else
|
|
return -1
|
|
end
|
|
end
|
|
|
|
|
|
|
|
#************************************ EMAILING ******************************************
|
|
|
|
#---------------------------------------------------------------------------------------------------------------------------
|
|
#
|
|
# Send an email
|
|
#
|
|
def send_email(sender, subject, text, user_email_addresses = [])
|
|
return unless EMAILER_ENABLED
|
|
|
|
begin
|
|
email_options = { :server => Pipeline::Config::instance().mailserver,
|
|
:port => Pipeline::Config::instance().mailport.to_i,
|
|
:username => nil,
|
|
:password => nil,
|
|
:domain => Pipeline::Config::instance().maildomain,
|
|
:auth_mode => :plain }
|
|
|
|
email = Util::Email.new( email_options )
|
|
|
|
emails = []
|
|
EMAIL_ADDRESSES.each { |email_addr| emails << { :address => email_addr } }
|
|
|
|
if (EMAILER_EMAILS_USERS)
|
|
user_email_addresses.each do |user_email_address|
|
|
emails << { :address => user_email_address }
|
|
end
|
|
end
|
|
|
|
from = { :address => "#{EMAIL_REPLY}@#{email_options[:domain]}", :alias => sender }
|
|
|
|
email.send( from,
|
|
emails,
|
|
subject,
|
|
text,
|
|
'text/html' )
|
|
rescue Exception => ex
|
|
CodebuilderStatsAnalyser.log.error "Error: Unhandled exception: #{ex.message}"
|
|
CodebuilderStatsAnalyser.log.error ex.backtrace().join("\n")
|
|
end
|
|
end
|
|
|
|
#---------------------------------------------------------------------------------------------------------------------------
|
|
#
|
|
# returns an html string for composing an html report email.
|
|
#
|
|
def construct_email_html(project_name, evaluation, modifications)
|
|
|
|
title = "Codebuilder Stats Analysis Report"
|
|
bg_col = BGCOL_DEFAULT
|
|
fg_col = FGCOL_DEFAULT
|
|
|
|
|
|
urls = get_urls_from_modifications()
|
|
max_cl = get_latest_changelist_number_from_modifications()
|
|
|
|
# Subject
|
|
title = evaluation.success? ? "Success" : "Error"
|
|
subject = "#{EMAIL_SUBJECT} #{project_name} : #{title}"
|
|
|
|
# Header
|
|
header = "<html>
|
|
<body>
|
|
<table border=\"1\" width=\"100%\">
|
|
<tr><td colspan=\"2\" align=\"center\" style=\"background-color: #{bg_col}; color: #{fg_col}; font-size: 16pt\">#{title}</td></tr>
|
|
<tr>
|
|
<td style=\"background-color: #{bg_col}; color: #{fg_col}\">Project</td>
|
|
<td>#{project_name}</td>
|
|
</tr>
|
|
<tr>
|
|
<td style=\"background-color: #{bg_col}; color: #{fg_col}\">Build Time</td>
|
|
<td>#{Time.now.strftime("%H:%M:%S %d-%m-%Y")}</td>
|
|
</tr>
|
|
<tr>
|
|
<td style=\"background-color: #{bg_col}; color: #{fg_col}\">Max Changelist</td>
|
|
<td>#{max_cl}</td>
|
|
</tr>"
|
|
|
|
# Messages
|
|
message_rows = ""
|
|
evaluation.errors.each do |msg|
|
|
message_rows += "<tr>
|
|
<td style=\"background-color: #{BGCOL_ERROR}; color: #{fg_col}\">Message</td>
|
|
<td>#{msg}</td>
|
|
</tr>"
|
|
end
|
|
evaluation.warnings.each do |msg|
|
|
message_rows += "<tr>
|
|
<td style=\"background-color: #{BGCOL_WARNING}; color: #{fg_col}\">Message</td>
|
|
<td>#{msg}</td>
|
|
</tr>"
|
|
end
|
|
evaluation.messages.each do |msg|
|
|
message_rows += "<tr>
|
|
<td style=\"background-color: #{bg_col}; color: #{fg_col}\">Message</td>
|
|
<td>#{msg}</td>
|
|
</tr>"
|
|
end
|
|
|
|
tableend = "</table>"
|
|
|
|
modification = report_modifications()
|
|
|
|
# Test results
|
|
test_results = evaluation.report_html()
|
|
|
|
footer = "</body>
|
|
</html>"
|
|
|
|
return subject, "#{header} #{message_rows} #{tableend} #{modification} #{test_results} #{footer}"
|
|
end
|
|
|
|
|
|
#---------------------------------------------------------------------------------------------------------------------------
|
|
#
|
|
# ah spit, this is a problem/hack we are hardcoding the studio domain here for now, it's not necessarily in our local domain
|
|
# TODO: get the perforce email addresses sorted out? - problem solved.
|
|
#
|
|
def username_to_email_address(username)
|
|
"#{username}@#{Pipeline::Config::instance().maildomain}"
|
|
end
|
|
|
|
|
|
|
|
#************************************ P4 OPS ******************************************
|
|
|
|
#---------------------------------------------------------------------------------------------------------------------------
|
|
#
|
|
# Create p4 connection
|
|
#
|
|
def create_p4(config)
|
|
CodebuilderStatsAnalyser.log.info "#{MSG_PREFIX} Create p4 connection"
|
|
|
|
@p4 = SCM::Perforce::create( config.sc_server, config.sc_username, config.sc_workspace )
|
|
@p4.connect( )
|
|
raise Exception if not @p4.connected?
|
|
end
|
|
|
|
|
|
|
|
|
|
#************************************ DB OPS ******************************************
|
|
|
|
|
|
#---------------------------------------------------------------------------------------------------------------------------
|
|
#
|
|
# Get the latest database file - can't assume we are on the head - people will tinker!
|
|
# Also, we have to work locally for some fairly complex publishing reasons.
|
|
#
|
|
def get_db()
|
|
CodebuilderStatsAnalyser.log.info "#{MSG_PREFIX_GREY} get_db"
|
|
|
|
# ---- sync to the head of the publish dest folder ----
|
|
sync = "#{@publish_folder_dst}\...#head"
|
|
CodebuilderStatsAnalyser.log.info("#{MSG_PREFIX} syncing to the latest publishing destination folder #{sync}")
|
|
@p4.run_sync(sync)
|
|
|
|
# ---- Copy db to local build dir to work on. ----
|
|
CodebuilderStatsAnalyser.log.info("#{MSG_PREFIX} Copy db to local build dir to work on. #{@orig_stats_db_filename} -> #{@stats_db_filename}")
|
|
|
|
FileUtils.cp(@orig_stats_db_filename, @stats_db_filename)
|
|
|
|
initialize_db(@stats_db_filename)
|
|
end
|
|
|
|
|
|
# Get the stats from the src folder
|
|
def get_stats()
|
|
CodebuilderStatsAnalyser.log.info "#{MSG_PREFIX_GREY} get_stats"
|
|
stats_filename = OS::Path::combine(@publish_folder_src, STATS_FILE)
|
|
if not File.exist?(stats_filename)
|
|
CodebuilderStatsAnalyser.log.warn "#{MSG_PREFIX} #{mods_filename} not found"
|
|
end
|
|
CodebuilderStatsAnalyser.log.info("#{MSG_PREFIX} Copy stats to local build dir to work on. #{stats_filename} -> #{@stats_capture_filename}")
|
|
|
|
FileUtils.cp(stats_filename, @stats_capture_filename)
|
|
end
|
|
|
|
#---------------------------------------------------------------------------------------------------------------------------
|
|
#
|
|
# Update the database file
|
|
# - please note this only happens locally in the build dir.
|
|
# - Cruise control should copy this to build folder.
|
|
# - Then elsewhere ( not in this script) a generalised publishing post build step will handle the submit of the DB to perforce.
|
|
#
|
|
def update_db()
|
|
CodebuilderStatsAnalyser.log.info "#{MSG_PREFIX_GREY} update_db"
|
|
|
|
# ---- for this example the capture file is just copied over the database ( this will need more development ) ----
|
|
CodebuilderStatsAnalyser.log.info("#{MSG_PREFIX} Test DB update : copy #{@stats_db_filename} -> #{@orig_stats_db_filename}")
|
|
FileUtils.cp(@stats_db_filename, @orig_stats_db_filename)
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
#************************************** MODIFICATION HANDLING ******************************************
|
|
|
|
#---------------------------------------------------------------------------------------------------------------------------
|
|
#
|
|
# read the modifications file - either to get a list of users or to understand the scope of the change if required.
|
|
#
|
|
# FYI - the modifications.xml 'schema'
|
|
#
|
|
#<!-- Start of the group of modifications (even if just one). -->
|
|
#<ArrayOfModification xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema">
|
|
# <!-- Start of one modification. -->
|
|
# <Modification>
|
|
# <!-- The change number. -->--filename=
|
|
# <ChangeNumber>... value ...</ChangeNumber>
|
|
# <!-- The comment. -->
|
|
# <Comment>... value ...</Comment>
|
|
# <!-- The user's email address. -->
|
|
# <EmailAddress>... value ...</EmailAddress>
|
|
# <!-- The affected file name. -->
|
|
# <FileName>... value ...</FileName>
|
|
# <!-- The affect file's folder name. -->
|
|
# <FolderName>... value ...</FolderName>
|
|
# <!-- The change timestamp, in yyyy-mm-ddThh:mm:ss.nnnn-hhmm format -->
|
|
# <ModifiedTime>... value ...</ModifiedTime>
|
|
# <!-- The operation type. -->
|
|
# <Type>... value ...</Type>
|
|
# <!-- The user name. -->
|
|
# <UserName>... value ...</UserName>
|
|
# <!-- The related URL. -->
|
|
# <Url>... value ...</Url>
|
|
# <!-- The file version. -->
|
|
# <Version>... value ...</Version>
|
|
# <!-- End of modification. -->
|
|
# </Modification>
|
|
# <!-- End of the group of modifications. -->
|
|
#</ArrayOfModification>
|
|
#Pasted from <http://confluence.public.thoughtworks.org/display/CCNET/Modification+Writer+Task>
|
|
|
|
def read_modifications()
|
|
|
|
CodebuilderStatsAnalyser.log.info "#{MSG_PREFIX_GREY} read_modifications"
|
|
|
|
@modifications = nil
|
|
mods_filename = OS::Path::combine(@publish_folder_src, AGGREGATE_MODIFICATIONS_FILE)
|
|
|
|
if File.exist?(mods_filename)
|
|
File.open(mods_filename) do |file|
|
|
doc = REXML::Document.new(file)
|
|
@modifications = doc.elements.to_a( MODIFICATIONS_XPATH )
|
|
CodebuilderStatsAnalyser.log.info("#{MSG_PREFIX} #{@modifications.length} modifications read") if @modifications
|
|
end
|
|
else
|
|
CodebuilderStatsAnalyser.log.warn "#{MSG_PREFIX} #{mods_filename} not found"
|
|
end
|
|
end
|
|
|
|
#---------------------------------------------------------------------------------------------------------------------------
|
|
#
|
|
# Derive user emails from modifications xml nodes
|
|
#
|
|
def get_user_emails_addresses_from_modifications()
|
|
usernames = []
|
|
if @modifications
|
|
@modifications.each do |mod|
|
|
username = mod.elements["UserName"].text
|
|
next if username=="buildernorth"
|
|
email_addr = username_to_email_address(username)
|
|
usernames << email_addr unless usernames.include?email_addr
|
|
end
|
|
end
|
|
|
|
usernames
|
|
end
|
|
|
|
#---------------------------------------------------------------------------------------------------------------------------
|
|
#
|
|
# Get the url from the modifications file
|
|
#
|
|
def get_urls_from_modifications()
|
|
urls = []
|
|
if @modifications
|
|
@modifications.each do |mod|
|
|
username = mod.elements["UserName"].text
|
|
next if username=="buildernorth"
|
|
url = mod.elements["Url"].text
|
|
urls << url unless urls.include?url
|
|
end
|
|
end
|
|
|
|
urls
|
|
end
|
|
|
|
#---------------------------------------------------------------------------------------------------------------------------
|
|
#
|
|
# Get the comments from the modifications file
|
|
#
|
|
def get_comments_from_modifications()
|
|
comments = []
|
|
if @modifications
|
|
@modifications.each do |mod|
|
|
username = mod.elements["UserName"].text
|
|
next if username=="buildernorth"
|
|
comment = mod.elements["Comment"].text
|
|
comments << comment unless comments.include?comment
|
|
end
|
|
end
|
|
|
|
comments
|
|
end
|
|
|
|
#---------------------------------------------------------------------------------------------------------------------------
|
|
#
|
|
# Create an html table of the modifications
|
|
#
|
|
def report_modifications()
|
|
bgcol = BGCOL_DEFAULT
|
|
fgcol = FGCOL_DEFAULT
|
|
report = "<table border=\"1\" width=\"100%\">"
|
|
report += "<tr>
|
|
<td style=\"background-color: #{bgcol}; color: #{fgcol}\">changelist</td>
|
|
<td style=\"background-color: #{bgcol}; color: #{fgcol}\">user</td>
|
|
<td style=\"background-color: #{bgcol}; color: #{fgcol}\">summary</td></tr>"
|
|
|
|
changelists = []
|
|
|
|
if @modifications
|
|
@modifications.each do |mod|
|
|
|
|
# Gather unique cls/comments
|
|
cl = mod.elements["ChangeNumber"].text.to_i
|
|
next if changelists.include?(cl)
|
|
changelists.push cl
|
|
|
|
# Ignore system commits
|
|
username = mod.elements["UserName"].text
|
|
next if username=="buildernorth"
|
|
|
|
comment = mod.elements["Comment"].text
|
|
url = mod.elements["Url"].text
|
|
|
|
report += "<tr>
|
|
<td><a href=\"#{url}\">#{cl}</a></td>
|
|
<td><a href=\"mailto:#{username_to_email_address(username)}\">#{username}</a></td>
|
|
<td>#{comment}</td></tr>"
|
|
end
|
|
end
|
|
|
|
report += "</table>"
|
|
end
|
|
|
|
#---------------------------------------------------------------------------------------------------------------------------
|
|
#
|
|
# Get the latest changelist from the modifications file
|
|
#
|
|
def get_latest_changelist_number_from_modifications()
|
|
changelist_numbers = []
|
|
if @modifications
|
|
@modifications.each do |mod|
|
|
changelist_number = mod.elements["ChangeNumber"].text.to_i
|
|
changelist_numbers << changelist_number unless changelist_numbers.include?changelist_number
|
|
end
|
|
end
|
|
|
|
max_cl_no = changelist_numbers.max
|
|
|
|
max_cl_no
|
|
end
|
|
|
|
|
|
#---------------------------------------------------------------------------------------------------------------------------
|
|
#
|
|
# Get the latest changelist from the modifications file
|
|
#
|
|
def get_latest_date_time_from_modifications()
|
|
times = []
|
|
if @modifications
|
|
@modifications.each do |mod|
|
|
time = DateTime.strptime(mod.elements["ModifiedTime"].text.to_s, "%Y-%m-%dT%H:%M:%S")
|
|
times << time unless times.include?time
|
|
end
|
|
end
|
|
|
|
return times.max
|
|
end
|
|
|
|
|
|
|
|
|
|
#************************************** STATISTICS HANDLING ******************************************
|
|
|
|
#---------------------------------------------------------------------------------------------------------------------------
|
|
#
|
|
# Read the database and compare the current capture against the last entry made, was it good?
|
|
# - return non zero and error messages upon bad stats.
|
|
#
|
|
def stats_evaluation()
|
|
CodebuilderStatsAnalyser.log.info "#{MSG_PREFIX_GREY} *** STATS ANALYSIS ***"
|
|
|
|
#---- 1) Store stats in database
|
|
CodebuilderStatsAnalyser.log.info("#{MSG_PREFIX} Store the new values in the database")
|
|
store_stats()
|
|
|
|
#---- 2) Run a series of evaluations on the collected stats in database ----
|
|
CodebuilderStatsAnalyser.log.info("#{MSG_PREFIX} Run a series of evaluations on the collected stats in database")
|
|
evaluation = evaluate_stats()
|
|
|
|
return evaluation
|
|
end
|
|
|
|
#---------------------------------------------------------------------------------------------------------------------------
|
|
#
|
|
# Send evaluation
|
|
#
|
|
def send_evaluation(evaluation)
|
|
CodebuilderStatsAnalyser.log.info "#{MSG_PREFIX_GREY} send_analysis_result"
|
|
|
|
users_email_addresses = get_user_emails_addresses_from_modifications()
|
|
|
|
sender = "#{EMAIL_SUBJECT} #{@project_name}"
|
|
subject, message = construct_email_html(@project_name,evaluation, @modifications)
|
|
|
|
File.open('report.htm', 'w') {|f| f.write(message) }
|
|
|
|
if (evaluation.success?)
|
|
send_email(sender, subject, message )
|
|
else
|
|
send_email(sender, subject, message, users_email_addresses)
|
|
end
|
|
end
|
|
|
|
# initialize the database
|
|
def initialize_db(databasefile)
|
|
begin
|
|
FileUtils.chmod(0777,databasefile)
|
|
@database = SQLite3::Database::open(databasefile)
|
|
rescue
|
|
# Initialisation by exception - novel
|
|
File.delete(databasefile) if File.exist?(databasefile)
|
|
@database = SQLite3::Database.new(databasefile)
|
|
database.execute( "create table session (id INTEGER PRIMARY KEY AUTOINCREMENT, changelist INT, description TEXT, timestamp DATETIME);" )
|
|
database.execute( "create table test (id INTEGER PRIMARY KEY AUTOINCREMENT, session INTEGER, name TEXT, FOREIGN KEY(session) REFERENCES session(id));" )
|
|
database.execute( "create table fpsResult (id INTEGER PRIMARY KEY AUTOINCREMENT, test INTEGER, min FLOAT, max FLOAT, avg FLOAT, FOREIGN KEY(test) REFERENCES test(id));" )
|
|
database.execute( "create table cpuResult (id INTEGER PRIMARY KEY AUTOINCREMENT, test INTEGER, idx INT, name TEXT, setname TEXT, min FLOAT, max FLOAT, avg FLOAT, FOREIGN KEY(test) REFERENCES test(id));" )
|
|
end
|
|
end
|
|
|
|
# generic database execute and display
|
|
def database_execute(sql)
|
|
# puts sql
|
|
@database.execute(sql)
|
|
end
|
|
|
|
# store the results from the tests
|
|
def store_results(filename, changelist, timestamp)
|
|
|
|
if (File::exists?(filename))
|
|
|
|
doc = REXML::Document.new( File.open( filename, 'r' ) )
|
|
# results
|
|
if (doc)
|
|
|
|
# create session
|
|
p database_execute("INSERT INTO session VALUES (
|
|
NULL,
|
|
#{changelist},
|
|
\"smoketest #{changelist}\",
|
|
\"#{timestamp.strftime("%Y-%m-%d %H:%M:%S")}\"
|
|
);")
|
|
sessionid = database_execute("select last_insert_rowid();").to_s.to_i
|
|
|
|
REXML::XPath.each(doc, "//debugLocationMetricsList/results/Item") do |result|
|
|
resultname = REXML::XPath.first(result, "name").text.to_s
|
|
|
|
database_execute("INSERT INTO test VALUES (
|
|
NULL,
|
|
#{sessionid},
|
|
\"#{resultname}\"
|
|
);")
|
|
testid = database_execute("select last_insert_rowid();").to_s.to_i
|
|
|
|
|
|
REXML::XPath.each(result, "fpsResult") do |fpsresult|
|
|
min = REXML::XPath.first(fpsresult, "min/@value").to_s.to_f
|
|
max = REXML::XPath.first(fpsresult, "max/@value").to_s.to_f
|
|
avg = REXML::XPath.first(fpsresult, "average/@value").to_s.to_f
|
|
|
|
database_execute("INSERT INTO fpsResult VALUES (
|
|
NULL,
|
|
#{testid},
|
|
#{min},
|
|
#{max},
|
|
#{avg}
|
|
);")
|
|
end
|
|
|
|
REXML::XPath.each(result, "cpuResults/Item") do |cpuresult|
|
|
index = REXML::XPath.first(cpuresult, "min/@value").to_s.to_i
|
|
name = REXML::XPath.first(cpuresult, "name").text.to_s
|
|
set = REXML::XPath.first(cpuresult, "set").text.to_s
|
|
min = REXML::XPath.first(cpuresult, "min/@value").to_s.to_f
|
|
max = REXML::XPath.first(cpuresult, "max/@value").to_s.to_f
|
|
avg = REXML::XPath.first(cpuresult, "average/@value").to_s.to_f
|
|
|
|
database_execute("INSERT INTO cpuResult VALUES (
|
|
NULL,
|
|
#{testid},
|
|
#{index},
|
|
\"#{name}\",
|
|
\"#{set}\",
|
|
#{min},
|
|
#{max},
|
|
#{avg}
|
|
);")
|
|
end
|
|
end
|
|
end
|
|
else
|
|
CodebuilderStatsAnalyser.log.error("Error: filename #{filename} does not exist")
|
|
end
|
|
end
|
|
|
|
#---------------------------------------------------------------------------------------------------------------------------
|
|
#
|
|
# Store associated values from database and capture
|
|
#
|
|
def store_stats()
|
|
changelist = get_latest_changelist_number_from_modifications()
|
|
timestamp = get_latest_date_time_from_modifications();
|
|
timestamp.strftime("%Y-%m-%d %H:%M:%S") unless timestamp.nil?;
|
|
|
|
CodebuilderStatsAnalyser.log.info "#{MSG_PREFIX} Store stats from #{@stats_capture_filename} in database"
|
|
|
|
# only add if we haven't seen yet
|
|
unique = true
|
|
@database.execute("SELECT * FROM session WHERE changelist == #{changelist}") do
|
|
unique = false
|
|
end
|
|
|
|
if unique
|
|
store_results(@stats_capture_filename, changelist, timestamp)
|
|
end
|
|
end
|
|
|
|
|
|
end # end class StatsAnalyser
|
|
|
|
|
|
# Loosely based on a UnitTest framework.
|
|
|
|
class SmokeTest
|
|
attr_accessor :parent
|
|
# Run a test and store results.
|
|
def run(evaluation)
|
|
end
|
|
|
|
# Create an html report for this test
|
|
def report_html()
|
|
return "<h1>SmokeTest</h1>"
|
|
end
|
|
end
|
|
|
|
# A suite of smoketests
|
|
class SmokeTestSuite < SmokeTest
|
|
attr_accessor :name, :smoketests, :testids
|
|
|
|
def initialize(name)
|
|
@name = name
|
|
@smoketests = Array.new
|
|
@testids = Array.new
|
|
end
|
|
|
|
def run(evaluation)
|
|
@smoketests.each do |test|
|
|
test.run(evaluation)
|
|
end
|
|
end
|
|
|
|
def report_html()
|
|
result = "<h2>#{@name}</h2>"
|
|
@smoketests.each do |test|
|
|
result += test.report_html()
|
|
end
|
|
return result
|
|
end
|
|
|
|
# Adds the test to the suite.
|
|
def <<(test)
|
|
@smoketests << test
|
|
test.parent = self
|
|
self
|
|
end
|
|
end
|
|
|
|
# ------------------------------------------------------------------------------
|
|
|
|
class SmokeTestFpsResult < SmokeTest
|
|
|
|
def run(evaluation)
|
|
# Tests the results for a number of sessions (in desc order), and grabs
|
|
# the fpsresults for a specific test by name.
|
|
|
|
@name = "Fps"
|
|
@min = Array.new
|
|
@max = Array.new
|
|
@avg = Array.new
|
|
|
|
tests = parent.testids.join ','
|
|
|
|
# session & test ids
|
|
parent.testids.each do |id|
|
|
evaluation.database.execute("SELECT * FROM fpsResult WHERE test == #{id};") do |row|
|
|
@max.push row['max']
|
|
@min.push row['min']
|
|
@avg.push row['avg']
|
|
end
|
|
end
|
|
|
|
if (@avg.length > 1)
|
|
average_growth = ((@avg[0]/@avg[1]))
|
|
|
|
if average_growth > 1.05
|
|
evaluation.errors.push "Slower by #{(average_growth - 1.0) * 100}%"
|
|
end
|
|
|
|
if average_growth < 0.95
|
|
evaluation.messages.push "Faster by #{-(average_growth - 1.0) * 100}%"
|
|
end
|
|
end
|
|
end
|
|
|
|
def report_html()
|
|
report =
|
|
"<h3>#{@name}</h3><table border=\"1\">"
|
|
|
|
# avg
|
|
report += "<tr><td><b>Avg</b></td>"
|
|
@avg[0..10].each do |avg|
|
|
report += "<td>#{avg.to_s}</td>"
|
|
end
|
|
report += "</tr>"
|
|
|
|
# min
|
|
report += "<tr><td><b>Min</b></td>"
|
|
@min[0..10].each do |min|
|
|
report += "<td>#{min.to_s}</td>"
|
|
end
|
|
report += "</tr>"
|
|
|
|
# max
|
|
report += "<tr><td><b>Max</b></td>"
|
|
@max[0..10].each do |max|
|
|
report += "<td>#{max.to_s}</td>"
|
|
end
|
|
report += "</tr>"
|
|
|
|
report += "</table>"
|
|
|
|
return report
|
|
end
|
|
end
|
|
|
|
|
|
|
|
#-------------------------------------------------------------------------------
|
|
# Evaluation of the current state of the database, as a runner for a set of
|
|
# individual tests.
|
|
#
|
|
class Evaluation < SmokeTestSuite
|
|
attr_accessor :errors, :warnings, :messages
|
|
attr_accessor :testids, :sessionids
|
|
attr_accessor :database
|
|
|
|
def initialize(database)
|
|
super("Evaluation")
|
|
@database = database
|
|
@description = Array.new
|
|
@errors = Array.new
|
|
@warnings = Array.new
|
|
@messages = Array.new
|
|
@testids = Array.new
|
|
@sessionids = Array.new
|
|
@average = 0
|
|
@stddev = 0
|
|
end
|
|
|
|
def run()
|
|
|
|
tests = testids.join ','
|
|
@avg = Array.new
|
|
|
|
database.execute("SELECT * FROM fpsResult WHERE test in (#{tests});") do |row|
|
|
@avg.push row['avg']
|
|
end
|
|
|
|
# Overall
|
|
sum = 0
|
|
@avg.inject { |sum,x| sum +x }
|
|
@average = sum/@avg.length.to_f
|
|
variance = @avg.inject{|acc,i|acc +(i-@average)**2} * 1/@avg.length.to_f
|
|
@stddev = Math::sqrt(variance)
|
|
|
|
@smoketests.each do |test|
|
|
test.run(self)
|
|
end
|
|
|
|
end
|
|
|
|
def report_html()
|
|
=begin
|
|
result = "<h1>All tests<h1>"
|
|
result += "<table border=\"1\">"
|
|
result += "<tr><td><b>Avg</b></td><td align=right>%3.2f</td></tr>" % @average
|
|
result += "<tr><td><b>Std</b></td><td align=right>%3.2f</td></tr>" % @stddev
|
|
result += "</table>"
|
|
=end
|
|
|
|
result = "<h1>Individual tests<h1>"
|
|
@smoketests.each do |test|
|
|
result += test.report_html()
|
|
end
|
|
return result
|
|
end
|
|
|
|
def success?()
|
|
@errors.empty?
|
|
end
|
|
|
|
def subject()
|
|
return "subject"
|
|
end
|
|
|
|
def message()
|
|
return "message"
|
|
end
|
|
end
|
|
|
|
#---------------------------------------------------------------------------------------------------------------------------
|
|
#
|
|
# Store associated values from database and capture
|
|
#
|
|
def evaluate_stats()
|
|
evaluation = Evaluation.new(@database)
|
|
|
|
tests = {}
|
|
|
|
# collect all the session from last week, and all the unique tests within those
|
|
# construct the smoke test suite with the required information.
|
|
nowtext = DateTime.now.strftime("%Y-%m-%d %H:%M:%S")
|
|
timestamp = DateTime.now - 1 # a week ago
|
|
timestamptext = timestamp.strftime("%Y-%m-%d %H:%M:%S")
|
|
|
|
@database.results_as_hash = true
|
|
@database.execute("SELECT * FROM session WHERE timestamp BETWEEN '#{timestamptext}' AND '#{nowtext}' ORDER BY timestamp ASC;") do |row|
|
|
evaluation.sessionids.push row['id']
|
|
@database.execute("SELECT * FROM test WHERE session == #{row['id']};") do |row2|
|
|
evaluation.testids.push row['id']
|
|
# New suite if this is a new named smoketest
|
|
testsuite = tests[row2['name']]
|
|
if testsuite.nil?
|
|
testsuite = SmokeTestSuite.new(row2['name'])
|
|
evaluation << testsuite
|
|
tests[row2['name']] = testsuite
|
|
|
|
# Add some specific smoketests
|
|
testsuite << SmokeTestFpsResult.new()
|
|
end
|
|
|
|
# Tell the suite about which tests it occurs in
|
|
testsuite.testids << row2['id']
|
|
end
|
|
end
|
|
|
|
evaluation.run
|
|
|
|
return evaluation
|
|
end
|
|
|
|
|
|
#----------------------------------------------------------------------------
|
|
# Application entry point
|
|
#-----------------------------------------------------------------------------
|
|
|
|
if ( __FILE__ == $0 )
|
|
begin
|
|
g_AppName = File::basename( __FILE__, '.rb' )
|
|
g_ProjectName = ENV['RS_PROJECT']
|
|
g_BranchName = ''
|
|
g_Project = nil
|
|
g_Config = Pipeline::Config.instance()
|
|
|
|
#--------------------------------------------------------------------
|
|
# --- PARSE COMMAND LINE ---
|
|
#--------------------------------------------------------------------
|
|
opts, trailing = OS::Getopt.getopts( OPTIONS )
|
|
if ( opts['help'] )
|
|
puts OS::Getopt.usage( OPTIONS )
|
|
puts ("Press Enter to continue...")
|
|
$stdin.getc( )
|
|
Process.exit!( 1 )
|
|
end
|
|
|
|
puts "#{MSG_PREFIX_GREY} #{$0} #{ARGV.join(" ")}"
|
|
|
|
Process.exit!( 2 ) unless (opts['publish_folder_src'])
|
|
publish_folder_src = opts['publish_folder_src']
|
|
|
|
Process.exit!( 3 ) unless (opts['publish_folder_dst'])
|
|
publish_folder_dst = opts['publish_folder_dst']
|
|
|
|
Process.exit!( 4 ) unless (opts['stats_db'])
|
|
stats_db = opts['stats_db']
|
|
|
|
Process.exit!( 5 ) unless (opts['stats_capture'])
|
|
stats_capture = opts['stats_capture']
|
|
|
|
Process.exit!( 6 ) unless (opts['orig_stats_db_filename'])
|
|
orig_stats_db_filename = opts['orig_stats_db_filename']
|
|
|
|
enable_checkin = false
|
|
enable_checkin = true if (opts['enable_checkin'])
|
|
|
|
#--------------------------------------------------------------------
|
|
# --- PROCESS ANALYSIS ---
|
|
#--------------------------------------------------------------------
|
|
stats_analyser = CodebuilderStatsAnalyser.new(g_ProjectName)
|
|
ret = stats_analyser.process(g_Config, publish_folder_src, publish_folder_dst, stats_capture, stats_db, enable_checkin, orig_stats_db_filename)
|
|
|
|
Process.exit! ret
|
|
|
|
rescue Exception => ex
|
|
$stderr.puts "Error: Unhandled exception: #{ex.message}"
|
|
$stderr.puts "Backtrace:"
|
|
ex.backtrace.each { |m| $stderr.puts "\t#{m}" }
|
|
Process.exit! -1
|
|
end
|
|
end
|