From e3f54d57f62943f9ccdecfde3ef99594e8a0909b Mon Sep 17 00:00:00 2001 From: Ian Jackson Date: Fri, 28 Aug 2020 14:38:17 +0100 Subject: [PATCH] resource reporting: Report host reuse/sharing in job report Compatibility: in principle this might generate erroneous reports which omit sharing/reuse information for allocations made by jobs using older versions of osstest. However, we do not share or reuse hosts across different osstest versions, so this cannot occur. Signed-off-by: Ian Jackson --- sg-report-flight | 331 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 330 insertions(+), 1 deletion(-) diff --git a/sg-report-flight b/sg-report-flight index a1f424c..0413a73 100755 --- a/sg-report-flight +++ b/sg-report-flight @@ -29,9 +29,10 @@ use POSIX; use IPC::Open2; use Data::Dumper; use File::Path; +use Carp; use Osstest; -use Osstest::Executive; +use Osstest::Executive qw(:DEFAULT :colours); our $specflight; our %specver; @@ -1122,6 +1123,68 @@ END return @failures; } +# Machinery for generating WITH ... VALUES common table expressions. +# Use it like this: +# +# 1. $some_accum = {} +# +# 2. valuestable_add_row($some_accum, $val, $val, $val) +# # ^ zero or more times +# +# 3. $qtxt = "WITH\n"; +# @qparams = (); +# valuestable_with(\$qtxt, \@qparams, 'cte_name', +# qw(txtcol1 txtcol2 intcol::integer boolcol::bool ...)); +# +# The resulting CTE table will have the name, and column names, +# you specified. For non-TEXT columns you must specify the type +# because [Postgre]SQL's type inference doesn't work properly here. +# +# valuestable_with will always leave $qtxt ending with ",\n" +# so you can call it multiple times. + +sub valuestable_add_row ($@) { + my ($accum, @row) = @_; + # $accum->{Ncols} + # $accum->{Params}[] + # $accum->{Qtxt} + $accum->{Ncols} //= scalar @row; + confess unless $accum->{Ncols} == @row; + push @{ $accum->{Params} }, @row; + $accum->{Qtxt} //= ''; + $accum->{Qtxt} =~ s/.$/$&,/; + $accum->{Qtxt} .= " (".join(',', ('?',) x @row).")\n"; +} +sub valuestable_with ($$$@) { + my ($qtxtr, $paramsr, $ctename, $accum, @cols) = @_; + my $limit = ''; + $accum->{Qtxt} //= do { + # Oh my god + # select * from (values ); + # => ERROR: syntax error at or near ")" + $limit = 'LIMIT 0'; + " (".join(',', + map { m/::/ ? "NULL::$'" : "NULL" } + @cols).")\n"; + }; + $accum->{Ncols} //= scalar @cols; + confess "$accum->{Ncols} != ".(scalar @cols) + unless $accum->{Ncols} == @cols; + my $cols = join(', ', @cols); + my $colsnotypes = join(', ', map { m/::/ ? $` : $_ } @cols); + $$qtxtr .= <{Qtxt} $limit) $ctename ($colsnotypes)), + +END + push @$paramsr, @{ $accum->{Params} // [ ] }; +} + +sub nullcols { + join ", ", map { m/::/ ? "NULL::$' as $`" : "NULL as $_" } @_; +} + sub htmloutjob ($$) { my ($fi,$job) = @_; return unless defined $htmldir; @@ -1213,6 +1276,272 @@ END Status:$ji->{status}

+END + + # ---------- resource reuse/sharing report ---------- + + # We translate the lifecycle runvars into a set of questions + # for the db. But rather than doing one db query for each + # such question, we aggregate the questions into VALUES + # expressions and ask the db to produce a collated list of + # relevant information. This has fewer round trips. + + my $shareq_elided_accum = {}; + my $shareq_tasks_accum = {}; + my $shareq_main_accum = {}; + foreach my $lc_var_row (@$runvar_table) { + next unless $lc_var_row->{name} =~ m{^(.*_?host)_lifecycle$}; + my $tident = $1; + my $hostname = ($runvar_map{$tident} // next)->{val}; + my $last_uncompr; + my $sort_index; + print DEBUG "SHARE LC $job $tident $lc_var_row->{val}\n"; + foreach (split / /, $lc_var_row->{val}) { + $sort_index++; + if (m/^[\@\+]$/) { + valuestable_add_row $shareq_elided_accum, + $tident, $hostname, undef, $&, $sort_index; + next; + } + if (m/^\[(\d+)\]$/) { # elided + valuestable_add_row $shareq_elided_accum, + $tident, $hostname, $1, undef, $sort_index; + next; + } + my $olive = s/^\+//; + if (m/^\?(\d+)$/) { # tasks + valuestable_add_row $shareq_tasks_accum, + $tident, $hostname, $olive+0, $1, $sort_index; + next; + } + my $oisprep = s/^\@//; + s{^\d+$}{ join ":$&", @$last_uncompr }e if $last_uncompr; + if (my ($tprefix, $oflight, $ojob, + $ostepno, $tsuffix, $oident) = + m{^((?:(\d+)\.)?([^:]+)?)\:(\d+)((?:,([^:]+))?)$}) { + # main + $last_uncompr = [ $tprefix, $tsuffix ]; + $oflight ||= $specflight; + $ojob ||= $job; + $oident ||= 'host'; + valuestable_add_row $shareq_main_accum, + $tident, $hostname, $oflight, $ojob, $ostepno, + $oisprep+0, $oident, $olive+0; + next; + } + confess "$tident $hostname $_ ?"; + } + } + my @shareq_params; + my $shareq_txt = <execute(@shareq_params); + + my $share_any; + my $altcolour=1; + while (my $srow = $shareq->fetchrow_hashref()) { + print DEBUG "SHARE SROW ".Dumper($srow); + print H <Task(s) which might have affected this job's host(s) +

+ + +END + my $bgcolour = report_altcolour($altcolour ^= 1); + printf H < + + +END + $srow->{tident}, + "$c{ResultsHtmlPubBaseUrl}/host/$srow->{hostname}.html", + $srow->{hostname}; + my $rel = $srow->{olive} ? + "" + : $srow->{prep_started} ? + "" + : + ""; + if (defined $srow->{flight}) { + my $furl = "$c{ReportHtmlPubBaseUrl}/$srow->{flight}/"; + my $jurl = "$furl/$srow->{job}/info.html"; + if ($srow->{flight} != $specflight) { + printf H <%s + +END + $furl, $srow->{flight}, + $jurl, $srow->{job}; + } elsif ($srow->{job} ne $job) { + printf H <this + +END + $jurl, $srow->{job}; + } else { + printf H < + + +END + } + printf H <%s + +END + encode_entities($srow->{oidents}), + map { $_ ? show_abs_time($_) : '' } + $srow->{prep_started}, + $srow->{rest_started}, + !$srow->{olive} && $srow->{finished}; + my $info = report_run_getinfo($srow); + print H <{ColourAttr}>$info->{Content} +END + } elsif (defined $srow->{elided}) { + printf H <{elided}; + +END + } elsif (defined $srow->{elided_sigil}) { + printf H < +this job incomplete, unknown number of other jobs elided + +END + } elsif (defined $srow->{taskid}) { + printf H <?%s: %s +END + $srow->{taskid}, + report_rogue_task_description($srow); + } else { + confess Dumper($srow)." ?"; + } + print H < +END + } + print H < END print H <
role
(here) +		hostname +rel. +flight +job +role(s)
(there) +		install / prep.
started +		use
started +		last step
ended +		job
status +
%s%sshareprep.reuse%s%sthisthis%s%s%s%d earlier job(s) elided