From e7c23e9b24c39240e41989f7b6f5e5d2eaaba7d9 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Tue, 14 Jul 2015 15:32:54 +0100 Subject: [PATCH] ms-flights-summary: Produce an HTML report of all active flights Jobs are categorised by a new ->Job field. This is added by ts-hosts-alllocate-Executive and propagated by the planner after recent patches. It contains $flight.$job. Jobs which do not include this are anonymous and are listed separately, using the resource name and info field (if present) as the job name. Signed-off-by: Ian Campbell Acked-by: Ian Jackson --- ms-flights-summary | 426 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 426 insertions(+) create mode 100755 ms-flights-summary diff --git a/ms-flights-summary b/ms-flights-summary new file mode 100755 index 0000000..fb03a26 --- /dev/null +++ b/ms-flights-summary @@ -0,0 +1,426 @@ +#!/usr/bin/perl -w + +use strict qw(vars refs); + +use Osstest; +use Osstest::Executive; + +use POSIX; +use Data::Dumper; +use HTML::Entities; + +# $flights{NNNN} = { +# Nr => NNNN, +# Stats => { +# Pass => , +# Fail => , +# ... +# }, +# Started => ..., +# Blessing => ..., +# Branch => ..., +# Intended => ..., +# ExpectedEnd => ..., +# Jobs => { +# JNAME1 => { +# Reso => { +# RNAME1 => { +# Start => ..., +# End => ..., +# Info => ..., +# }, +# RNAME2 => { ... }, +# }, +# OverallTime => { Start => ..., End => ... }, +# Receipe => ..., +# Status => pass|fail|..., +# }, +# JNAME2 => { ... }, +# }, +# TotalJobs => NNN, +# UnqueuedJobs => MMM, # Not queued +# }; +our %flights; + +# As for $flights{NNN}{Jobs} but each Job only has the Reso and +# OverallTime keys, plus an additional Anon => 1 +our %anon_jobs; + +# As for $flights{NNNN} => Stats, but cummulative for all flights. +our %global_stats; +our $tot_jobs = 0; + +our $plan; + +sub get_current_plan ($) { + my ($fn) = @_; + open P, $fn or die $!; + local $/; + my $plandump=

; + P->error and die $!; + close P or die $!; + $plan= eval $plandump; + #print STDERR Dumper($plan); +} + +# Find all the flights referenced by an Event and insert into %flights. +sub enumerate_flights() { + while (my ($reso,$evts) = each %{ $plan->{Events} }) { + foreach my $evt ( @{$evts} ) { + next unless $evt->{Type} =~ m/^(Start|End)$/; + next unless $evt->{Job}; + $evt->{Job} =~ m/^([0-9]+)\.(.*)/ or die; + + my $f = $1; + + next if $flights{$f}; + + my $flightinfo= $dbh_tests->selectrow_hashref(< $f, + Stats => {}, + Jobs => {} }; + foreach my $fld (qw(started blessing branch intended)) { + $flights{$f}->{ucfirst($fld)}= $flightinfo->{$fld}; + }; + $flights{$f}->{ExpectedEnd} = 0; + } + } +} + +# Enumerate all jobs of every known flight and populate the +# corresponding ->{Jobs}. +sub enumerate_jobs($) { + my ($f) = @_; + + $f->{Jobs} = {}; + + my $jobs= $dbh_tests->selectall_arrayref(< {} }); + SELECT job,recipe,status FROM jobs + WHERE flight=$f->{Nr} +END + + for my $row (@{$jobs}) { + $f->{Jobs}{$row->{job}} = + { + Status => $row->{status}, + Recipe => $row->{recipe}, + Reso => {}, + }; + $tot_jobs++; + } +} + +# Gather statistics around number of pass/fail/etc jobs in each +# flight. +sub gather_stats($) { + my ($f) = @_; + + my $stats= $dbh_tests->selectall_arrayref(<{Nr} + GROUP BY status +END + for my $row (@{$stats}) { + my ($stat,$count) = @$row; + $f->{Stats}{lc($stat)} = $count; + $global_stats{lc($stat)} += $count; + } + + $f->{NrJobs} = keys %{$f->{Jobs}}; + $f->{UnqueuedJobs} = $f->{NrJobs} - ($f->{Stats}{queued}//0); +} + +sub sort_stats($) { + my ($stats) = @_; + my %so = ( + queued => 1, + preparing => 2, + blocked => 3, + running => 4, + pass => 5, + fail => 6, + broken => 7, + ); + return sort { ($so{$a}//0) <=> ($so{$b}//0) } (keys %{$stats}); +} + +sub add_event($$$$$) { + my ($job,$reso,$type,$info,$time) = @_; + + die unless $type =~ m/^(Start|End)/; + + $job->{OverallTime} //= {}; + + $job->{Reso}{$reso} //= {}; + $job->{Reso}{$reso}{$type} = $time; + + $job->{Reso}{$reso}{Info} = $info + if $type eq "Start"; + + my $cmp = $type eq "Start" ? + sub { $_[0] < $_[1] ? $_[0] : $_[1] } : + sub { $_[0] > $_[1] ? $_[0] : $_[1] }; + + $job->{OverallTime}{$type} //= $time; + + $job->{OverallTime}{$type} = + $cmp->($time, $job->{OverallTime}{$type}); +} + +# Longest common prefix. First argument is cumulative over several +# iterations and therefore we simply shorten it until it is a common +# prefix of the second. +sub update_lcp ($$) { + my ($a,$b) = @_; + + return $b unless $a; + return $a unless $b; + + chop $a while $b !~ m/^\Q$a\E/; + return $a; +} + +# Walk all events in the plan and update the corresponding flight/job +# with the timespan. Events relating to unknown jobs are added to +# %anon_jobs. +sub gather_events() { + while (my ($reso,$evts) = each %{ $plan->{Events} }) { + foreach my $evt ( @{$evts} ) { + my ($f,$job); + next unless $evt->{Type} =~ m/^(Start|End)$/; + if ( $evt->{Job} ) { + my ($fnum,$j); + $evt->{Job} =~ m/^([0-9]+)\.(.*)/ or die; + ($fnum,$j) = ($1,$2); + goto anon unless $flights{$fnum}; + $f = $flights{$fnum}; + goto anon unless $f->{Jobs}{$j}; + $job = $f->{Jobs}{$j}; + } else { + anon: + # Fake up a name from the $reso and the event's info + # field (if available). + my $anon_job = join(" ", ($reso,$evt->{Info})); + $anon_jobs{$anon_job} //= { Reso => {}, Anon => 1 }; + $job = $anon_jobs{$anon_job}; + } + + my $time = $evt->{Time}; + + add_event($job, $reso, $evt->{Type}, $evt->{Info}, $time); + + $f->{Info} = update_lcp($f->{Info}, $evt->{Info}) if $f; + + if ($f && $evt->{Type} eq "End") { + $f->{ExpectedEnd} = + $time > $f->{ExpectedEnd} ? + $time : $f->{ExpectedEnd}; + } + } + } +} + +############ + +my @cols = ("Job", "Status", "Resource", "Scheduled"); + +sub fmttime($) +{ + my ($t) = @_; + return "Unknown" if !$t; + return strftime("%Y-%b-%d %a %H:%M:%S", gmtime $t); +} + +sub fmt_timespan($) { + my ($s) = @_; + + return undef unless $s; + + if ( $s->{Start} || $s->{End} ) { + return join(" — ", + map { encode_entities(fmttime($s->{$_})) } + qw(Start End)); + } else { + return "???"; + } +} + +sub currently_running($) { + my ($info) = @_; + + return 0 unless $info->{OverallTime}{Start}; + return 0 unless $info->{OverallTime}{End}; + + return 0 if $info->{OverallTime}{Start} > $plan->{Start}; + return 0 if $info->{OverallTime}{End} < $plan->{Start}; + + return 1; +} + +sub cols_hdr() { + printf("\n"); + printf(" %s\n", encode_entities($_)) foreach @cols; + printf("\n"); +} + +sub flight_hdr($) { + my $text = encode_entities(shift); + printf(" $text\n", scalar @cols); +} + +sub cell($;$$$) { + my ($text,$colourattr,$fontattr,$tag) = @_; + $text //= ''; + $colourattr = $colourattr ? $colourattr : ''; + $text = "$text" if $fontattr; + $text = "<$tag>$text" if $tag; + printf(" $text\n"); +} + +sub do_one_job($$$$) { + my ($alt,$fl,$job,$info) = @_; + my $status = $info->{Status}//''; + + my $bgcolour = report_altcolour(${$alt}); + + my @resos = sort keys %{ $info->{Reso} }; + + my ($resos,$spans); + my $resopfx = ''; + if (@resos > 1) { + $resos = "Overall
\n "; + $resopfx = "— "; + $spans = fmt_timespan($info->{OverallTime})."
\n "; + } + + $resos .= join "
\n ", map { "$resopfx$_" } @resos; + $spans .= join "
\n ", map { fmt_timespan($info->{Reso}{$_}) } @resos; + + print "\n"; + + my $tag = $status eq "running" ? "b" : undef; + + $tag = "b" if $info->{Anon} && currently_running($info); + + cell(encode_entities($job), undef, undef, $tag); + + # Anonymous/rogue jobs may not have a flight or status + if ($fl && $status) { + my $info = report_run_getinfo({flight=>$fl, + job=>$job, + status=>$status}); + cell($info->{Content}, $info->{ColourAttr}, undef, $tag); + } else { + cell("(unknown)"); + } + cell($resos); + cell($spans, + $resos ? + "bgcolor=".($status eq "running" ? "#882222" : "#448844") : + "", + "color=\"#ffffff\""); + + print "\n"; + ${$alt} ^= 1; +} + +########### + +@ARGV == 1 or die "need a data.pl"; + +# Required by parts of Osstest::Executive. +open DEBUG, ">/dev/null"; + +csreadconfig(); + +get_current_plan($ARGV[0]); + +enumerate_flights(); + +foreach my $f (keys %flights) { + enumerate_jobs($flights{$f}); + gather_stats($flights{$f}); +} + +gather_events(); + +printf("

Report at ".fmttime(time)." based on plan at ".fmttime($plan->{Start})."

\n"); + +printf("

Overview

\n"); +printf("

%d flight(s) consisting of %s job(s)
%s
%s anonymous/rogue job(s)

\n", + scalar keys %flights, $tot_jobs, + join(" + ", map { "$global_stats{$_} $_" } (sort_stats(\%global_stats))), + scalar keys %anon_jobs); + +printf("\n"); +printf("\n"); +printf(" \n") foreach ("Flight", "Branch", "Blessing", + "(Active+Complete)/Total Jobs", "Counts", + "End of current phase"); +printf("\n"); + +my $alt = 0; +foreach my $f (sort keys %flights) { + my $fi = $flights{$f}; + + my $bgcolour = report_altcolour($alt); + + print "\n"; + print " \n"; + print " \n"; + print " \n"; + print " \n"; + print " \n"; + print " \n"; + print "\n"; + $alt ^= 1; +} +printf("
$_
$f$fi->{Branch}$fi->{Intended}$fi->{UnqueuedJobs}/$fi->{NrJobs}". + join(" + ", map { "$fi->{Stats}{$_} $_" } (sort_stats(\%{$fi->{Stats}}))) + ."".fmttime($fi->{ExpectedEnd})."
\n"); + + +printf("

Details

\n"); +printf("\n"); + +my $first = 1; +foreach my $f (sort keys %flights) { + my $fi = $flights{$f}; + + $alt = 0; + + printf ("\n"); + + cols_hdr(); + + do_one_job(\$alt,$fi->{Nr},$_, $fi->{Jobs}{$_}) + foreach sort { $a cmp $b } keys %{$fi->{Jobs}}; +} +print "\n"; + +printf ("\n"); +cols_hdr(); +$alt = 0; +foreach my $j (sort keys %anon_jobs) { + do_one_job(\$alt,undef,$j, $anon_jobs{$j}); +} + +print "
\n \n", scalar @cols); + print (" \n") if !$first; + $first = 0; + print("\n"); + flight_hdr("Flight: $f [$fi->{Branch} $fi->{Intended}]"); + flight_hdr("Common info (active jobs only): $fi->{Info}") if $fi->{Info}; + flight_hdr("Started: ".fmttime($fi->{Started})); + flight_hdr("Current phase ($fi->{UnqueuedJobs}/$fi->{NrJobs} jobs)". + " expected end: ".fmttime($fi->{ExpectedEnd})); + flight_hdr("Jobs: $fi->{NrJobs} = ". + join(" + ", map { "$fi->{Stats}{$_} $_" } (sort_stats(\%{$fi->{Stats}}))) + ); + print ("
 
\n
\n \n", scalar @cols); +print (" \n"); +flight_hdr("Anonymous/Rogue Jobs"); +print ("
 
\n
\n"; -- 2.39.5