From: Lars Kurth Date: Sat, 29 Jul 2017 09:53:44 +0000 (+0100) Subject: Laid the foundation for verbose output X-Git-Url: http://xenbits.xensource.com/gitweb?a=commitdiff_plain;h=fafd098ad9dd4d3d548a83e3f6411649c45ec86f;p=people%2Flarsk%2Fxen-release-scripts.git Laid the foundation for verbose output --- diff --git a/match-xsa b/match-xsa index 5916558..93e3fc4 100755 --- a/match-xsa +++ b/match-xsa @@ -59,7 +59,7 @@ if ($GETLOGS) { my $short="$VERSION$MAJOR$SINCE-$UNTIL"; # Set up XSADIR and XSAFETCH, if not set -if (!$XSADIR && $SMART!=0) { +if (!$XSADIR) { $XSAFETCH = "https://xenbits.xenproject.org/xsa/"; $XSADIR = "../xsaweb"; system("rm -rf $XSADIR"); @@ -115,6 +115,15 @@ while (my $line = <$data>) { close($data); $XSAs = $i-1; +# Process advisories +my %XSA_VS; +my %XSA_RESVERBATIM; +my %XSA_RES; +my @XSA_PATCHFORVERSION; +my @XSA_REGEXFORVERSION; + +processadvisoryfiles(); + # Get logs my @LOGXEN; my @LOGXEN_hash; @@ -509,3 +518,215 @@ sub normalize_diff { return join "+++ ", @newplines; } + +# From http://perldoc.perl.org/perlfaq4.html# +# How-can-I-remove-duplicate-elements-from-a-list-or-array%3f +sub uniq { + + my %seen; + grep !$seen{$_}++, @_; +} + +sub advisorysnippet { + + my $advisory = shift; + my $headline = shift; + my $endline = shift; + + my $regex1 = "^".$headline; + my $regex2 = "^".$endline; + my $offset = length ($regex1) * 2; + + my $start; + my $end = -1; + my $string; + + if ($advisory =~ qr/$regex1/mp) { + $start = $-[0]; + + my $snippet = substr $advisory, $start + $offset ; + + if ($snippet =~ qr/$regex2/mp) { + $end = $-[0] + $offset ; + } + $string = substr $advisory, $start, $end; + # Now pop last line, as we don't want it + my @lines = split "\n", $string; + pop @lines; + return join "\n", @lines; + + return $string; + } + + return "No section '$headline'\n"; +} + +sub processadvisoryfiles { + + my $xsa_dir = $XSADIR; + #my $version = $VERSION.".".$MAJOR; + #note: 4.6 contains [12], 4.5 , + my $version = "4."; + + print "* $version\n"; + + # Get a unique array of XSA numbers in the file + my @xsa = uniq(@XSA); + + my $i; + foreach $i (@xsa) { + + # If needed, fetch advisory-XXX.html files from + # https://xenbits.xenproject.org/xsa/ + + if ($XSAFETCH) { + system("wget --quiet --no-check-certificate -P $xsa_dir ". + "$XSAFETCH"."advisory-$i.html"); + + # Ensure that files from xsa.git and fetched files have the same + # name. Can ignore the html, as the relevant text is the same + # in the generated html and the file in xsa.git + rename "$xsa_dir/advisory-$i.html", + "$xsa_dir/advisory-$i.txt"; + } + + # Read the advisory + my $advisory = read_file("$xsa_dir/advisory-$i.txt") or die; + + # Now we get the section "VULNERABLE SYSTEMS" and save it + my $vulnerable_systems = advisorysnippet($advisory, + "VULNERABLE SYSTEMS", + "==="); + $XSA_VS{$i} = $vulnerable_systems; + + # Now we get the section "RESOLUTION" and save it + my $resolution; + if ($i >= 119 ) { + # At this point we introduced "DEPLOYMENT UNDER EMBARGO" + $resolution = advisorysnippet($advisory, + "RESOLUTION", + "==="); + } else { + $resolution = advisorysnippet($advisory, + "RESOLUTION", + "\-\-\-"); + } + $XSA_RESVERBATIM{$i} = $resolution; + + # Filter out all the relevant lines with patches + # Note that older XSAs use free-form text + my @matches = ( $resolution =~ /^xsa[^\n]+/gm ); + $XSA_RES{$i} = join("\n", @matches)."\n"; + + # Filter out all the relevant lines with patches for a version + my $j; + my @version_matches; + foreach $j (@matches) { + # Split line into 2 + # (the descriptor should always be in the second part) + my @lines = split qr/[ ][ ]+/mp, $j; + + if ($lines[1] && index($lines[1], $version) != -1) { + push @version_matches, split ", ", $lines[0]; + } + } + # Remove empty entries + @version_matches = grep { defined && m/[^\s]/ } @version_matches; + push @XSA_PATCHFORVERSION, @version_matches; + } + @XSA_PATCHFORVERSION = uniq(@XSA_PATCHFORVERSION); + + # To get the regular expression replace + # . with \. + # * with [^\t\n .]+ + # / with \/ + # + # Expressions such as + # {01..06} and {1,2} need to be manually expanded + # {1,2} => [12], {1,2,3} => [123], {1,3} => [13] + # {01,02} => [012][[012] + # {01..06} => 0[123456] + # This needs to happen *before* . is replaced + # + # Expressions such as the following are already search patterns + # [12] + # + foreach $i (@XSA_PATCHFORVERSION) { + my $tmp = $i; + + $tmp =~ s/\//\\\//g; + $tmp =~ s/\*/[^\\t\\n .]+/g; + + # Find brackets for processing {[0123456789.,]+} + my $regex = qr/{[0123456789.,]+}/p; + if ($tmp =~ /$regex/g) { + # Use index, as \$-[0] and \$+[0] isn't working + my $s = index($tmp, "{"); + my $e = index($tmp, "}"); + + if ($s != -1 && $e != -1) { + + my $start = substr $tmp, 0, $s; + my $middle = substr $tmp, $s+1, $e-$s-1; + my $end = substr $tmp, $e+1; + + #print "<<<<".$tmp."\n"; + #print $start."\n"; + #print "----\n"; + #print $middle."\n"; + #print "----\n"; + #print $end."\n"; + #print ">>>>\n"; + + # Process the content + my $leadingzero; + my @numbers; + if ( index($middle, "..") != -1 ) { + my $t = index ($middle, ".."); + my $first = substr $middle, 0, $t; + my $last = substr $middle, $t+2; + print "1st: ".$first."\n"; + print "last: ".$last."\n"; + + if ( substr($first, 0, 1) eq "0" and $first != 0 ) { + $leadingzero = 1; + } + $middle = substr "0123456789", $first, $last-$first+1; + + } elsif ( index($middle, ",") != -1 ) { + @numbers = split ",", $middle; + + my $first = $numbers[0]; + + if ( substr($first, 0, 1) eq "0" and $first != 0 ) { + $leadingzero = 1; + @numbers = split "0", join("",@numbers); + } + $middle = join("",@numbers); + } + + if ($leadingzero) { + $middle = "0[".$middle."]"; + } else { + $middle = "[".$middle."]"; + } + + # Re-assemble + $tmp = $start.$middle.$end; + #print "OUTPUT: ".$tmp."\n\n"; + } + } + + $tmp =~ s/\./\\./g; + push @XSA_REGEXFORVERSION, $tmp; + } + @XSA_REGEXFORVERSION = uniq(@XSA_REGEXFORVERSION); + + # print join("\n", @XSA_REGEXFORVERSION); + + # TODO: check whether the found matches can be found + # Note: For testing at the beginning of the function set + # $version to "4." or "4.6" contains [12], "4.5", etc. + + # TODO: maybe make @XSA_REGEXFORVERSION a hash (see later) +}