#!/usr/bin/perl # # make_tests.pl - generate WPT test cases from the testable statements wiki # # This script assumes that a wiki has testable statement entries # in the format described by the specification at # https://spec-ops.github.io/atta-api/index.html # # usage: make_tests.pl -f file | -w wiki_title | -s spec -d dir use strict; use IO::String ; use JSON ; use MediaWiki::API ; use Getopt::Long; my %specs = ( "aria11" => { title => "ARIA_1.1_Testable_Statements", specURL => "https://www.w3.org/TR/wai-aria11/", dir => "aria11" }, "svg" => { title => "SVG_Accessibility/Testing/Test_Assertions_with_Tables_for_ATTA", specURL => "https://www.w3.org/TR/svg-aam-1.0/", dir => "svg", fragment => '' } ); my @apiNames = qw(UIA MSAA ATK IAccessible2 AXAPI); my $apiNamesRegex = "(" . join("|", @apiNames) . ")"; # the suffix to attach to the automatically generated test case names my $theSuffix = "-manual.html"; # dir is determined based upon the short name of the spec and is defined # by the input or on the command line my $file = undef ; my $spec = undef ; my $wiki_title = undef ; my $dir = undef; my $theSpecFragment = "%code%"; my $preserveWiki = ""; my $fake = 0; my $result = GetOptions( "f|file=s" => \$file, "p=s" => \$preserveWiki, "w|wiki=s" => \$wiki_title, "s|spec=s" => \$spec, "f|fake" => \$fake, "d|dir=s" => \$dir) || usage(); my $wiki_config = { "api_url" => "https://www.w3.org/wiki/api.php" }; my $io ; our $theSpecURL = ""; if ($spec) { print "Processing spec $spec\n"; $wiki_title = $specs{$spec}->{title}; $theSpecURL = $specs{$spec}->{specURL}; if (!$dir) { $dir = "../" . $specs{$spec}->{dir}; } $theSpecFragment = $specs{$spec}->{fragment}; } if (!$dir) { $dir = "../raw"; } if (!-d $dir) { print STDERR "No such directory: $dir\n"; exit 1; } if ($file) { open($io, "<", $file) || die("Failed to open $file: " . $@); } elsif ($wiki_title) { my $MW = MediaWiki::API->new( $wiki_config ); $MW->{config}->{on_error} = \&on_error; sub on_error { print "Error code: " . $MW->{error}->{code} . "\n"; print $MW->{error}->{stacktrace}."\n"; die; } my $page = $MW->get_page( { title => $wiki_title } ); my $theContent = $page->{'*'}; print "Loaded " . length($theContent) . " from $wiki_title\n"; if ($preserveWiki) { if (open(OUTPUT, ">$preserveWiki")) { print OUTPUT $theContent; close OUTPUT; print "Wiki preserved in $preserveWiki\n"; exit 0; } else { print "Failed to create $preserveWiki. Terminating.\n"; exit 1; } } $io = IO::String->new($theContent); } else { usage() ; } # Now let's walk through the content and build a test page for every item # # iterate over the content # my $io ; # open($io, "<", "raw") ; # data structure: # # steps is a list of steps to be performed. # Each step is an object that has a type property and other properties based upon that type. # # Types include: # # 'test' - has a property for each ATAPI for which there are tests # 'attribute' - has a property for the target id, attribute name, and value # 'event' - has a property for the target id and event name my $state = 0; # between items my $theStep = undef; my $current = ""; my $theCode = ""; my $theAttributes = {}; my @steps ; my $theAsserts = {} ; my $theAssertCount = 0; my $theAPI = ""; my $typeRows = 0; my $theType = ""; my $theName = ""; my $theRef = ""; my $lineCounter = 0; my $skipping = 0; our $testNames = {} ; while (<$io>) { if (m//) { last; } $lineCounter++; # look for state if (m/^SpecURL: (.*)$/) { $theSpecURL = $1; $theSpecURL =~ s/^ *//; $theSpecURL =~ s/ *$//; } if ($state == 5 && m/^; \/\/ (.*)/) { # we found another test inside a block # we were in an item; dump it build_test($current, $theAttributes, $theCode, \@steps, $theSpecFragment) ; # print "Finished $current and new subblock $1\n"; $state = 1; $theAttributes = {} ; $theAPI = ""; @steps = (); $theCode = ""; $theAsserts = undef; $theName = ""; } elsif (m/^=== +(.*[^ ]) +===/) { if ($state != 0) { if ($skipping) { print STDERR "Flag on assertion $current; skipping\n"; } else { # we were in an item; dump it build_test($current, $theAttributes, $theCode, \@steps, $theSpecFragment) ; # print "Finished $current\n"; } } $state = 1; $current = $1; $theAttributes = {} ; @steps = (); $theCode = ""; $theAsserts = undef; $theAPI = ""; $theName = ""; if ($current =~ m/\(/) { # there is a paren in the name -skip it $skipping = 1; } else { $skipping = 0; } } if ($state == 1) { if (m/
/) {
# we are now in the code block
$state = 2;
next;
} elsif (m/==== +(.*) +====/) {
# we are in some other block
$theName = lc($1);
$theAttributes->{$theName} = "";
next;
}
if (m/^Reference: +(.*)$/) {
$theAttributes->{reference} = $theSpecURL . "#" . $1;
} elsif ($theName ne "") {
# accumulate whatever was in the block under the data for it
chomp();
$theAttributes->{$theName} .= $_;
} elsif (m/TODO/) {
$state = 0;
}
}
if ($state == 2) {
if (m/<\/pre>/) {
# we are done with the code block
$state = 3;
} else {
if (m/^\s/ && !m/if given/) {
# trim any trailing whitespace
$theCode =~ s/ +$//;
$theCode =~ s/\t/ /g;
$theCode .= $_;
# In MediaWiki, to display & symbol escapes as literal text, one
# must use "&&" for the "&" character. We need to undo that.
$theCode =~ s/&(\S)/&$1/g;
}
}
} elsif ($state == 3) {
# look for a table
if (m/^\{\|/) {
# table started
$state = 4;
}
} elsif ($state == 4) {
if (m/^\|-/) {
if ($theAPI
&& exists($theAsserts->{$theAPI}->[$theAssertCount])
&& scalar(@{$theAsserts->{$theAPI}->[$theAssertCount]})) {
$theAssertCount++;
}
# start of a table row
if ($theType ne "" && $typeRows) {
# print qq($theType typeRows was $typeRows\n);
# we are still processing items for a type
$typeRows--;
# populate the first cell
$theAsserts->{$theAPI}->[$theAssertCount] = [ $theType ] ;
} else {
$theType = "";
}
} elsif (m/^\|\}/) {
# ran out of table
$state = 5;
# adding processing for additional block types
# a colspan followed by a keyword triggers a start
# so |colspan=5|element triggers a new collection
# |colspan=5|attribute triggers the setting of an attribute
} elsif (m/^\|colspan="*([0-9])"*\|([^ ]+) (.*)$/) {
my $type = $2;
my $params = $3;
my $obj = {} ;
if ($type eq "attribute") {
if ($params =~ m/([^:]+):([^ ]+) +(.*)$/) {
$obj = {
type => $type,
element => $1,
attribute => $2,
value => $3
};
$theStep = undef;
push(@steps, $obj);
} else {
print STDERR "Malformed attribute instruction at line $lineCounter: " . $_ . "\n";
}
} elsif ($type eq "event") {
if ($params =~ m/([^:]+):([^ ]+).*$/) {
$obj = {
type => $type,
element => $1,
value => $2
};
$theStep = undef;
push(@steps, $obj);
} else {
print STDERR "Malformed event instruction at line $lineCounter: " . $_ . "\n";
}
} elsif ($type eq "element") {
$obj = {
type => "test",
element => $3
};
push(@steps, $obj);
$theStep = scalar(@steps) - 1;
$theAsserts = $steps[$theStep];
} else {
print STDERR "Unknown operation type: $type at line " . $lineCounter . "; skipping.\n";
}
} elsif (m/($apiNamesRegex)$/) {
my $theString = $1;
$theString =~ s/ +$//;
$theString =~ s/^ +//;
if ($theString eq "IA2") {
$theString = "IAccessible2" ;
}
my $rows = 1;
if (m/^\|rowspan="*([0-9])"*\|(.*)$/) {
$rows = $1
}
if (grep { $_ eq $theString } @apiNames) {
# we found an API name - were we already processing assertions?
if (!$theAsserts) {
# nope - now what?
$theAsserts = {
type => "test",
element => "test"
};
push(@steps, $theAsserts);
}
$theAssertCount = 0;
# this is a new API section
$theAPI = $theString ;
$theAsserts->{$theAPI} = [ [] ] ;
$theType = "";
} else {
# this is a multi-row type
$theType = $theString;
$typeRows = $rows;
# print qq(Found multi-row $theString for $theAPI with $typeRows rows\n);
$typeRows--;
# populate the first cell
if ($theAPI
&& exists($theAsserts->{$theAPI}->[$theAssertCount])
&& scalar(@{$theAsserts->{$theAPI}->[$theAssertCount]})) {
$theAssertCount++;
}
$theAsserts->{$theAPI}->[$theAssertCount] = [ $theType ] ;
}
} elsif (m/^\|(.*)$/) {
my $item = $1;
$item =~ s/^ *//;
$item =~ s/ *$//;
$item =~ s/^['"]//;
$item =~ s/['"]$//;
# add into the data structure for the API
if (!exists $theAsserts->{$theAPI}->[$theAssertCount]) {
$theAsserts->{$theAPI}->[$theAssertCount] = [ $item ] ;
} else {
push(@{$theAsserts->{$theAPI}->[$theAssertCount]}, $item);
}
}
}
};
if ($state != 0) {
build_test($current, $theAttributes, $theCode, \@steps, $theSpecFragment) ;
print "Finished $current\n";
}
exit 0;
# build_test
#
# create a test file
#
# attempts to create unique test names
sub build_test() {
my $title = shift ;
my $attrs = shift ;
my $code = shift ;
my $steps = shift;
my $frag = shift ;
if ($title eq "") {
print STDERR "No name provided!";
return;
}
if ($frag ne "") {
$frag =~ s/%code%/$code/;
$code = $frag;
}
$code =~ s/ +$//m;
$code =~ s/\t/ /g;
my $title_reference = $title;
if ($code eq "") {
print STDERR "No code for $title; skipping.\n";
return;
}
if ( $steps eq {}) {
print STDERR "No assertions for $title; skipping.\n";
return;
}
my $testDef =
{ "title" => $title,
"steps" => []
};
my $stepCount = 0;
foreach my $asserts (@$steps) {
$stepCount++;
my $step =
{
"type" => $asserts->{"type"},
"title"=> "step " . $stepCount,
};
if ($asserts->{type} eq "test") {
# everything in the block is about testing an element
$step->{"element"} = ( $asserts->{"element"} || "test" );
my $tests = {};
if ($fake) {
$tests->{"WAIFAKE"} = [ [ "property", "role", "is", "ROLE_TABLE_CELL" ], [ "property", "interfaces", "contains", "TableCell" ] ];
}
foreach my $name (@apiNames) {
if (exists $asserts->{$name} && scalar(@{$asserts->{$name}})) {
$tests->{$name} = $asserts->{$name};
}
};
$step->{test} = $tests;
} elsif ($asserts->{type} eq "attribute") {
$step->{type} = "attribute";
$step->{element} = $asserts->{"element"};
$step->{attribute} = $asserts->{"attribute"};
$step->{value} = $asserts->{value};
} elsif ($asserts->{type} eq "event") {
$step->{type} = "event";
$step->{element} = $asserts->{"element"};
$step->{event} = $asserts->{value};
} else {
print STDERR "Invalid step type: " . $asserts->{type} . "\n";
next;
}
push(@{$testDef->{steps}}, $step);
}
# populate the rest of the test definition
if (scalar(keys(%$attrs))) {
while (my $key = each(%$attrs)) {
# print "Copying $key \n";
$testDef->{$key} = $attrs->{$key};
}
}
if (exists $attrs->{reference}) {
$title_reference = "" . $title_reference . "" ;
}
my $testDef_json = to_json($testDef, { canonical => 1, pretty => 1, utf8 => 1});
my $fileName = $title;
$fileName =~ s/\s*$//;
$fileName =~ s/\///g;
$fileName =~ s/\s+/_/g;
$fileName =~ s/[,=:]/_/g;
$fileName =~ s/['"]//g;
my $count = 2;
if ($testNames->{$fileName}) {
while (exists $testNames->{$fileName . "_$count"}) {
$count++;
}
$fileName .= "_$count";
}
$fileName = lc($fileName);
$testNames->{$fileName} = 1;
$fileName .= $theSuffix;
my $template = qq(
$title
This test examines the ARIA properties for $title_reference.
$code
);
my $file ;
if (open($file, ">", "$dir/$fileName")) {
print $file $template;
print $file "\n";
close $file;
} else {
print STDERR qq(Failed to create file "$dir/$fileName" $!\n);
}
return;
}
sub usage() {
print STDERR q(usage: make_tests.pl -f file | -w wiki_title | -s spec [-n -v -d dir ]
-s specname - the name of a spec known to the system
-w wiki_title - the TITLE of a wiki page with testable statements
-f file - the file from which to read
-n - do nothing
-v - be verbose
-d dir - put generated tests in directory dir
);
exit 1;
}
# vim: ts=2 sw=2 ai: