#! /usr/bin/perl -w use strict; $| = 1; # $Id: reporter.t 1133 2007-12-24 00:45:59Z abeltje $ use File::Spec::Functions; my $findbin; use File::Basename; BEGIN { $findbin = dirname $0; } use lib $findbin; use TestLib; use File::Copy; my $verbose = exists $ENV{SMOKE_VERBOSE} ? $ENV{SMOKE_VERBOSE} : 0; my $showcfg = 0; use Test::More tests => 77; use_ok 'Test::Smoke::Reporter'; my $config_sh = catfile( $findbin, 'config.sh' ); { create_config_sh( $config_sh, version => '5.6.1' ); my $reporter = Test::Smoke::Reporter->new( ddir => $findbin, v => $verbose, outfile => '', showcfg => $showcfg, cfg => \( my $bcfg = <<__EOCFG__ ), -Dcc='ccache gcc' = -Uuseperlio __EOCFG__ ); isa_ok( $reporter, 'Test::Smoke::Reporter' ); my $timer = time - 300; $reporter->read_parse( \(my $result = <{_rpt}{started}, $timer - 300, "Start time" ); is( $reporter->{_rpt}{patch}, 20000, "Changenumber $reporter->{_rpt}{patch}" ); my $cfgarg = "-Dcc='ccache gcc' -Uuseperlio"; is( $reporter->{_rpt}{$cfgarg}{N}{stdio}, "O", "'$cfgarg' reports ok" ); is( $reporter->{_rpt}{$cfgarg}{D}{stdio}, "O", "'$cfgarg -DDEBUGGING' reports ok" ); my @r_lines = split /\n/, $reporter->smoke_matrix; is_deeply \@r_lines, [split /\n/, <<__EOM__], "Matrix"; 20000 Configuration (common) -Dcc='ccache gcc' ----------- --------------------------------------------------------- O O -Uuseperlio __EOM__ chomp( my $summary = $reporter->summary ); is $summary, 'Summary: PASS', $summary; unlike $reporter->report, "/Build configurations:\n$bcfg=/", "hasn't the configurations"; # diag Dumper $reporter->{_counters}; # diag $reporter->report; } { create_config_sh( $config_sh, version => '5.8.3' ); my $reporter = Test::Smoke::Reporter->new( ddir => $findbin, v => $verbose, outfile => '', showcfg => $showcfg, cfg => \( my $bcfg = <<__EOCFG__ ), -Dcc='ccache gcc' __EOCFG__ ); isa_ok( $reporter, 'Test::Smoke::Reporter' ); my $timer = time - 1000; my $patchlevel = 21000; $reporter->read_parse( \(my $result = <{_rpt}{patch}, $patchlevel, "Changenumber $reporter->{_rpt}{patch}" ); my $cfgarg = "-Dcc='ccache gcc'"; { local $" = "', '"; my @bldenv = sort keys %{ $reporter->{_rpt}{$cfgarg}{N} }; is_deeply( \@bldenv, [qw( locale:nl_NL.utf8 perlio stdio )], "Buildenvironments '@bldenv'" ); } is( $reporter->{_rpt}{$cfgarg}{N}{stdio}, 'F', "'$cfgarg' (stdio) reports failure" ); is( $reporter->{_rpt}{$cfgarg}{D}{stdio}, 'F', "'$cfgarg -DDEBUGGING' (stdio) reports failure" ); is( $reporter->{_rpt}{$cfgarg}{N}{perlio}, 'O', "'$cfgarg' (perlio) reports OK" ); is( $reporter->{_rpt}{$cfgarg}{D}{perlio}, 'O', "'$cfgarg -DDEBUGGING' (perlio) reports OK" ); is( $reporter->{_rpt}{$cfgarg}{N}{'locale:nl_NL.utf8'}, 'F', "'$cfgarg' (utf8) reports failure" ); is( $reporter->{_rpt}{$cfgarg}{D}{'locale:nl_NL.utf8'}, 'F', "'$cfgarg -DDEBUGGING' (utf8) reports Failure" ); my @r_lines = split /\n/, $reporter->smoke_matrix; is_deeply \@r_lines, [split /\n/, <<__EOM__], "Matrix"; 21000 Configuration (common) -Dcc='ccache gcc' ----------- --------------------------------------------------------- F O F F O F __EOM__ chomp( my $summary = $reporter->summary ); is $summary, 'Summary: FAIL(F)', $summary; if ( $showcfg ) { like $reporter->report, "/Build configurations:\n$bcfg=/", "has the configurations"; } else { unlike $reporter->report, "/Build configurations:\n$bcfg=/", "hasn't the configurations"; } # diag Dumper $reporter->{_counters}; # diag $reporter->report; } { create_config_sh( $config_sh, version => '5.9.0' ); my $reporter = Test::Smoke::Reporter->new( ddir => $findbin, v => $verbose, outfile => '', ); isa_ok( $reporter, 'Test::Smoke::Reporter' ); my $patchlevel = 19000; $reporter->read_parse( \(my $result = <{_rpt}{patch}, $patchlevel, "Changenumber $reporter->{_rpt}{patch}" ); my $cfgarg = ""; { local $" = "', '"; my @bldenv = sort keys %{ $reporter->{_rpt}{$cfgarg}{N} }; is_deeply( \@bldenv, [qw( perlio stdio )], "Buildenvironments '@bldenv'" ); @bldenv = sort @{ $reporter->{_tstenv} }; is_deeply( \@bldenv, [qw( perlio stdio )], "Buildenvironments '@bldenv'" ); } is( $reporter->{_rpt}{$cfgarg}{N}{stdio}, 'O', "'$cfgarg' (stdio) reports OK" ); is( $reporter->{_rpt}{$cfgarg}{D}{stdio}, 'F', "'$cfgarg -DDEBUGGING' (stdio) reports failure" ); is( $reporter->{_rpt}{$cfgarg}{N}{perlio}, 'O', "'$cfgarg' (perlio) reports OK" ); is( $reporter->{_rpt}{$cfgarg}{D}{perlio}, 'F', "'$cfgarg -DDEBUGGING' (perlio) reports Failure" ); my @r_lines = split /\n/, $reporter->smoke_matrix; is_deeply \@r_lines, [split /\n/, <<__EOM__], "Matrix"; 19000 Configuration (common) none ----------- --------------------------------------------------------- O O F F __EOM__ chomp( my $summary = $reporter->summary ); is $summary, 'Summary: FAIL(F)', $summary; like $reporter->report, qq@/^ Failures: \\s+ \\(common-args\\) \\s+ none \\n \\[stdio\\/perlio\\] \\s* -DDEBUGGING /xm@, "Failures:"; } unlink $config_sh; { # This test is just to test 'PASS' (and not PASS-so-far) # create_config_sh( $config_sh, version => '5.00504' ); my $reporter = Test::Smoke::Reporter->new( ddir => $findbin, v => $verbose, outfile => '', is56x => 1, ); my $patchlevel = 22111; isa_ok $reporter, 'Test::Smoke::Reporter'; $reporter->read_parse( \(my $result = <summary ); is $summary, 'Summary: PASS', $summary; is $reporter->ccinfo, "ccache egcc version 3.2", "ccinfo()"; like $reporter->report, "/^Summary: PASS\n/m", "Summary from report"; my @r_lines = split /\n/, $reporter->smoke_matrix; is_deeply \@r_lines, [split /\n/, <<__EOM__], "Matrix"; 22111 Configuration (common) -Dcc='ccache egcc' ----------- --------------------------------------------------------- O O -Uuseperlio __EOM__ } { # Test a bug reported by Merijn # the c's were reported for locale: only ok( my $reporter = Test::Smoke::Reporter->new( ddir => catfile( $findbin, 'ftppub' ), is56x => 0, defaultenv => 0, locale => 'EN_US.UTF-8', outfile => 'bugtst01.out', v => $verbose, showcfg => $showcfg, cfg => \( my $bcfg = <<__EOCFG__ ), -Dcc=gcc = -Duselongdouble -Duse564bitint __EOCFG__ ), "new()" ); isa_ok $reporter, 'Test::Smoke::Reporter'; is $reporter->ccinfo, "? unknown cc version ", "ccinfo(bugstst01)"; my @r_lines = split /\n/, $reporter->smoke_matrix; my $r = is_deeply \@r_lines, [split /\n/, <<__EOM__], "Matrix"; 22154 Configuration (common) -Dcc=gcc ----------- --------------------------------------------------------- F F F F F F c - - c - - -Duselongdouble F F F - - - -Duse64bitint __EOM__ $r or diag $reporter->smoke_matrix, $reporter->bldenv_legend; if ( $showcfg ) { like $reporter->report, "/Build configurations:\n$bcfg=/", "has the configurations"; } else { unlike $reporter->report, "/Build configurations:\n$bcfg=/", "hasn't the configurations"; } } { # report from cygwin ok( my $reporter = Test::Smoke::Reporter->new( ddir => catdir( $findbin, 'ftppub' ), is56x => 0, defaultenv => 0, outfile => 'bugtst02.out', v => $verbose, showcfg => $showcfg, cfg => \( my $bcfg = <<__EOCFG__ ), -Duseithreads = -Duse64bitint __EOCFG__ ), "new reporter for bugtst02.out" ); isa_ok $reporter, 'Test::Smoke::Reporter'; is $reporter->ccinfo, "gcc version 3.3.1 (cygming special)", "ccinfo(bugstst02)"; my @r_lines = split /\n/, $reporter->smoke_matrix; my $r = is_deeply \@r_lines, [split /\n/, <<__EOM__], "Matrix 2"; 22302 Configuration (common) none ----------- --------------------------------------------------------- F F M - F F M - -Duse64bitint F F M - -Duseithreads F F M - -Duseithreads -Duse64bitint __EOM__ $r or diag $reporter->smoke_matrix, $reporter->bldenv_legend; if ( $showcfg ) { like $reporter->report, "/Build configurations:\n$bcfg=/", "has the configurations"; } else { unlike $reporter->report, "/Build configurations:\n$bcfg=/", "hasn't the configurations"; } } { # report from Win32 ok( my $reporter = Test::Smoke::Reporter->new( ddir => catdir( $findbin, 'ftppub' ), is56x => 0, defaultenv => 1, is_win32 => 1, outfile => 'bugtst03.out', v => $verbose, showcfg => $showcfg, cfg => \( my $bcfg = <<'__EOCFG__' ), -DINST_TOP=$(INST_DRV)\Smoke\doesntexist = -Duseithreads = -Duselargefiles = -Dusemymalloc = -Accflags='-DPERL_COPY_ON_WRITE' = -DDEBUGGING __EOCFG__ ), "new reporter for bugtst03.out" ); isa_ok $reporter, 'Test::Smoke::Reporter'; is $reporter->ccinfo, "cl version 12.00.8804", "ccinfo(bugstst03)"; my @r_lines = split /\n/, $reporter->smoke_matrix; my $r = is_deeply \@r_lines, [split /\n/, <<'__EOM__'], "Matrix 3"; 22423 Configuration (common) -DINST_TOP=$(INST_DRV)\Smoke\doesntexist ----------- --------------------------------------------------------- O O F F -Accflags='-DPERL_COPY_ON_WRITE' O O -Dusemymalloc F F -Dusemymalloc -Accflags='-DPERL_COPY_ON_WRITE' O O -Duselargefiles F F -Duselargefiles -Accflags='-DPERL_COPY_ON_WRITE' O O -Duselargefiles -Dusemymalloc F F -Duselargefiles -Dusemymalloc -Accflags='-DPERL_COPY_ON_WRITE' X O -Duseithreads F F -Duseithreads -Accflags='-DPERL_COPY_ON_WRITE' O O -Duseithreads -Dusemymalloc F F -Duseithreads -Dusemymalloc -Accflags='-DPERL_COPY_ON_WRITE' X O -Duseithreads -Duselargefiles F F -Duseithreads -Duselargefiles -Accflags='-DPERL_COPY_ON_WRITE' X O -Duseithreads -Duselargefiles -Dusemymalloc F F -Duseithreads -Duselargefiles -Dusemymalloc -Accflags='-DPERL_COPY_ON_WRITE' __EOM__ $r or diag $reporter->smoke_matrix, $reporter->bldenv_legend; if ( $showcfg ) { like $reporter->report, "/Build configurations:\n$bcfg=/", "has the configurations"; } else { unlike $reporter->report, "/Build configurations:\n$bcfg=/", "hasn't the configurations"; } my @f_lines = split /\n/, $reporter->failures; is_deeply \@f_lines, [split /\n/, <<'__EOFAIL__'], "Failures(bugtst03)"; [default] -Accflags='-DPERL_COPY_ON_WRITE' [default] -Dusemymalloc -Accflags='-DPERL_COPY_ON_WRITE' [default] -Duselargefiles -Accflags='-DPERL_COPY_ON_WRITE' [default] -Duselargefiles -Dusemymalloc -Accflags='-DPERL_COPY_ON_WRITE' ../ext/Cwd/t/cwd.t......................FAILED 10 12 14 16-18 op/magic.t..............................FAILED 37-53 op/tie.t................................FAILED 22 [default] -DDEBUGGING -Accflags='-DPERL_COPY_ON_WRITE' [default] -DDEBUGGING -Duselargefiles -Accflags='-DPERL_COPY_ON_WRITE' ../ext/Cwd/t/cwd.t......................FAILED 9-20 op/magic.t..............................FAILED 37-53 op/tie.t................................FAILED 22 [default] -DDEBUGGING -Dusemymalloc -Accflags='-DPERL_COPY_ON_WRITE' [default] -DDEBUGGING -Duselargefiles -Dusemymalloc -Accflags='-DPERL_COPY_ON_WRITE' ../ext/Cwd/t/cwd.t......................FAILED 9-20 ../lib/DBM_Filter/t/utf8.t..............FAILED 13 19 op/magic.t..............................FAILED 37-53 op/tie.t................................FAILED 22 [default] -Duseithreads Inconsistent test results (between TEST and harness): ../ext/threads/t/thread.t...............FAILED test 25 [default] -Duseithreads -Accflags='-DPERL_COPY_ON_WRITE' [default] -Duseithreads -Dusemymalloc -Accflags='-DPERL_COPY_ON_WRITE' ../ext/Cwd/t/cwd.t......................FAILED 18-20 [default] -DDEBUGGING -Duseithreads -Accflags='-DPERL_COPY_ON_WRITE' [default] -DDEBUGGING -Duseithreads -Dusemymalloc -Accflags='-DPERL_COPY_ON_WRITE' [default] -DDEBUGGING -Duseithreads -Duselargefiles -Accflags='-DPERL_COPY_ON_WRITE' [default] -DDEBUGGING -Duseithreads -Duselargefiles -Dusemymalloc -Accflags='-DPERL_COPY_ON_WRITE' ../ext/Cwd/t/cwd.t......................FAILED 9-20 [default] -Duseithreads -Duselargefiles [default] -Duseithreads -Duselargefiles -Dusemymalloc Inconsistent test results (between TEST and harness): ../ext/threads/t/problems.t.............dubious [default] -Duseithreads -Duselargefiles -Accflags='-DPERL_COPY_ON_WRITE' [default] -Duseithreads -Duselargefiles -Dusemymalloc -Accflags='-DPERL_COPY_ON_WRITE' ../ext/Cwd/t/cwd.t......................FAILED 18-20 Inconsistent test results (between TEST and harness): ../ext/threads/shared/t/sv_simple.t.....dubious __EOFAIL__ } { # Test the grepccmsg() feature my $testdir = catdir $findbin, 'perl-current'; mkpath $testdir; copy( catfile( $findbin, 'gccmsg.out'), catfile( $testdir, 'mktest.out' ) ); ok( my $reporter = Test::Smoke::Reporter->new( ddir => $testdir, is56x => 0, defaultenv => 0, lfile => catfile( $findbin, 'gccmsg.log' ), v => $verbose, showcfg => $showcfg, cfg => \( my $bcfg = <<'__EOCFG__' )), "init for grepccmsg() "); == -Duseithreads == __EOCFG__ my @ccmsg = split /\n/, <<'EOCCMSG'; Compiler messages(gcc): regcomp.c: In function `S_make_trie': regcomp.c:905: warning: `scan' might be used uninitialized in this function regcomp.c: In function `S_study_chunk': regcomp.c:1618: warning: comparison is always false due to limited range of data type pp_sys.c:311: warning: `S_emulate_eaccess' defined but not used byterun.c: In function `byterun': byterun.c:906: warning: comparison is always false due to limited range of data type DProf.xs:140: warning: `unused' attribute ignored re_comp.c: In function `S_study_chunk': re_comp.c:1618: warning: comparison is always false due to limited range of data type EOCCMSG # `stupid emacs ok my $ccmsg = $reporter->ccmessages, "Got compiler messages"; for my $line ( @ccmsg ) { like $ccmsg, "/\Q$line\E/ms", "$line"; } rmtree $testdir, $verbose; } { # Test the registered_patches() feature my $testdir = catdir $findbin, 'perl-current'; mkpath $testdir; copy( catfile( $findbin, 'gccmsg.out'), catfile( $testdir, 'mktest.out' ) ); my $plhsrc = catdir( $findbin, qw( ftppub perl-current ) ); copy( catfile( $plhsrc, 'patchlevel.h' ), catfile( $testdir, 'patchlevel.h' ) ); require Test::Smoke::Util; Test::Smoke::Util::set_local_patch( $testdir, "[PATCH] Just testing" ); ok( my $reporter = Test::Smoke::Reporter->new( ddir => $testdir, is56x => 0, defaultenv => 0, lfile => catfile( $findbin, 'gccmsg.log' ), v => $verbose, showcfg => $showcfg, cfg => \( my $bcfg = <<'__EOCFG__' )), "init for get_local_patches() "); == -Duseithreads == __EOCFG__ my $lp_list = $reporter->registered_patches; like $lp_list, "/\Q[PATCH] Just testing\E/", "Found the patch"; my $rpt = $reporter->report; like $rpt, "/\\nLocally\\ applied\\ patches:\\n \\ \\ \\ \\ DEVEL19999\\n \\ \\ \\ \\ \\[PATCH\\]\\ Just\\ testing/x", "Found patches section"; unlike $rpt, "/^Tests skipped on user request:\n/m", "Report does not contain user_skipped_tests()"; rmtree $testdir, $verbose; } { # Test the user_skipped_tests() feature my $testdir = catdir $findbin, 'perl-current'; mkpath $testdir; copy( catfile( $findbin, 'gccmsg.out'), catfile( $testdir, 'mktest.out' ) ); my $plhsrc = catdir( $findbin, qw( ftppub perl-current ) ); copy( catfile( $plhsrc, 'patchlevel.h' ), catfile( $testdir, 'patchlevel.h' ) ); my $skip_tests = catfile $findbin, 'tests.skip'; put_file( (my $no_tests = <<__EOSKIP__), $skip_tests ); lib/Benchmark.t __EOSKIP__ ok( my $reporter = Test::Smoke::Reporter->new( ddir => $testdir, is56x => 0, defaultenv => 0, lfile => catfile( $findbin, 'gccmsg.log' ), skip_tests => $skip_tests, v => $verbose, showcfg => $showcfg, cfg => \( my $bcfg = <<'__EOCFG__' )), "init for get_local_patches() "); == -Duseithreads == __EOCFG__ $no_tests = join "\n", map " $_" => split /\n/, $no_tests; my $st_list = $reporter->user_skipped_tests; is $st_list, "\nTests skipped on user request:\n$no_tests", "user_skipped_tests()"; my $rpt = $reporter->report; like $rpt, "/^Tests skipped on user request:\n$no_tests/m", "Report contains user_skipped_tests()"; rmtree $testdir, $verbose; 1 while unlink $skip_tests; } { my $bcfg = ""; my $plhsrc = catdir( $findbin, qw( ftppub perl-current ) ); copy( catfile( $plhsrc, 'patchlevel.h' ), catfile( $findbin, 'patchlevel.h' ) ); my $report = Test::Smoke::Reporter->new( ddir => $findbin, outfile => 'multilocale.out', verbose => $verbose, defaultenv => 0, cfg => \$bcfg, ); isa_ok $report, 'Test::Smoke::Reporter'; is $report->bldenv_legend, <<__EOL__, "legend ok"; | | | | | | | +- LC_ALL = nl_NL.UTF-8 -DDEBUGGING | | | | | | +--- LC_ALL = be_BY.UTF-8 -DDEBUGGING | | | | | +----- PERLIO = perlio -DDEBUGGING | | | | +------- PERLIO = stdio -DDEBUGGING | | | +--------- LC_ALL = nl_NL.UTF-8 | | +----------- LC_ALL = be_BY.UTF-8 | +------------- PERLIO = perlio +--------------- PERLIO = stdio __EOL__ 1 while unlink catfile( $findbin, 'patchlevel.h' ); } { my $bcfg = <<__EOC__; -Duse64bit -Duselongdouble -Dusemorebits = -Duseithreads = /-DDEBUGGING/ -DDEBUGGING __EOC__ my $plhsrc = catdir( $findbin, qw( ftppub perl-current ) ); copy( catfile( $plhsrc, 'patchlevel.h' ), catfile( $findbin, 'patchlevel.h' ) ); my $report = Test::Smoke::Reporter->new( ddir => $findbin, outfile => 'pc09.out', verbose => $verbose, defaultenv => 0, cfg => \$bcfg, ); isa_ok $report, 'Test::Smoke::Reporter'; is $report->bldenv_legend, <<__EOL__, "legend ok"; | | | | | +- LC_ALL = en_US.utf8 -DDEBUGGING | | | | +--- PERLIO = perlio -DDEBUGGING | | | +----- PERLIO = stdio -DDEBUGGING | | +------- LC_ALL = en_US.utf8 | +--------- PERLIO = perlio +----------- PERLIO = stdio __EOL__ 1 while unlink catfile( $findbin, 'patchlevel.h' ); } sub create_config_sh { my( $file, %cfg ) = @_; my $cfg_sh = "# This is a testfile config.sh\n"; $cfg_sh .= "# created by $0\n"; $cfg_sh .= join "", map "$_='$cfg{$_}'\n" => keys %cfg; put_file( $cfg_sh, $file ); }