Merge pull request #114 from hanxizh9910/feature/automated-test-failure-detector

Feature/automated test failure detector

Author: hanxizh9910

.github/workflows/daily.yml

@@ -25,7 +25,7 @@ set ::dont_clean 0
 set ::simulate_error 0
 set ::failed 0
 set ::failed_tests {}
-set ::failures_json_file ""
+set ::failures_output_file ""
 set ::sentinel_instances {}
 set ::valkey_instances {}
 set ::global_config {}
@@ -317,8 +317,8 @@ proc parse_options {} {
             set ::stop_on_failure 1
         } elseif {$opt eq {--loop}} {
             set ::loop 1
-        } elseif {$opt eq {--failures-json}} {
-            set ::failures_json_file [file normalize "../../../$val"]
+        } elseif {$opt eq {--failures-output}} {
+            set ::failures_output_file [file normalize "../../../$val"]
             incr j
         } elseif {$opt eq {--log-req-res}} {
             set ::log_req_res 1
@@ -339,7 +339,7 @@ proc parse_options {} {
             puts "--fast-fail             Exit immediately once the first test fails."
             puts "--stop                  Blocks once the first test fails."
             puts "--loop                  Execute the specified set of tests forever."
-            puts "--failures-json <path>  Write test failures to the specified JSON file."
+            puts "--failures-output <path>  Write test failures to the specified JSON file."
             puts "--help                  Shows this help."
             exit 0
         } else {
@@ -533,7 +533,7 @@ while 1 {
 }
 
 proc write_test_failures {} {
-    if {$::failures_json_file eq ""} {
+    if {$::failures_output_file eq ""} {
         return
     }
 
@@ -550,11 +550,11 @@ proc write_test_failures {} {
         lappend failures "\{\"test_name\":\"$test_name\",\"test_file\":\"$test_file\",\"status\":\"err\",\"error\":\"$error_msg\"\}"
     }
 
-    set outdir [file dirname $::failures_json_file]
+    set outdir [file dirname $::failures_output_file]
     if {$outdir ne "."} {
         file mkdir $outdir
     }
-    set fp [open $::failures_json_file w]
+    set fp [open $::failures_output_file w]
     puts $fp "\[[join $failures ","]\]"
     close $fp
 }

tests/instances.tcl

@@ -46,7 +46,7 @@ test "SENTINEL MYID return the sentinel instance ID" {
 test "SENTINEL INFO CACHE returns the cached info" {
     set res [S 0 SENTINEL INFO-CACHE mymaster]
     assert_morethan_equal [llength $res] 2
-    assert_equal "mymaster" [lindex $res 0]
+    assert_equal "mymasterr" [lindex $res 0]
 
     set res [lindex $res 1]
     assert_morethan_equal [llength $res] 2

tests/sentinel/tests/00-base.tcl

@@ -74,7 +74,7 @@ set ::file ""; # If set, runs only the tests in this comma separated list
 set ::curfile ""; # Hold the filename of the current suite
 set ::accurate 0; # If true runs fuzz tests with more iterations
 set ::force_failure 0
-set ::failures_json_file ""; # If set, write failures JSON to this path
+set ::failures_output_file ""; # If set, write failures JSON to this path
 set ::timeout 1200; # 20 minutes without progresses will quit the test.
 set ::last_progress [clock seconds]
 set ::active_servers {} ; # Pids of active server instances.
@@ -613,7 +613,7 @@ proc print_test_summary {} {
 }
 
 proc write_test_failures {} {
-    if {$::failures_json_file eq ""} {
+    if {$::failures_output_file eq ""} {
         return
     }
 
@@ -644,11 +644,11 @@ proc write_test_failures {} {
         lappend failures "\{\"test_name\":\"$test_name\",\"test_file\":\"$test_file\",\"status\":\"$status\",\"error\":\"$error_msg\"\}"
     }
 
-    set outdir [file dirname $::failures_json_file]
+    set outdir [file dirname $::failures_output_file]
     if {$outdir ne "."} {
         file mkdir $outdir
     }
-    set fp [open $::failures_json_file w]
+    set fp [open $::failures_output_file w]
     puts $fp "\[[join $failures ","]\]"
     close $fp
 }
@@ -734,7 +734,7 @@ proc print_help_screen {} {
         "--clients <num>    Number of test clients (default 16)."
         "--timeout <sec>    Test timeout in seconds (default 20 min)."
         "--force-failure    Force the execution of a test that always fails."
-        "--failures-json <path>"
+        "--failures-output <path>"
         "                   Write test failures to the specified JSON file."
         "--config <k> <v>   Extra config file argument."
         "--skipfile <file>  Name of a file containing test names or regexp patterns (if"
@@ -862,8 +862,8 @@ for {set j 0} {$j < [llength $argv]} {incr j} {
         set ::accurate 1
     } elseif {$opt eq {--force-failure}} {
         set ::force_failure 1
-    } elseif {$opt eq {--failures-json}} {
-        set ::failures_json_file $arg
+    } elseif {$opt eq {--failures-output}} {
+        set ::failures_output_file $arg
         incr j
     } elseif {$opt eq {--single}} {
         foreach unit [expand_unit_spec $arg] {

tests/test_helper.tcl

@@ -24,7 +24,7 @@ start_multiple_servers 3 [list overrides $base_conf] {
                             127.0.0.1:[srv -2 port]
 
         wait_for_condition 1000 50 {
-            [CI 1 cluster_state] eq {ok} &&
+            [CI 1 cluster_state] eq {okk} &&
             [CI 2 cluster_state] eq {ok}
         } else {
             fail "Cluster doesn't stabilize"

tests/unit/cluster/cli.tcl

@@ -0,0 +1,17 @@
+start_server {tags {"dummy"}} {
+    test "dummy-flaky - basic SET and GET" {
+        r SET mykey myvalue
+        assert_equal [r GET mykey] "myvalue"
+    }
+
+    test "dummy-flaky - intentional failure" {
+        r SET mykey myvalue
+        assert_equal [r GET mykey] "wrongvalue"
+    }
+
+    test "dummy-flaky - another passing test" {
+        r SET counter 0
+        r INCR counter
+        assert_equal [r GET counter] "1"
+    }
+}

tests/unit/dummy-flaky.tcl

@@ -14,7 +14,7 @@ start_server {tags {"modules"}} {
 
         # Verify returned id is the same as our current id and
         # we are authenticated with the specified user
-        assert_equal [r acl whoami] "global"
+        assert_equal [r acl whoami] "globall"
     }
 
     test {De-authenticating clients is tracked and kills clients} {