The framework [testify](https://github.com/stretchr/testify/tree/v1.8.4) might be the most popular. Surprisingly, I find the documentation is not clear enough to answer my questions and I can't find answers anywhere. Therefore, I have to write experiments to verify my guesses.

One of my questions is: when will the registered Cleanup functions run?

The summary of my experiment shows that the order of Cleanup functions is running as we normally expected except for those from subtests. The exception could lead severe problems.

The experiment

The idea is simple: register Cleanup functions from all setup/teardown methods and use the log to infer the execution order.

Let's have a peek at the result first.

We can see that the Cleanup functions registered at TearDownSubTest and SetupSubTest are called after the subtests are finished. However, it's NOT guaranteed that they will be called immediately after a subtest is finished.

There are two scenarios:

For nested subtests, the Cleanup functions registered at TearDownSubTest and SetupSubTest are called once their parents are finished but not cleaned.
For direct subtests whose parents are NOT subtests, their Cleanup functions registered at TearDownSubTest and SetupSubTest are called once the parents' AfterTest and TearDownTest are finished.

The impact is:

Nested subtests could pollute each other because some Cleanup functions are running after all sibling subtests.
Children subtest can even pollute their parents.

The source code can be found here: https://github.com/xuanyuwang/testify-examples/blob/main/cleanup-order/cleanup_order_test.go

package cleanuporder_test

import (
    "fmt"
    "testing"

    "github.com/stretchr/testify/suite"
)

var tearDownSubTest []string

type TestCleanupOrder struct {
    suite.Suite
}

func (s *TestCleanupOrder) SetupSuite() {
    fmt.Println("Run Interface SetupAllSuite: SetupSuite")
    s.T().Cleanup(func() {
        fmt.Println("Cleanup Interface SetupAllSuite: SetupSuite")
    })
}

func (s *TestCleanupOrder) SetupTest() {
    fmt.Println("Run Interface SetupTestSuite: SetupTest")
    s.T().Cleanup(func() {
        fmt.Println("Cleanup Interface SetupTestSuite: SetupTest")
    })
}

func (s *TestCleanupOrder) TeatDownSuite() {
    fmt.Println("Run Interface TearDownAllSuite: TearDownSuite")
    s.T().Cleanup(func() {
        fmt.Println("Cleanup Interface TearDownAllSuite: TearDownSuite")
    })
}

func (s *TestCleanupOrder) TearDownTest() {
    fmt.Println("Run Interface TearDownTestSuite: TearDownTest")
    s.T().Cleanup(func() {
        fmt.Println("Cleanup Interface TearDownTestSuite: TearDownTest")
    })
}

func (s *TestCleanupOrder) BeforeTest(suiteName, testName string) {
    fmt.Printf("Run Interface BeforeTest: BeforeTest for suite %s - test %s\n", suiteName, testName)
    s.T().Cleanup(func() {
        fmt.Printf("Cleanup Interface BeforeTest: BeforeTest for suite %s - test %s\n", suiteName, testName)
    })
}

func (s *TestCleanupOrder) AfterTest(suiteName, testName string) {
    fmt.Printf("Run Interface AfterTest: AfterTest for suite %s - test %s\n", suiteName, testName)
    s.T().Cleanup(func() {
        fmt.Printf("Cleanup Interface AfterTest: AfterTest for suite %s - test %s\n", suiteName, testName)
    })
}

func (s *TestCleanupOrder) HandleStats(suiteName string, stats *suite.SuiteInformation) {
    fmt.Printf("Run Interface WithStats: HandleStats for suite %s\n", suiteName)
    s.T().Cleanup(func() {
        fmt.Printf("Cleanup Interface WithStats: HandleStats for suite %s\n", suiteName)
    })
}

func (s *TestCleanupOrder) SetupSubTest() {
    fmt.Println("Run Interface SetupSubTest: SetupSubTest")
    s.T().Cleanup(func() {
        fmt.Println("Cleanup Interface SetupSubTest: SetupSubTest")
    })
}

func (s *TestCleanupOrder) TearDownSubTest() {
    fmt.Println("Run Interface TearDownSubTest: TearDownSubTest")
    s.T().Cleanup(func() {
        fmt.Println("Cleanup Interface TearDownSubTest: TearDownSubTest")
        fmt.Printf("\tTear down sub test %v\n", tearDownSubTest)
    })
}

// Test case
func (s *TestCleanupOrder) TestCaseA() {
    fmt.Println("Run TestCase A")
    s.T().Cleanup(func() {
        fmt.Println("Cleanup TestCase A")
    })

    s.Run("Sub-TestCase A", func() {
        fmt.Println("Run Sub-TestCase A")
        s.T().Cleanup(func() {
            fmt.Println("Cleanup Sub-TestCase A")
        })
        tearDownSubTest = append(tearDownSubTest, "Sub-TestCase A")

        s.Run("Sub-Sub-TestCase A", func() {
            s.T().Cleanup(func() {
                fmt.Println("Cleanup Sub-Sub-TestCase A")
            })
            fmt.Println("Run Sub-Sub-TestCase A")
            tearDownSubTest = append(tearDownSubTest, "Sub-Sub-TestCase A")
            fmt.Println("Finish Sub-Sub-TestCase A")
        })
        s.Run("Sub-Sub-TestCase B", func() {
            s.T().Cleanup(func() {
                fmt.Println("Cleanup Sub-Sub-TestCase B")
            })
            fmt.Println("Run Sub-Sub-TestCase B")
            tearDownSubTest = append(tearDownSubTest, "Sub-Sub-TestCase B")
            fmt.Println("Finish Sub-Sub-TestCase B")
        })

        fmt.Println("Finish Sub-TestCase A")
    })
    fmt.Println("Finish TestCase A")
}

// Entry point of the whole test suite
func TestCleanupOrderSuite(t *testing.T) {
    suite.Run(t, &TestCleanupOrder{})
}

Blog

Test in Go: the order of Cleanup is not what you think

Xuanyu Wang

The experiment

Join Our Newsletter. No Spam, Only the good stuff.

Related