CPPUnit: A Simple Testing Framework for C++ Programs

Testing Hierarchical Systems with Hierarchical Tests

A system is usually designed as a collection of subsystems (e.g., packages, namespaces, libraries, files). A subsystem is a collection of classes, functions, and sub-subsystems, etc:

A testing framework for such a system is also organized into a hierarch. The top level system test runs tests of all of the subsystems. A subsystem test runs tests of each of its classes, functions, and sub-subsystems. A class test runs tests of each of its member functions:

Subsystem tests-- tests that run tests of member components-- are called test suites. Tests that test a single class or function are called unit tests.

Note: In this paper we are talking about functional or black-box testing, not structural testing.

Demonstration

Assume a library consists of three sub-libraries: Math1, Math2, and Strings. Math1 contains three global functions: fun1, fun2, and fun3. The Math2 library only contains two functions: fun4 and fun5. Finally, the string library contains fun6 and fun7. The author of each function also writes a constructor that generates a unit test for that function: makeFun1Test(), makeFun2Test(), etc. A constructor associated with each sub-library creates a suite test containing all of the unit tests for functions in that library. Finally, a system test containing all of the sub-library tests is created and run.

math1.h

#include "test.h"
double fun1(double x);
double fun2(double x);
double fun3(double x);
Test* makeMathTest();

math2.h

#include "test.h"
double fun4(double x);
double fun5(double x);
Test* makeMath2Test();

strings.h

#include "test.h"
#include <string>
using namespace std;
int fun6(string s);
string fun7(string s);
Test* makeStringTest();

math1.cpp

#include "math1.h"

double fun1(double x) { return 2 * x * x + 3 * x + 4; }
double fun2(double x) { return -2 * x + 9; }
double fun3(double x) { return x * x * x - x * x + x; }

// And now the tests:

Test* makeFun1Test()
{
UnitTest<double, double>* test
= new UnitTest<double, double>(fun1, "fun1 test");
(*test)[0] = 4;
(*test)[1] = 9;
(*test)[-1] = 3;
return test;
}

Test* makeFun2Test()
{
UnitTest<double, double>* test
= new UnitTest<double, double>(fun2, "fun2 test");
(*test)[2] = 5;
(*test)[0] = 9;
(*test)[5] = 0; // oops!
(*test)[50] = -91;
return test;
}

Test* makeFun3Test()
{
UnitTest<double, double>* test
= new UnitTest<double, double>(fun3, "fun3 test");
(*test)[1] = 1;
(*test)[0] = 0;
(*test)[2] = 6;
(*test)[50] = -91; // oops!
return test;
}

Test* makeMath1Test()
{
SuiteTest* test = new SuiteTest("math1 tests");
test->add(makeFun1Test());
test->add(makeFun2Test());
test->add(makeFun3Test());
return test;
}

math2.cpp

#include "math2.h"

double fun4(double x) { return (x + 1) * (x - 1); }
double fun5(double x) { return .5 * x + 1; }

// And now the tests:

Test* makeFun4Test()
{
UnitTest<double, double>* test
= new UnitTest<double, double>(fun4, "fun4 test");
(*test)[3] = 8;
(*test)[1] = 0;
(*test)[.3] = -.91;
(*test)[5] = 7; // oops
// etc.
return test;
}

Test* makeFun5Test()
{
UnitTest<double, double>* test
= new UnitTest<double, double>(fun5, "fun5 test");
(*test)[1] = 1.5;
(*test)[2] = 2;
(*test)[.1] = 1.05;
// etc.
return test;
}

Test* makeMath2Test()
{
SuiteTest* test = new SuiteTest("math2 tests");
test->add(makeFun4Test());
test->add(makeFun5Test());
return test;
}

strings.cpp

#include "strings.h"

int fun6(string s) { return s.length() + 1; }
string fun7(string s) { return s + s; }

// And now the tests:

Test* makeFun6Test()
{
UnitTest<string, int>* test
= new UnitTest<string, int>(fun6, "unit test 6");
(*test)["hello"] = 6;
(*test)["Mississippi"] = 12;
return test;
}

Test* makeFun7Test()
{
UnitTest<string, string>* test
= new UnitTest<string, string>(fun7, "unit test 7");
(*test)["hello"] = "hellohello";
(*test)["Mississippi"] = "MississippiMississippi";
return test;
}

Test* makeStringTest()
{
SuiteTest *suite = new SuiteTest("string tests");
suite->add(makeFun6Test());
suite->add(makeFun7Test());
return suite;
}

main.cpp

#include "math1.h"
#include "math2.h"
#include "strings.h"
#include "test.h"

int main()
{
SuiteTest systemTest("System Test");
systemTest.add(makeMath1Test());
systemTest.add(makeMath2Test());
systemTest.add(makeStringTest());
systemTest.run();
cout << systemTest;
return 0;
}

Program output

vvvvvvvvvvvvvvvvvvvvvvvvvv
Test suite name: System Test
#passes = 1
#fails = 2
Details:
vvvvvvvvvvvvvvvvvvvvvvvvvv
Test suite name: math fun tests
#passes = 1
#fails = 2
Details:
... Unit test fun1 test passed
... Unit test fun2 test
..... on input 5 expected 0 but computed -1
... Unit test fun3 test
..... on input 50 expected -91 but computed 122550
^^^^^^^^^^^^^^^^^^^^^^^^^^
vvvvvvvvvvvvvvvvvvvvvvvvvv
Test suite name: math2 tests
#passes = 1
#fails = 1
Details:
... Unit test fun4 test
..... on input 0.3 expected -0.91 but computed -0.91
..... on input 5 expected 7 but computed 24
... Unit test fun5 test passed
^^^^^^^^^^^^^^^^^^^^^^^^^^
vvvvvvvvvvvvvvvvvvvvvvvvvv
Test suite name: string tests
#passes = 2
#fails = 0
Details:
... Unit test unit test 6 passed
... Unit test unit test 7 passed
^^^^^^^^^^^^^^^^^^^^^^^^^^
^^^^^^^^^^^^^^^^^^^^^^^^^^

Design

CPPUnit is based on the Composite Design Pattern:

Note that a suite test may contain other suite tests as well as unit tests.

A unit test compares the actual outputs produced by a single function with the expected outputs. The inputs, together with the expected outputs, are stored in a table called the oracle. The types of the inputs and outputs are given by template parameters in and out.

Finally, the function to be tested is a member variable rather than a member function. This is called the Pluggable Adapter Design pattern.

Implementation

test.h

#ifndef TEST_H
#define TEST_H
#include <iostream>
#include <string>
#include <list>
#include <map>
using namespace std;

class Test { ... };
class SuiteTest extends Test{ ... };
ostream& operator<<(ostream& os, const Test& test);

template <typename InputType, typename OutputType>
class UnitTest: public Test { ... };

template <typename InputType, typename OutputType>
bool UnitTest<InputType, OutputType>::run()
{ ... }
template <typename InputType, typename OutputType>

void UnitTest<InputType, OutputType>::display(ostream& os) const
{ ... }

#endif

class Test

class Test
{
public:
Test(string name = "no name")
{
this->name = name;
}
virtual bool run() = 0;
string getName() const { return name; }
virtual void display(ostream& os = cout) const {}
protected:
string name;
};

class SuiteTest

class SuiteTest: public Test
{
public:
SuiteTest(string name = "suite test")
:Test(name)
{
passes = fails = 0;
}
bool run();
void add(Test* test) { tests.push_back(test); }
virtual void display(ostream& os = cout) const {}
private:
list<Test*> tests;
int passes, fails;
};

SuiteTest::run() (from test.cpp):

bool SuiteTest::run()
{
list<Test*>::iterator p;
for(p = tests.begin(); p != tests.end(); p++)
{
bool temp = (*p)->run();
if (!temp)
{
fails++;
}
else
{
passes++;
}
result = result && temp;
}
return result;
}

A Sample Unit Test

typedef double (*R2RFun)(double);

class R2RUnitTest: public Test
{
public:
R2RUnitTest(R2RFun f, string name = "unit test")
: Test(name)
{
fun = f;
delta = 1e-10;
}
bool run();
double& operator[](double d)
{
return oracle[d];
}
void setDelta(double d) { delta = d; }
double getDelta() const { return delta; }
protected:
map<double, double> oracle;
R2RFun fun;
double delta;
};

R2RUnitTest::run() (from test.cpp)

bool R2RUnitTest::run()
{
bool result = true;
map<double, double>::iterator p;
for(p = oracle.begin(); p != oracle.end(); p++)
{
double input = (*p).first;
double expected = (*p).second;
double output = fun(input);
bool temp = fabs(expected - output) <= delta;
if (!temp)
{
cout << "...error: " << name << " expected " << expected;
cout << " but computed " << output << endl;
}
result &&= temp;
}
return result;
}

UnitTest as a template (from test.h)

template <typename InputType, typename OutputType>
class UnitTest: public Test
{
public:
typedef OutputType (*Fun)(InputType);
UnitTest(Fun f, string name = "unit test")
: Test(name)
{
fun = f;
}
class FailureReport
{
public:
FailureReport(InputType i, OutputType e, OutputType c)
{
expected = e;
computed = c;
input = i;
}
OutputType expected;
OutputType computed;
InputType input;
};
bool run();
OutputType& operator[](InputType d)
{
return oracle[d];
}
void display(ostream& os = cout) const;
protected:
map<InputType, OutputType> oracle;
list<ErrorReport*> failures;
Fun fun;
};

UnitTest::run() as a template method (also from test.h)

template <typename InputType, typename OutputType>
bool UnitTest<InputType, OutputType>::run()
{
bool result = true;
map<InputType, OutputType>::iterator p;
for(p = oracle.begin(); p != oracle.end(); p++)
{
InputType input = (*p).first;
OutputType expected = (*p).second;
OutputType output = fun(input);
bool temp = expected == output;
if (!temp)
{
failures.push_back(
new FailureReport(input, expected, output));
}
result = result && temp;
}
return result;
}

UnitTest::display (also from test.h)

template <typename InputType, typename OutputType>
void UnitTest<InputType, OutputType>::display(ostream& os) const
{

os << "... Unit test " << name;
if (failures.empty())
{
os << " passed\n";
}
else
{
list<FailureReport*>::const_iterator p;
os << endl;
for(p = failures.begin(); p != failures.end(); p++)
{
os << "..... on input " << (*p)->input;
os << " expected " << (*p)->expected;
os << " but computed " << (*p)->computed << endl;
}
}
}

test.cpp

void SuiteTest::display(ostream& os) const
{
os << "vvvvvvvvvvvvvvvvvvvvvvvvvv\n";
os << "Test suite name: " << name << endl;
os << " #passes = " << passes << endl;
os << " #fails = " << fails << endl;
os << "Details:\n";
list<Test*>::const_iterator p;
for(p = tests.begin(); p != tests.end(); p++)
{
(*p)->display(os);
}
os << "^^^^^^^^^^^^^^^^^^^^^^^^^^\n";
}

ostream& operator<<(ostream& os, const Test& test)
{
test.display(os);
return os;
}