Code coverage - complete toolchain for building, running, generating coverage and html reports

In this post I will show how to write a batch file that will:

1. Build project
2. Execute project
3. Generate code coverage
4. Generate html report with code coverage

The project is written in C++ and it is meant for PC environment - for example this can be some modules containing only logic that we want to test on PC instead of testing on actual HW.
Some module tests can be written by hand and assertion can write output to the console.
In the toolchain program output will be captured to a file so that all of the tests results can be viewed after execution.

Source code for eclipse C++ project can be downloaded from GitHub.

Batch file for building:

REM ************************************
REM * date: 22.07.2014
REM * autor: Filip Ryborz
REM * website: simply-embedded.blogspot.com
REM *
REM * Script for build automation :
REM * 1. Compilation and linking using GNU GCC compiler
REM * 2. Generation of code coverage files with GCOV
REM * 3. Demangling files using C++filt 
REM * 4. Generation of HTML coverage reports using LCOV
REM *
REM * Prerequisites:
REM * g++, gcov, c++filt in PATH or add paths!
REM * Perl and association of files with .Perl extension
REM * Problems when associating using Windows dialogs! Instead use command line!
REM * command: assoc .perl=Perl.File
REM * command: ftype Perl.File=C:\Perl\bin\perl.exe "%1" %*
REM ************************************

REM change directory to build.bat directory and create folders needed for output data
cd %~dp0
md HTML
md Debug
md Debug\Drivers
md Debug\Logic
md Debug\info
md Debug\info\built
md Debug\info\built\Drivers
md Debug\info\built\Logic
md Debug\info\gcov
md Debug\info\gcov\Drivers
md Debug\info\gcov\Logic

REM compile source code
cd %~dp0\Drivers
for /R "%~dp0\Drivers" %%f in (*.cpp) do g++  -I../Drivers -I../Logic -O0 -g3 -p -pg -Wall -c -fpermissive -fmessage-length=0 -coverage -fno-access-control -o ../Debug/Drivers/%%~nf.o %%~nf.cpp 2> "%~dp0\Debug\info\built\Drivers\%%~nf.log"
cd %~dp0\Logic
for /R "%~dp0\Logic" %%f in (*.cpp) do g++ -I../Drivers -I../Logic -O0 -g3 -p -pg -Wall -c -fpermissive -fmessage-length=0 -coverage -fno-access-control -o ../Debug/Logic/%%~nf.o %%~nf.cpp 2> "%~dp0\Debug\info\built\Logic\%%~nf.log"

REM link files
cd %~dp0\Debug
g++ -coverage -g -p -pg -o CodeCoverage.exe Drivers/serialPort.o Logic/interfaceHandler.o Logic/main.o 2> "%~dp0\Debug\info\built\CodeCoverage.log"

REM execute program
cd %~dp0\Debug
%~dp0\Debug\CodeCoverage.exe > %~dp0\Debug\info\CodeCoverage.log

REM copying of source files needed for GCOV
copy %~dp0\Drivers\*.* %~dp0\Debug\Drivers
copy %~dp0\Logic\*.* %~dp0\Debug\Logic


REM demangle files and store copies - not needed, only an option to have the GCOV files
cd %~dp0\Debug\Drivers
for %%f in (%~dp0\Debug\Drivers\*.gcno) do (
gcov -f -b "%%~nf.gcno" | C++filt -n  > "%~dp0\Debug\info\gcov\Drivers\%%~nf.log"
c++filt -n < %%~nf.cpp.gcov > %%~nf.cpp.gcov1
c++filt -n < %%~nf.hpp.gcov > %%~nf.hpp.gcov1
)

REM demangle files and store copies - not needed, only an option to have the GCOV files
cd %~dp0\Debug\Logic
for %%f in (%~dp0\Debug\Logic\*.gcno) do (
gcov -f -b "%%~nf.gcno" | C++filt -n  > "%~dp0\Debug\info\gcov\Logic\%%~nf.log"
c++filt -n < %%~nf.cpp.gcov > %%~nf.cpp.gcov1
c++filt -n < %%~nf.hpp.gcov > %%~nf.hpp.gcov1
)

REM remove original GCOV files
cd %~dp0\Debug\Drivers
del *.cpp.gcov
del *.hpp.gcov

cd %~dp0\Debug\Logic
del *.cpp.gcov
del *.h.gcov

cd %~dp0\Debug
del *.cpp.gcov
del *.hpp.gcov

REM launch LCOV and GENHTML to get HTML coverage report
cd %~dp0
C:\Perl\bin\Perl.exe lcov.perl --capture --directory Debug --output-file debug/coverage.info --rc lcov_branch_coverage=1
C:\Perl\bin\Perl.exe genhtml.perl --output HTML debug/coverage.info --demangle-cpp --prefix %~dp0 --function-coverage --branch-coverage --show-details

Code coverage - HTML reports using LCOV on windows

After generating code coverage there is a need to present the data in form other than gcov text files. There is a plugin (gcov plugin) for eclipse that it used for visualization but what when somebody without eclipse would like to view the reports?
There is a very good tool called LCOV which consists of: lcov and geninfo which provide a file with code coverage data and genhtml which transforms the coverage file to set of html files.
The problem is that these scripts are written for Linux environment and use Perl.
To run them on windows it is needed to:

1. Install Perl (preferably under C:\Perl)
2. Associate perl scripts with perl - very important is not to use windows association but to use commands:
   assoc .perl=Perl.File
   ftype Perl.File=C:\Perl\bin\perl.exe "%1" %*
   After associating files under windows the arguments are not passed?
3. Download of the modified script available here.

After finishing these steps it is possible to invoke perl scripts by simply using windows command line and typing for example:

C:\Perl\bin\Perl.exe lcov.perl --capture --directory Debug --output-file debug/coverage.info --rc lcov_branch_coverage=1

in order to generate file with code coverage and typing:

C:\Perl\bin\Perl.exe genhtml.perl --output HTML debug/coverage.info --demangle-cpp --prefix %~dp0 --function-coverage --branch-coverage --show-details

to generate html reports.

Code coverage - embedded target

As I have promised before now I will try to explain how to get code coverage from an embedded target in "the proper way".
The proper way means that it will be done automatically and that files with coverage will be merged instead of being overwritten.

To achieve all of the above two thing are needed:

1. Interface used for handling files from the target
2. PC application that will interact with the target through the provided interface and handle code coverage files

For the interface I have used serial port. The methods for: _open(), _read(), _write() and _close() have been changed so that they will interface with PC application and send the name of the file, check if it exists and finally read and write the file. Source code of newlib_stub.c with methods that interface serial port is available here.

The PC application is quite simple. It just captures the data from the serial port and searches for some special KEY words like fopen, fread, fwrite.
The program can be downloaded from my sourceforge page.
Source code available on GitHub.

newlib_stubs.c

/*
 * newlib_stubs.c
 *
 *  Created: 22.07.2014 / 2 Nov 2010
 *  Author: Filip Ryborz / nanoage.co.uk
 *  Website: simply-embedded.blogspot.com
 */

#include <errno.h>
#include "stm32f10x_usart.h"

#undef errno
extern int errno;

/*
 environ
 A pointer to a list of environment variables and their values.
 For a minimal environment, this empty list is adequate:
 */
char *__env[1] = { 0 };
char **environ = __env;

int _write(int file, char *ptr, int len);

/*
 Status of an open file. For consistency with other minimal implementations in these examples,
 all files are regarded as character special devices.
 The `sys/stat.h' header file required is distributed in the `include' subdirectory for this C library.
 */
int _fstat(int file, struct stat *st)
{
 st->st_mode = S_IFCHR;

 return 0;
}

/*
 Process-ID; this is sometimes used to generate strings unlikely to conflict with other processes. Minimal implementation, for a system without processes:
 */
int _getpid()
{
    return 1;
}

/*
 Query whether output stream is a terminal. For consistency with the other minimal implementations,
 */
int _isatty(int file)
{
    switch (file)
    {
    case STDOUT_FILENO:
    case STDERR_FILENO:
    case STDIN_FILENO:
        return 1;
    default:
        errno = EBADF;
        return 0;
    }
}

/*
 Send a signal. Minimal implementation:
 */
int _kill(int pid, int sig)
{
 errno = EINVAL;

 return (-1);
}

/*
 Set position in a file. Minimal implementation:
 */
int _lseek(int file, int ptr, int dir)
{
 return 0;
}

/*
 Increase program data space.
 Malloc and related functions depend on this
 */
caddr_t _sbrk(int incr)
{
 extern char _ebss; // Defined by the linker
 static char *heap_end;
 char *prev_heap_end;

 if (heap_end == 0)
 {
  heap_end = &_ebss;
 }
 prev_heap_end = heap_end;

char * stack = (char*) __get_MSP();
 if (heap_end + incr >  stack)
 {
  _write (STDERR_FILENO, "Heap and stack collision\n", 25);
  errno = ENOMEM;
  return  (caddr_t) -1;
 }

 heap_end += incr;

 return (caddr_t) prev_heap_end;
}

/*
 Remove a file's directory entry. Minimal implementation:
 */
int _unlink(char *name)
{
 errno = ENOENT;

 return -1;
}

void _exit(int status)
{
 __gcov_flush();
}

int _open (const char *ptr, int mode)
{
 int ret = 0;
 unsigned char c;

 // send file open command
 tiny_printf("fOpen(%s, %d) %\0", ptr, mode);

 while (tiny_get(&c) == 0) {}
 ret = c << 8;
 while (tiny_get(&c) == 0) {}
 ret += c;

 return ret;   /* Number of bytes written. */
}

/*
 Read a character to a file. `libc' subroutines will use this system routine for input from all files, including stdin
 Returns -1 on error or blocks until the number of characters have been read.
 */
int _read(int file, char *ptr, int len)
{
 int n;
 int num = 0;
 int size = 0;
 unsigned char c;

 // send file read command
 tiny_printf("fRead %d, %d| %\0", file, len);

 // get file size
 while (tiny_get(&c) == 0) {}
 size = c << 8;
 while (tiny_get(&c) == 0) {}
 size += c;
 if (size < len) {len = size;}

 switch (file) {
 case STDIN_FILENO:
   for (n = 0; n < len; n++)
   {
    while (tiny_get(&c) == 0) {}
    *ptr++ = c;
    num++;
   }
   break;
 default:
   errno = EBADF;
   return -1;
 }

 return num;
}

/*
 Write a character to a file. `libc' subroutines will use this system routine for output to all files, including stdout
 Returns -1 on error or number of bytes sent
 */
int _write(int file, char *ptr, int len) {

 int n;
 unsigned char c;

 // send file write command
 tiny_printf("fWrite %d, %d| %\0", file, len);

 while (tiny_get(&c) == 0) {} // wait for ack

 switch (file) {
 case STDOUT_FILENO: /*stdout*/
   for (n = 0; n < 1; n++)
   {
   }
   break;
 case STDERR_FILENO: /* stderr */
   for (n = 0; n < 1; n++)
   {
   }
   break;
 default:
   //errno = EBADF;
   //return -1;
  for (n = 0; n < len; n++)
  {
   fputc(*ptr++, NULL);
  }
 }

 return len;
}

int _close(int file)
{
 unsigned char c;

 // send close file command
 tiny_printf("fClose() %\0");

 while (tiny_get(&c) == 0) {} // wait for ack

 return -1;
}

Code coverage - embedded target - simple approach

In this example I will try to show how to get code coverage data from an embedded target using only GDB debugger.
Project is written for STM32F105 in C under Eclipse IDE using GCC.

As I said before in order to build the project we need to provide some stubs for the methods described in Code coverage - introduction. I have used file with stubs from nano-age and made most of the methods empty. It is enough for the code to compile and it will do in the simple approach. In the next post I will show how to do it "the proper way".

In the simple approach we will use GDB and dump command to dump the file with code coverage from the target to hard drive.
The reason why I'm not calling this "the proper way" is that this approach is not fully automated (our intervention is needed to store the files) and we loose data because we overwrite files instead of merging them. Normally the files should be merged in order to accumulate code coverage from different runs.

The newlib_stubs.c file with needed stubs (most of them empty) is located at the end of this post.

Now that stubs are provided we can successfully compile the code.
In the example I have implemented simple serial port methods and checking character received from serial port in the main program loop.
After receiving 'e' character program calls _exit(0) method and starts generating code coverage.

Step by step instructions:

1. Build project with "--coverage" option both for compiling and linking
2. Set breakpoints in stubs of _open() and _write() methods
3. Start debugging session with GDB
4. Perform some operations
5.  Call _exit(0) either from an external interface or by setting PC to that method (address can be found in lss file)
6. When program reaches breakpoint in _open method then copy the name of file  (FILENAME) to be opened from ptr argument and run the program
7. When program reaches breakpoint in _write method then copy the memory address (HEX_START_ADDR) from the ptr argument and size from len argument
8. Open GDB console and dump memory using: "dump binary memory FILENAME.gcda HEX_START_ADDR HEX_END_ADDR"
9. Repeat 6-8 if coverage for more than one file is needed
10. Now both .gcno and .gcda files are present. Refresh the project to invoke gcov and obtain code coverage (eclipse gcov plugin needed)

Here is a short video with demonstration:

And here is how the output will more or less look like in eclipse:

newlib_stubs.c

/*
 * newlib_stubs.c
 *
 *  Created on: 2 Nov 2010
 *      Author: nanoage.co.uk
 */
extern int errno;

char *__env[1] = { 0 };
char **environ = __env;

int _write(int file, char *ptr, int len);

void _exit(int status)
{
 __gcov_flush();
}

int _close(int file)
{
    return -1;
}

int _fstat(int file, struct stat *st)
{
    st->st_mode = S_IFCHR;
    return 0;
}

int _getpid()
{
    return 1;
}

int _isatty(int file) 
{
    switch (file)
    {
    case STDOUT_FILENO:
    case STDERR_FILENO:
    case STDIN_FILENO:
        return 1;
    default:
        errno = EBADF;
        return 0;
    }
}

int _kill(int pid, int sig)
{
    errno = EINVAL;
    return (-1);
}

int _lseek(int file, int ptr, int dir)
{
    return 0;
}

void *_sbrk(int incr)
{
    extern char _ebss; // Defined by the linker
    static char *heap_end;
    char *prev_heap_end;

    if (heap_end == 0)
    {
        heap_end = &_ebss;
    }
    prev_heap_end = heap_end;

    char * stack = (char*) __get_MSP();

     if (heap_end + incr >  stack)
     {
         _write (STDERR_FILENO, "Heap and stack collision\n", 25);
         errno = ENOMEM;
         return  (void *) -1;
     }

    heap_end += incr;
    return (void *) prev_heap_end;
}

int _read(int file, char *ptr, int len)
{
 return 0;
}

int _unlink(char *name)
{
    errno = ENOENT;
    return -1;
}

int _write(int file, char *ptr, int len)
{
    return 0;
}

int _open (const char *ptr, int mode)
{
 return 0;
}