Merge pull request #4607 from telefonicaid/hardening/4418-fix-coverag…

…e-functional-tests

Improve coverage framework

CMakeLists.txt

@@ -305,13 +305,13 @@ link_directories("/usr/lib/x86_64-linux-gnu")
 #
 # Enabling test harness
 #
-if (UNIT_TEST)
+if (COVERAGE AND NOT UNIT_TEST)
    ENABLE_TESTING()
    INCLUDE(Dart)
    SET (CMAKE_TEST_TIMEOUT 60)
    SET (DART_TIMEOUT 60)
-   #ADD_SUBDIRECTORY(test/harness)
-endif (UNIT_TEST)
+   ADD_SUBDIRECTORY(test/functionalTest/cases)
+endif (COVERAGE AND NOT UNIT_TEST)
 
 #
 # Continuing if OK

doc/manuals/admin/build_source.md

@@ -148,3 +148,5 @@ You can generate coverage reports for the Orion Context Broker using the followi
 * Run coverage
 
         make coverage INSTALL_DIR=~
+
+*NOTE*: Functional tests relying in debug traces are expected to fail under coverage execution (e.g. notification_different_sizes or not_posix_regex_idpattern.test). This is due to the LM_T macros used by the debug traces are disabled in the coverage code build, as they add "noise" in condition coverage. This way coverage reports are more useful.

makefile

@@ -54,6 +54,19 @@ ifndef MONGO_HOST
     MONGO_HOST=localhost
 endif
 
+# Macros
+define CLEAN_COVERAGE_REPORT
+	lcov -r coverage/broker.info "/usr/include/*" -o coverage/broker.info
+	lcov -r coverage/broker.info "/usr/local/include/*" -o coverage/broker.info
+	lcov -r coverage/broker.info "/opt/local/include/google/*" -o coverage/broker.info
+	# Remove unit test libraries and libraries developed before contextBroker project init
+	lcov -r coverage/broker.info "*/test/unittests/*" -o coverage/broker.info
+	lcov -r coverage/broker.info "*/src/lib/logMsg/*" -o coverage/broker.info
+	lcov -r coverage/broker.info "*/src/lib/parseArgs/*" -o coverage/broker.info
+	# app/ contains application itself, not libraries which make sense to measure unit_test coverage
+	lcov -r coverage/broker.info "*/src/app/*" -o coverage/broker.info
+endef
+
 all: prepare_release release
 
 di: install_debug
@@ -73,14 +86,17 @@ prepare_debug: compile_info
 	cd BUILD_DEBUG && cmake .. -DCMAKE_BUILD_TYPE=DEBUG -DBUILD_ARCH=$(BUILD_ARCH) -DDEBUG=True -DCMAKE_INSTALL_PREFIX=$(INSTALL_DIR)
 
 prepare_coverage: compile_info
+	# FIXME: should we use BUILD_COVERAGE for this or use BUILD_DEBUG? (note that for unit test we don't have BUILT_UNITTEST_COVERAGE....)
 	mkdir -p  BUILD_COVERAGE || true
-	cd BUILD_COVERAGE && cmake .. -DCMAKE_BUILD_TYPE=DEBUG -DBUILD_ARCH=$(BUILD_ARCH) -DUNIT_TEST=True -DCOVERAGE=True -DCMAKE_INSTALL_PREFIX=$(INSTALL_DIR)
+	cd BUILD_COVERAGE && cmake .. -DCMAKE_BUILD_TYPE=DEBUG -DBUILD_ARCH=$(BUILD_ARCH) -DCOVERAGE=True -DCMAKE_INSTALL_PREFIX=$(INSTALL_DIR)
 
 prepare_unit_test: compile_info
-	@echo '------------------------------- prepare_unit_test starts ---------------------------------'
+	mkdir -p  BUILD_UNITTEST || true
+	cd BUILD_UNITTEST && cmake .. -DCMAKE_BUILD_TYPE=DEBUG -DBUILD_ARCH=$(BUILD_ARCH) -DUNIT_TEST=True -DCMAKE_INSTALL_PREFIX=$(INSTALL_DIR)
+
+prepare_unit_test_coverage: compile_info
 	mkdir -p  BUILD_UNITTEST || true
 	cd BUILD_UNITTEST && cmake .. -DCMAKE_BUILD_TYPE=DEBUG -DBUILD_ARCH=$(BUILD_ARCH) -DUNIT_TEST=True -DCOVERAGE=True -DCMAKE_INSTALL_PREFIX=$(INSTALL_DIR)
-	@echo '------------------------------- prepare_unit_test ended ---------------------------------'
 
 release: prepare_release
 	cd BUILD_RELEASE && make -j$(CPU_COUNT)
@@ -209,18 +225,17 @@ uncrustify_changed:
 	git diff --name-only | grep "\.cpp\|\.h" | xargs uncrustify --no-backup -c scripts/uncrustify.cfg
 
 build_unit_test: prepare_unit_test 
-	@echo '------------------------------- build_unit_test starts ---------------------------------'
 	cd BUILD_UNITTEST && make -j$(CPU_COUNT)
-	@echo '------------------------------- build_unit_test ended ---------------------------------'
+
+build_unit_test_coverage: prepare_unit_test_coverage 
+	cd BUILD_UNITTEST && make -j$(CPU_COUNT)
 
 unit_test: build_unit_test
-	@echo '------------------------------- unit_test starts ---------------------------------'
 	if [ -z "${TEST_FILTER}" ]; then \
 	   BUILD_UNITTEST/test/unittests/unitTest -t 0-255 -dbURI mongodb://${MONGO_HOST} --gtest_output=xml:BUILD_UNITTEST/unit_test.xml; \
         else \
 	   BUILD_UNITTEST/test/unittests/unitTest -t 0-255 -dbURI mongodb://${MONGO_HOST} --gtest_output=xml:BUILD_UNITTEST/unit_test.xml --gtest_filter=${TEST_FILTER}; \
         fi
-	@echo '------------------------------- unit_test ended ---------------------------------'
 
 functional_test: install
 	./test/functionalTest/testHarness.sh
@@ -233,66 +248,34 @@ ftd: functional_test_debug
 
 test: unit_test functional_test
 
-coverage: install_coverage
-	# FIXME #4418: the functional test part of this target is not working properly. Check issue for details.
-	# Init coverage
-	echo "Initializing coverage files"
-	mkdir -p coverage
-	lcov -i --zerocounters --directory BUILD_COVERAGE/
-	lcov --capture --initial --directory BUILD_COVERAGE -b BUILD_COVERAGE --output-file coverage/broker.init.info
-	# Execute test for coverage
-	echo "Executing coverage test"
-	BUILD_COVERAGE/test/unittests/unitTest -t 0-255 --gtest_output=xml:BUILD_COVERAGE/unit_test.xml
-	if [ -z "${CONTEXTBROKER_TESTENV_SOURCED}" ]; then \
-	    echo "Execute '. scripts/testEnv.sh' before executing the tests"; \
-	    exit 1; \
-	fi
-	make test -C BUILD_COVERAGE ARGS="-D ExperimentalTest" TEST_VERBOSE=1 || true
-	@if [ -e test/functionalTest/cases/*.diff ]; then \
-           echo "A .diff file was found in test/functionalTest/cases, which means that ctest failed running the test. This can happen if a \"Ok\""; \
-           echo "token is used in the tests specification. Run \"test/functionalTest/testHarness.sh test/functionalTest/cases\" manually to find the problem."; \
-	   exit 1; \
-	fi
-	@xsltproc scripts/cmake2junit.xsl BUILD_COVERAGE/Testing/`cat BUILD_COVERAGE/Testing/TAG| head -n1`/Test.xml  > BUILD_COVERAGE/functional_test.xml
+coverage: coverage_functional_test coverage_unit_test
+	rm -rf coverage
+	lcov --capture --directory BUILD_UNITTEST --output-file BUILD_UNITTEST/coverage.info
+	lcov --capture --directory BUILD_COVERAGE --output-file BUILD_COVERAGE/coverage.info
+	mkdir coverage
 	# Generate test report
 	echo "Generating coverage report"
-	lcov --directory BUILD_COVERAGE --capture -b BUILD_COVERAGE --output-file coverage/broker.test.info 
-	lcov --add-tracefile coverage/broker.init.info --add-tracefile coverage/broker.test.info --output-file coverage/broker.info
-	lcov -r coverage/broker.info "/usr/include/*" -o coverage/broker.info
-	lcov -r coverage/broker.info "/usr/local/include/*" -o coverage/broker.info
-	lcov -r coverage/broker.info "/opt/local/include/google/*" -o coverage/broker.info
-	# Remove unit test libraries and libraries developed before contextBroker project init
-	lcov -r coverage/broker.info "*/test/unittests/*" -o coverage/broker.info
-	lcov -r coverage/broker.info "*/src/lib/logMsg/*" -o coverage/broker.info
-	lcov -r coverage/broker.info "*/src/lib/parseArgs/*" -o coverage/broker.info
+	lcov --add-tracefile BUILD_UNITTEST/coverage.info --add-tracefile BUILD_COVERAGE/coverage.info --output-file coverage/broker.info
+	$(CLEAN_COVERAGE_REPORT)
 	genhtml -o coverage coverage/broker.info
 
-coverage_unit_test: build_unit_test
+coverage_unit_test: build_unit_test_coverage
 	# Init coverage
 	echo "Initializing coverage files"
 	mkdir -p coverage
 	lcov -i --zerocounters --directory BUILD_UNITTEST/
 	lcov --capture --initial --directory BUILD_UNITTEST -b BUILD_UNITTEST --output-file coverage/broker.init.info
 	# Execute test for coverage
 	echo "Executing coverage test"
-	BUILD_UNITTEST/test/unittests/unitTest -t 0-255 --gtest_output=xml:BUILD_UNITTEST/unit_test.xml
+	BUILD_UNITTEST/test/unittests/unitTest -t 0-255 -dbURI mongodb://${MONGO_HOST} --gtest_output=xml:BUILD_UNITTEST/unit_test.xml
 	# Generate test report
 	echo "Generating coverage report"
 	lcov --directory BUILD_UNITTEST --capture -b BUILD_UNITTEST --output-file coverage/broker.test.info 
 	lcov --add-tracefile coverage/broker.init.info --add-tracefile coverage/broker.test.info --output-file coverage/broker.info
-	lcov -r coverage/broker.info "/usr/include/*" -o coverage/broker.info
-	lcov -r coverage/broker.info "/usr/local/include/*" -o coverage/broker.info
-	lcov -r coverage/broker.info "/opt/local/include/google/*" -o coverage/broker.info
-	# Remove unit test libraries and libraries developed before contextBroker project init
-	lcov -r coverage/broker.info "*/test/unittests/*" -o coverage/broker.info
-	lcov -r coverage/broker.info "*/src/lib/logMsg/*" -o coverage/broker.info
-	lcov -r coverage/broker.info "*/src/lib/parseArgs/*" -o coverage/broker.info
-	# app/ contains application itself, not libraries which make sense to measure unit_test coverage
-	lcov -r coverage/broker.info "*/src/app/*" -o coverage/broker.info
+	$(CLEAN_COVERAGE_REPORT)
 	genhtml -o coverage coverage/broker.info
 
 coverage_functional_test: install_coverage
-	# FIXME #4418: this target is not working properly. Check issue for details.
 	# Init coverage
 	echo "Initializing coverage files"
 	mkdir -p coverage
@@ -315,13 +298,7 @@ coverage_functional_test: install_coverage
 	echo "Generating coverage report"
 	lcov --directory BUILD_COVERAGE --capture -b BUILD_COVERAGE --output-file coverage/broker.test.info 
 	lcov --add-tracefile coverage/broker.init.info --add-tracefile coverage/broker.test.info --output-file coverage/broker.info
-	lcov -r coverage/broker.info "/usr/include/*" -o coverage/broker.info
-	lcov -r coverage/broker.info "/usr/local/include/*" -o coverage/broker.info
-	lcov -r coverage/broker.info "/opt/local/include/google/*" -o coverage/broker.info
-	# Remove unit test libraries and libraries developed before contextBroker project init
-	lcov -r coverage/broker.info "*/test/unittests/*" -o coverage/broker.info
-	lcov -r coverage/broker.info "*/src/lib/logMsg/*" -o coverage/broker.info
-	lcov -r coverage/broker.info "*/src/lib/parseArgs/*" -o coverage/broker.info
+	$(CLEAN_COVERAGE_REPORT)
 	genhtml -o coverage coverage/broker.info
 
 valgrind: install_debug

test/unittests/convenience/AppendContextElementRequest_test.cpp

@@ -110,7 +110,6 @@ TEST(AppendContextElementRequest, release)
    AppendContextElementRequest  acer;
    std::string                  out;
    ContextAttribute*            caP = new ContextAttribute("caName", "caType", "121");
-   Metadata*                    mdP = new Metadata("mdName", "mdType", "122");
 
    acer.contextAttributeVector.push_back(caP);
 

test/unittests/ngsi10/UpdateContextResponse_test.cpp

@@ -44,7 +44,6 @@ TEST(UpdateContextResponse, jsonRender)
 
   UpdateContextResponse*   ucrP;
   ContextElementResponse*  cerP;
-  Metadata*                mdP;
   ContextAttribute*        caP;
   std::string              out;
 

test/unittests/ngsi9/DiscoverContextAvailabilityResponse_test.cpp

@@ -95,7 +95,6 @@ TEST(DiscoverContextAvailabilityResponse, jsonRender)
   ContextRegistrationResponse*          crrP;
   EntityId*                             eidP;
   ContextRegistrationAttribute*         attrP;
-  Metadata*                             mdP;
 
   utInit();