From 165907550c8f6ce7506beef591f55cd3f8458d78 Mon Sep 17 00:00:00 2001 From: nicole mazzuca Date: Fri, 2 Aug 2019 09:52:39 -0700 Subject: [PATCH] Update tests, and add documentation! (#7506) This PR does the following: * fix tests -- now, they're always built in the CMake scripts, and they work on VS2015 *add a new flag, BUILD_TESTING, which allows one to turn off testing builds * Add documentation for running tests --- docs/index.md | 5 + docs/tool-maintainers/testing.md | 162 +++++++++++++++++++++++++++++++ scripts/bootstrap.sh | 2 +- toolsrc/CMakeLists.txt | 78 +++++++-------- toolsrc/src/vcpkg-test/util.cpp | 19 ++-- 5 files changed, 212 insertions(+), 54 deletions(-) create mode 100644 docs/tool-maintainers/testing.md diff --git a/docs/index.md b/docs/index.md index 5d83b5804..b2f0a53b2 100644 --- a/docs/index.md +++ b/docs/index.md @@ -21,6 +21,11 @@ Vcpkg helps you manage C and C++ libraries on Windows, Linux and MacOS. This too - [Portfile functions](maintainers/portfile-functions.md) - [Maintainer Guidelines](maintainers/maintainer-guide.md) +### Tool Maintainer Help + +- [Testing](tool-maintainers/testing.md) +- [Maintainer Guidelines](maintainers/maintainer-guide.md) + ### Specifications - [Export](specifications/export-command.md) diff --git a/docs/tool-maintainers/testing.md b/docs/tool-maintainers/testing.md new file mode 100644 index 000000000..28cc9e099 --- /dev/null +++ b/docs/tool-maintainers/testing.md @@ -0,0 +1,162 @@ +Testing +======= + +Testing vcpkg is important whenever one makes changes to the tool itself, and +writing new tests and keeping them up to date is also very important. If one's +code is subtly broken, we'd rather find it out right away than a few weeks down +the line when someone complains! + +Running Tests +------------- + +Before anything else, we should know whether you can actually run the tests! +All you should need is a way to build vcpkg -- anything will do! All you have to +do is follow the guide 😄 + +With `$VCPKG_DIRECTORY` being the directory where you have cloned vcpkg, create +a build directory in `$VCPKG_DIRECTORY/toolsrc` (commonly named `out`), and +`cd` into it. Make sure to clean it out if it already exists! + +```sh +$ cmake .. -DCMAKE_BUILD_TYPE=Debug -G Ninja +$ cmake --build . +$ ./vcpkg-test # ./vcpkg-test [$SPECIFIC_TEST] for a specific set of tests +$ # i.e., ./vcpkg-test [arguments] +``` + +If you make any modifications to `vcpkg`, you'll have to do the +`cmake --build .` step again. + +Writing Tests +------------- + +In your journey to write new tests, and to modify existing tests, reading the +[Catch2 documentation] will be very helpful! Come back after reading those 😀 + +You'll want to place your tests in one of the existing files, or, if it doesn't +belong in any of those, in a [new file](#adding-new-test-files). + +The layout of these tests is as follows: + +```cpp +// ... includes + +TEST_CASE("Name of test", "[filename without the .cpp]") { + // setup and the like + REQUIRE(some boolean expression); +} + +// etc. +``` + +You want to give these test cases good, descriptive, unique names, like +`SourceParagraph construct minimum` -- it doesn't need to be extremely clear +english, and shorthand is good, but make sure it's clear what the test is from +the name. For the latter parameter, known as "tags", you should at least put the +name of the file which the test case is in -- e.g., in `arguments.cpp`, you'd +tag all of the test cases with `[arguments]`. + +If you wish to add helper functions, make sure to place them in an anonymous +namespace -- this will ensure that they don't trample over anybody else's +space. Additionally, there are a few helper functions that live in +`` and `src/vcpkg-test/util.cpp` -- make sure to look into +them so that you're not rewriting functionality. + +That should be all you need to know to start writing your own tests! +Remember to check out the [Catch2 documentation] +if you'd like to get more advanced with your tests, +and good luck on your testing journey! + +Adding New Test Files +--------------------- + +Adding new test files should be easy and straightforward. All it requires is +creating a new source file in `toolsrc/src/vcpkg-test`, and then rerunning +`CMake` in order to pick up the glob changes. + +### Example + +Let's try writing a new test file called `example` (very creative, I know). + +First, we should create a file, `example.cpp`, in `toolsrc/src/vcpkg-test`: + +```cpp +// vcpkg-test/example.cpp +#include +``` + +This is the minimum file needed for tests; let's rebuild our CMake directory. +You'll have to clean out the existing `out` directory for CMake to rerun +globbing. + +```sh +$ cmake .. -DCMAKE_BUILD_TYPE=Debug -G Ninja +# ... +-- Build files have been written to: $VCPKG_DIRECTORY/toolsrc/out +$ cmake --build . +[80/80] Linking CXX executable vcpkg.exe +``` + +Okay, now let's make sure this worked; add a test case to `example.cpp`: + +```cpp +TEST_CASE("Example 1 - fail", "[example]") { + REQUIRE(false); +} +``` + +Now build the tests again, and run them: + +```sh +$ cmake --build . +[2/2] Linking CXX executable vcpkg-test.exe +$ ./vcpkg-test + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +vcpkg-test.exe is a Catch v2.9.1 host application. +Run with -? for options + +------------------------------------------------------------------------------- +Example 1 - fail +------------------------------------------------------------------------------- +$VCPKG_DIRECTORY/toolsrc/src/vcpkg-test/example.cpp(3) +............................................................................... + +$VCPKG_DIRECTORY/toolsrc/src/vcpkg-test/example.cpp(14): FAILED: + REQUIRE( false ) + +=============================================================================== +test cases: 102 | 101 passed | 1 failed +assertions: 3611 | 3610 passed | 1 failed +``` + +Hopefully, that worked! It should compile correctly, and have one failing test. +Now let's try a more complex test, after deleting the old one; + +```cpp +// add #include to the top of the file +namespace Strings = vcpkg::Strings; + +TEST_CASE("Example 2 - success", "[example]") { + std::string hello = "Hello"; + REQUIRE(Strings::case_insensitive_ascii_equals(hello, "hELLo")); + REQUIRE_FALSE(Strings::case_insensitive_ascii_starts_with(hello, "E")); +} +``` + +Now compile and build the tests, and this time let's only run our example tests: + +```sh +$ cmake --build . +[2/2] Linking CXX executable vcpkg-test.exe +$ ./vcpkg-test [example] +Filters: [example] +=============================================================================== +All tests passed (2 assertions in 1 test case) +``` + +Hopefully you have one test running and succeeding! If you have that, you have +succeeded at adding a new file to vcpkg's tests. Congratulations! Have fun on +the rest of your journey 🐱‍👤😁 + +[Catch2 documentation]: https://github.com/catchorg/Catch2/blob/master/docs/tutorial.md#top diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh index b0e4f98a5..09d77b07d 100644 --- a/scripts/bootstrap.sh +++ b/scripts/bootstrap.sh @@ -249,7 +249,7 @@ buildDir="$vcpkgRootDir/toolsrc/build.rel" rm -rf "$buildDir" mkdir -p "$buildDir" -(cd "$buildDir" && CXX=$CXX "$cmakeExe" .. -DCMAKE_BUILD_TYPE=Release -G "Ninja" "-DCMAKE_MAKE_PROGRAM=$ninjaExe" "-DDEFINE_DISABLE_METRICS=$vcpkgDisableMetrics" "-DVCPKG_ALLOW_APPLE_CLANG=$vcpkgAllowAppleClang") || exit 1 +(cd "$buildDir" && CXX=$CXX "$cmakeExe" .. -DCMAKE_BUILD_TYPE=Release -G "Ninja" "-DCMAKE_MAKE_PROGRAM=$ninjaExe" "-DBUILD_TESTING=OFF" "-DDEFINE_DISABLE_METRICS=$vcpkgDisableMetrics" "-DVCPKG_ALLOW_APPLE_CLANG=$vcpkgAllowAppleClang") || exit 1 (cd "$buildDir" && "$cmakeExe" --build .) || exit 1 rm -rf "$vcpkgRootDir/vcpkg" diff --git a/toolsrc/CMakeLists.txt b/toolsrc/CMakeLists.txt index 5bfae1511..3874412db 100644 --- a/toolsrc/CMakeLists.txt +++ b/toolsrc/CMakeLists.txt @@ -2,11 +2,16 @@ cmake_minimum_required(VERSION 3.3) project(vcpkg C CXX) -enable_testing() - +OPTION(BUILD_TESTING "Option for enabling testing" ON) OPTION(DEFINE_DISABLE_METRICS "Option for disabling metrics" OFF) OPTION(VCPKG_ALLOW_APPLE_CLANG "Option for allowing apple clang" OFF) +if (DEFINE_DISABLE_METRICS) + set(DISABLE_METRICS_VALUE "1") +else() + set(DISABLE_METRICS_VALUE "0") +endif() + if(CMAKE_COMPILER_IS_GNUXX OR CMAKE_CXX_COMPILER_ID MATCHES "GNU") set(GCC 1) elseif(CMAKE_CXX_COMPILER_ID MATCHES "AppleClang") @@ -27,46 +32,12 @@ else() message(FATAL_ERROR "Unknown compiler: ${CMAKE_CXX_COMPILER_ID}") endif() -if(GCC OR CLANG) - add_compile_options(-std=c++1z) - if(WERROR) - add_compile_options(-Wall -Wno-unknown-pragmas -Werror) - endif() -endif() - -file(GLOB_RECURSE VCPKGLIB_SOURCES src/vcpkg/*.cpp) -file(GLOB_RECURSE VCPKGTEST_SOURCES src/vcpkg-test/*.cpp) - -if (DEFINE_DISABLE_METRICS) - set(DISABLE_METRICS_VALUE "1") -else() - set(DISABLE_METRICS_VALUE "0") -endif() - -add_library(vcpkglib OBJECT ${VCPKGLIB_SOURCES}) -add_executable(vcpkg src/vcpkg.cpp $) -add_executable(vcpkg-test - EXCLUDE_FROM_ALL - ${VCPKGTEST_SOURCES} - $) - -foreach(TARGET_NAME vcpkglib vcpkg vcpkg-test) - target_compile_definitions(${TARGET_NAME} PRIVATE -DDISABLE_METRICS=${DISABLE_METRICS_VALUE}) - target_include_directories(${TARGET_NAME} PRIVATE include) -endforeach() - set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) -target_link_libraries(vcpkg PRIVATE Threads::Threads) -target_link_libraries(vcpkg-test PRIVATE Threads::Threads) -foreach(TEST_NAME - arguments chrono dependencies files - paragraph plan specifier statusparagraphs - strings supports update -) - add_test(${TEST_NAME} vcpkg-test [${TEST_NAME}]) -endforeach() +add_definitions(-DDISABLE_METRICS=${DISABLE_METRICS_VALUE}) +include_directories(include) +link_libraries(Threads::Threads) if(CLANG) include(CheckCXXSourceCompiles) @@ -80,11 +51,32 @@ if(CLANG) endif() if(GCC OR (CLANG AND USES_LIBSTDCXX)) - target_link_libraries(vcpkg PRIVATE stdc++fs) - target_link_libraries(vcpkg-test PRIVATE stdc++fs) + link_libraries(stdc++fs) elseif(CLANG) - target_link_libraries(vcpkg PRIVATE c++fs) - target_link_libraries(vcpkg-test PRIVATE c++fs) + link_libraries(c++fs) +endif() + +if(GCC OR CLANG) + add_compile_options(-std=c++1z) + if(WERROR) + add_compile_options(-Wall -Wno-unknown-pragmas -Werror) + endif() +endif() + +file(GLOB_RECURSE VCPKGLIB_SOURCES src/vcpkg/*.cpp) + +add_library(vcpkglib OBJECT ${VCPKGLIB_SOURCES}) +add_executable(vcpkg src/vcpkg.cpp $) + +if (BUILD_TESTING) + file(GLOB_RECURSE VCPKGTEST_SOURCES src/vcpkg-test/*.cpp) + + enable_testing() + add_executable(vcpkg-test + ${VCPKGTEST_SOURCES} + $) + + add_test(NAME default COMMAND vcpkg-test [${TEST_NAME}]) endif() if(MSVC) diff --git a/toolsrc/src/vcpkg-test/util.cpp b/toolsrc/src/vcpkg-test/util.cpp index a80ab36a0..5359b0fad 100644 --- a/toolsrc/src/vcpkg-test/util.cpp +++ b/toolsrc/src/vcpkg-test/util.cpp @@ -76,12 +76,13 @@ namespace vcpkg::Test static bool system_allows_symlinks() { -#if defined(_WIN32) - if (!__cpp_lib_filesystem) - { - return false; - } - +#if FILESYSTEM_SYMLINK == FILESYSTEM_SYMLINK_NONE + return false; +#elif FILESYSTEM_SYMLINK == FILESYSTEM_SYMLINK_UNIX + return true; +#elif !defined(_WIN32) // FILESYSTEM_SYMLINK == FILESYSTEM_SYMLINK_STD + return true; +#else HKEY key; bool allow_symlinks = true; @@ -97,8 +98,6 @@ namespace vcpkg::Test if (status == ERROR_SUCCESS) RegCloseKey(key); return allow_symlinks; -#else - return true; #endif } @@ -125,8 +124,8 @@ namespace vcpkg::Test const bool SYMLINKS_ALLOWED = system_allows_symlinks(); const fs::path TEMPORARY_DIRECTORY = internal_temporary_directory(); -#if FILESYSTEM_SYMLINK == FILSYSTEM_SYMLINK_NONE - constexpr inline char no_filesystem_message[] = +#if FILESYSTEM_SYMLINK == FILESYSTEM_SYMLINK_NONE + constexpr char no_filesystem_message[] = " doesn't exist; on windows, we don't attempt to use the win32 calls to create symlinks"; #endif