Merge bitcoin-core/secp256k1#1760: cmake: Add dynamic test discovery to improve parallelism

8354618e02 cmake: Set `LABELS` property for tests (Hennadii Stepanov) 29f26ec3cf cmake: Integrate DiscoverTests and normalize test names (Hennadii Stepanov) f95b263f23 cmake: Add DiscoverTests module (Hennadii Stepanov) 4ac651144b cmake, refactor: Deduplicate test-related code (Hennadii Stepanov) Pull request description: This PR implements the idea suggested in https://github.com/bitcoin-core/secp256k1/pull/1734#pullrequestreview-3284918572 and is based on the work from https://github.com/bitcoin/bitcoin/pull/33483. Here is an example of the `ctest` output: ``` $ ctest --test-dir build -j $(nproc) Test project /home/hebasto/dev/secp256k1/secp256k1/build Start 1: secp256k1.noverify_tests.selftest_tests Start 2: secp256k1.noverify_tests.all_proper_context_tests Start 3: secp256k1.noverify_tests.all_static_context_tests Start 4: secp256k1.noverify_tests.deprecated_context_flags_test <snip> 193/196 Test #31: secp256k1.noverify_tests.ecmult_constants ......................... Passed 5.32 sec 194/196 Test #184: secp256k1.tests.ellswift_xdh_correctness_tests .................... Passed 5.62 sec 195/196 Test #191: secp256k1.exhaustive_tests ........................................ Passed 6.97 sec 196/196 Test #126: secp256k1.tests.ecmult_constants .................................. Passed 9.60 sec 100% tests passed, 0 tests failed out of 196 Label Time Summary: secp256k1_example = 0.02 sec*proc (5 tests) secp256k1_exhaustive = 6.97 sec*proc (1 test) secp256k1_noverify_tests = 23.77 sec*proc (95 tests) secp256k1_tests = 43.67 sec*proc (95 tests) Total Test time (real) = 10.21 sec ``` For comparison, here is the output for the master branch on the same machine: ``` $ ctest --test-dir build -j $(nproc) Test project /home/hebasto/dev/secp256k1/secp256k1/build Start 1: secp256k1_noverify_tests Start 2: secp256k1_tests Start 3: secp256k1_exhaustive_tests Start 4: secp256k1_ecdsa_example Start 5: secp256k1_ecdh_example Start 6: secp256k1_schnorr_example Start 7: secp256k1_ellswift_example Start 8: secp256k1_musig_example 1/8 Test #4: secp256k1_ecdsa_example .......... Passed 0.00 sec 2/8 Test #5: secp256k1_ecdh_example ........... Passed 0.00 sec 3/8 Test #6: secp256k1_schnorr_example ........ Passed 0.00 sec 4/8 Test #7: secp256k1_ellswift_example ....... Passed 0.00 sec 5/8 Test #8: secp256k1_musig_example .......... Passed 0.00 sec 6/8 Test #3: secp256k1_exhaustive_tests ....... Passed 6.26 sec 7/8 Test #1: secp256k1_noverify_tests ......... Passed 14.31 sec 8/8 Test #2: secp256k1_tests .................. Passed 31.65 sec 100% tests passed, 0 tests failed out of 8 Total Test time (real) = 31.65 sec ``` --- **New Feature:** As the number of tests has grown, the _labels_ have been introduced to simplify test management. Now, one can run: ``` $ ctest --test-dir build -j $(nproc) -L example Test project /home/hebasto/dev/secp256k1/secp256k1/build Start 192: secp256k1.example.ecdsa Start 193: secp256k1.example.ecdh Start 194: secp256k1.example.schnorr Start 195: secp256k1.example.ellswift Start 196: secp256k1.example.musig 1/5 Test #192: secp256k1.example.ecdsa .......... Passed 0.00 sec 2/5 Test #193: secp256k1.example.ecdh ........... Passed 0.00 sec 3/5 Test #194: secp256k1.example.schnorr ........ Passed 0.00 sec 4/5 Test #195: secp256k1.example.ellswift ....... Passed 0.00 sec 5/5 Test #196: secp256k1.example.musig .......... Passed 0.00 sec 100% tests passed, 0 tests failed out of 5 Label Time Summary: secp256k1_example = 0.01 sec*proc (5 tests) Total Test time (real) = 0.01 sec ``` or ``` $ ctest --test-dir build -j $(nproc) -LE tests Test project /home/hebasto/dev/secp256k1/secp256k1/build Start 192: secp256k1.example.ecdsa Start 193: secp256k1.example.ecdh Start 194: secp256k1.example.schnorr Start 195: secp256k1.example.ellswift Start 196: secp256k1.example.musig Start 191: secp256k1.exhaustive_tests 1/6 Test #192: secp256k1.example.ecdsa .......... Passed 0.00 sec 2/6 Test #193: secp256k1.example.ecdh ........... Passed 0.00 sec 3/6 Test #194: secp256k1.example.schnorr ........ Passed 0.00 sec 4/6 Test #195: secp256k1.example.ellswift ....... Passed 0.00 sec 5/6 Test #196: secp256k1.example.musig .......... Passed 0.00 sec 6/6 Test #191: secp256k1.exhaustive_tests ....... Passed 6.19 sec 100% tests passed, 0 tests failed out of 6 Label Time Summary: secp256k1_example = 0.01 sec*proc (5 tests) secp256k1_exhaustive = 6.19 sec*proc (1 test) Total Test time (real) = 6.20 sec ``` ACKs for top commit: purpleKarrot: ACK 8354618e02 furszy: Tested ACK 8354618 Tree-SHA512: 8c506ab08491aba4836b3058a8a09c929c6dd097c11e4e6f4deb20cf602285e73c3fd8a2c2040f7e92a058c7f8fc09752fa9de2ce80f7673adbdd505237ed262
2026-02-19 15:02:44 +01:00
parent c7a7f732bd 8354618e02
commit ac561601b8
3 changed files with 96 additions and 11 deletions
--- a/cmake/DiscoverTests.cmake
+++ b/cmake/DiscoverTests.cmake
@@ -0,0 +1,71 @@
+# TODO: rework/remove once test discovery is implemented upstream:
+# https://gitlab.kitware.com/cmake/cmake/-/issues/26920
+function(discover_tests target)
+  set(options "")
+  set(oneValueArgs DISCOVERY_MATCH TEST_NAME_REPLACEMENT TEST_ARGS_REPLACEMENT)
+  set(multiValueArgs DISCOVERY_ARGS PROPERTIES)
+  cmake_parse_arguments(PARSE_ARGV 1 arg "${options}" "${oneValueArgs}" "${multiValueArgs}")
+
+  set(file_base ${CMAKE_CURRENT_BINARY_DIR}/${target})
+  set(include_file ${file_base}_include.cmake)
+
+  set(properties_content)
+  list(LENGTH arg_PROPERTIES properties_len)
+  if(properties_len GREATER "0")
+    set(properties_content "      set_tests_properties(\"\${test_name}\" PROPERTIES\n")
+    math(EXPR num_properties "${properties_len} / 2")
+    foreach(i RANGE 0 ${num_properties} 2)
+      math(EXPR value_index "${i} + 1")
+      list(GET arg_PROPERTIES ${i} name)
+      list(GET arg_PROPERTIES ${value_index} value)
+      string(APPEND properties_content "        \"${name}\" \"${value}\"\n")
+    endforeach()
+    string(APPEND properties_content "      )\n")
+  endif()
+
+  string(CONCAT include_content
+    "set(runner   [[$<TARGET_FILE:${target}>]])\n"
+    "set(launcher [[$<TARGET_PROPERTY:${target},TEST_LAUNCHER>]])\n"
+    "set(emulator [[$<$<BOOL:${CMAKE_CROSSCOMPILING}>:$<TARGET_PROPERTY:${target},CROSSCOMPILING_EMULATOR>>]])\n"
+    "\n"
+    "execute_process(\n"
+    "  COMMAND \${launcher} \${emulator} \${runner} ${arg_DISCOVERY_ARGS}\n"
+    "  OUTPUT_VARIABLE output OUTPUT_STRIP_TRAILING_WHITESPACE\n"
+    "  ERROR_VARIABLE  output ERROR_STRIP_TRAILING_WHITESPACE\n"
+    "  RESULT_VARIABLE result\n"
+    ")\n"
+    "\n"
+    "if(NOT result EQUAL 0)\n"
+    "  add_test([[${target}_DISCOVERY_FAILURE]] \${launcher} \${emulator} \${runner} ${arg_DISCOVERY_ARGS})\n"
+    "else()\n"
+    "  string(REPLACE \"\\n\" \";\" lines \"\${output}\")\n"
+    "  foreach(line IN LISTS lines)\n"
+    "    if(line MATCHES \"${arg_DISCOVERY_MATCH}\")\n"
+    "      string(REGEX REPLACE \"${arg_DISCOVERY_MATCH}\" \"${arg_TEST_NAME_REPLACEMENT}\" test_name \"\${line}\")\n"
+    "      string(REGEX REPLACE \"${arg_DISCOVERY_MATCH}\" \"${arg_TEST_ARGS_REPLACEMENT}\" test_args \"\${line}\")\n"
+    "      separate_arguments(test_args)\n"
+    "      add_test(\"\${test_name}\" \${launcher} \${emulator} \${runner} \${test_args})\n"
+    ${properties_content}
+    "    endif()\n"
+    "  endforeach()\n"
+    "endif()\n"
+  )
+
+  get_property(is_multi_config GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG)
+  if(is_multi_config)
+    file(GENERATE
+      OUTPUT ${file_base}_include-$<CONFIG>.cmake
+      CONTENT "${include_content}"
+    )
+    file(WRITE ${include_file}
+      "include(\"${file_base}_include-\${CTEST_CONFIGURATION_TYPE}.cmake\")"
+    )
+  else()
+    file(GENERATE
+      OUTPUT ${include_file}
+      CONTENT "${include_content}"
+    )
+  endif()
+
+  set_property(DIRECTORY APPEND PROPERTY TEST_INCLUDE_FILES ${include_file})
+endfunction()
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -8,8 +8,10 @@ function(add_example name)
    secp256k1
    $<$<PLATFORM_ID:Windows>:bcrypt>
  )
-  set(test_name ${name}_example)
-  add_test(NAME secp256k1_${test_name} COMMAND ${target_name})
+  add_test(NAME secp256k1.example.${name} COMMAND ${target_name})
+  set_tests_properties(secp256k1.example.${name} PROPERTIES
+    LABELS secp256k1_example
+  )
 endfunction()

 add_example(ecdsa)
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -144,15 +144,24 @@ if(SECP256K1_BUILD_TESTS)
    list(APPEND TEST_DEFINITIONS SUPPORTS_CONCURRENCY=1)
  endif()

-  add_executable(noverify_tests tests.c)
-  target_link_libraries(noverify_tests secp256k1_precomputed secp256k1_asm)
-  target_compile_definitions(noverify_tests PRIVATE ${TEST_DEFINITIONS})
-  add_test(NAME secp256k1_noverify_tests COMMAND noverify_tests)
+  function(add_executable_and_tests exe_name verify_definition)
+    add_executable(${exe_name} tests.c)
+    target_link_libraries(${exe_name} secp256k1_precomputed secp256k1_asm)
+    target_compile_definitions(${exe_name} PRIVATE ${verify_definition} ${TEST_DEFINITIONS})
+    include(DiscoverTests)
+    discover_tests(${exe_name}
+      DISCOVERY_ARGS        "--list_tests"
+      DISCOVERY_MATCH       "^\\t\\\\[ *[0-9]+\\\\] ([^ ].*)$"
+      TEST_NAME_REPLACEMENT "secp256k1.${exe_name}.\\\\1"
+      TEST_ARGS_REPLACEMENT "--target=\\\\1 --log=1"
+      PROPERTIES
+        LABELS "secp256k1_${exe_name}"
+    )
+  endfunction()
+
+  add_executable_and_tests(noverify_tests "")
  if(NOT CMAKE_BUILD_TYPE STREQUAL "Coverage")
-    add_executable(tests tests.c)
-    target_compile_definitions(tests PRIVATE VERIFY ${TEST_DEFINITIONS})
-    target_link_libraries(tests secp256k1_precomputed secp256k1_asm)
-    add_test(NAME secp256k1_tests COMMAND tests)
+    add_executable_and_tests(tests VERIFY)
  endif()
  unset(TEST_DEFINITIONS)
 endif()
@@ -162,7 +171,10 @@ if(SECP256K1_BUILD_EXHAUSTIVE_TESTS)
  add_executable(exhaustive_tests tests_exhaustive.c)
  target_link_libraries(exhaustive_tests secp256k1_asm)
  target_compile_definitions(exhaustive_tests PRIVATE $<$<NOT:$<CONFIG:Coverage>>:VERIFY>)
-  add_test(NAME secp256k1_exhaustive_tests COMMAND exhaustive_tests)
+  add_test(NAME secp256k1.exhaustive_tests COMMAND exhaustive_tests)
+  set_tests_properties(secp256k1.exhaustive_tests PROPERTIES
+    LABELS secp256k1_exhaustive
+  )
 endif()

 if(SECP256K1_BUILD_CTIME_TESTS)