project(CompilerRT C CXX ASM)
set(COMPILER_RT_STANDALONE_BUILD TRUE)
set_property(GLOBAL PROPERTY USE_FOLDERS ON)
+ if ("${CMAKE_VERSION}" VERSION_LESS "3.20.0")
+ message(WARNING
+ "Your CMake version is ${CMAKE_VERSION}. Starting with LLVM 17.0.0, the "
+ "minimum version of CMake required to build LLVM will become 3.20.0, and "
+ "using an older CMake will become an error. Please upgrade your CMake to "
+ "at least 3.20.0 now to avoid issues in the future!")
+ endif()
endif()
+set(LLVM_COMMON_CMAKE_UTILS "${CMAKE_CURRENT_SOURCE_DIR}/../cmake")
+
# Add path for custom compiler-rt modules.
list(INSERT CMAKE_MODULE_PATH 0
"${CMAKE_CURRENT_SOURCE_DIR}/cmake"
"${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules"
+ "${LLVM_COMMON_CMAKE_UTILS}"
+ "${LLVM_COMMON_CMAKE_UTILS}/Modules"
)
if(CMAKE_CONFIGURATION_TYPES)
set(CMAKE_CFG_RESOLVED_INTDIR "")
endif()
+include(SetPlatformToolchainTools)
include(base-config-ix)
include(CompilerRTUtils)
+include(CMakeDependentOption)
option(COMPILER_RT_BUILD_BUILTINS "Build builtins" ON)
mark_as_advanced(COMPILER_RT_BUILD_BUILTINS)
+option(COMPILER_RT_DISABLE_AARCH64_FMV "Disable AArch64 Function Multi Versioning support" OFF)
+mark_as_advanced(COMPILER_RT_DISABLE_AARCH64_FMV)
option(COMPILER_RT_BUILD_CRT "Build crtbegin.o/crtend.o" ON)
mark_as_advanced(COMPILER_RT_BUILD_CRT)
option(COMPILER_RT_CRT_USE_EH_FRAME_REGISTRY "Use eh_frame in crtbegin.o/crtend.o" ON)
mark_as_advanced(COMPILER_RT_BUILD_XRAY_NO_PREINIT)
option(COMPILER_RT_BUILD_ORC "Build ORC runtime" ON)
mark_as_advanced(COMPILER_RT_BUILD_ORC)
+option(COMPILER_RT_BUILD_GWP_ASAN "Build GWP-ASan, and link it into SCUDO" ON)
+mark_as_advanced(COMPILER_RT_BUILD_GWP_ASAN)
+option(COMPILER_RT_ENABLE_CET "Build Compiler RT with CET enabled" OFF)
-set(COMPILER_RT_ASAN_SHADOW_SCALE ""
- CACHE STRING "Override the shadow scale to be used in ASan runtime")
-
-if (NOT COMPILER_RT_ASAN_SHADOW_SCALE STREQUAL "")
- # Check that the shadow scale value is valid.
- if (NOT (COMPILER_RT_ASAN_SHADOW_SCALE GREATER -1 AND
- COMPILER_RT_ASAN_SHADOW_SCALE LESS 8))
- message(FATAL_ERROR "
- Invalid ASan Shadow Scale '${COMPILER_RT_ASAN_SHADOW_SCALE}'.")
- endif()
-
- set(COMPILER_RT_ASAN_SHADOW_SCALE_LLVM_FLAG
- -mllvm -asan-mapping-scale=${COMPILER_RT_ASAN_SHADOW_SCALE})
- set(COMPILER_RT_ASAN_SHADOW_SCALE_DEFINITION
- ASAN_SHADOW_SCALE=${COMPILER_RT_ASAN_SHADOW_SCALE})
- set(COMPILER_RT_ASAN_SHADOW_SCALE_FLAG
- -D${COMPILER_RT_ASAN_SHADOW_SCALE_DEFINITION})
-endif()
+option(COMPILER_RT_SCUDO_STANDALONE_SYSROOT_PATH "Set custom sysroot for building SCUDO standalone" OFF)
+mark_as_advanced(COMPILER_RT_SCUDO_STANDALONE_SYSROOT_PATH)
+option(COMPILER_RT_SCUDO_STANDALONE_BUILD_SHARED "Build SCUDO standalone for shared libraries" ON)
+mark_as_advanced(COMPILER_RT_SCUDO_STANDALONE_BUILD_SHARED)
+option(COMPILER_RT_BUILD_SCUDO_STANDALONE_WITH_LLVM_LIBC "Build SCUDO standalone with LLVM's libc headers" OFF)
+mark_as_advanced(COMPILER_RT_BUILD_SCUDO_STANDALONE_WITH_LLVM_LIBC)
if(FUCHSIA)
set(COMPILER_RT_HWASAN_WITH_INTERCEPTORS_DEFAULT OFF)
"Build for a bare-metal target.")
if (COMPILER_RT_STANDALONE_BUILD)
- load_llvm_config()
+ set(CMAKE_CXX_STANDARD 17 CACHE STRING "C++ standard to conform to")
+ set(CMAKE_CXX_STANDARD_REQUIRED YES)
+ set(CMAKE_CXX_EXTENSIONS NO)
+
+ if (NOT LLVM_RUNTIMES_BUILD)
+ load_llvm_config()
+ endif()
if (TARGET intrinsics_gen)
# Loading the llvm config causes this target to be imported so place it
# under the appropriate folder in an IDE.
CHECK_SYMBOL_EXISTS (__thumb__ "" COMPILER_RT_ARM_THUMB)
endif()
endif()
+if (${COMPILER_RT_DEFAULT_TARGET_ARCH} MATCHES "^mips")
+ CHECK_SYMBOL_EXISTS (_MIPS_ARCH_MIPS32R6 "" COMPILER_RT_MIPS32R6)
+ CHECK_SYMBOL_EXISTS (_MIPS_ARCH_MIPS64R6 "" COMPILER_RT_MIPS64R6)
+ CHECK_SYMBOL_EXISTS (__mips64 "" COMPILER_RT_MIPS_64)
+ CHECK_SYMBOL_EXISTS (__MIPSEL__ "" COMPILER_RT_MIPS_EL)
+ if ("${COMPILER_RT_MIPS_64}")
+ set(COMPILER_RT_DEFAULT_TARGET_ARCH "mips64")
+ else()
+ set(COMPILER_RT_DEFAULT_TARGET_ARCH "mips")
+ endif()
+ if ("${COMPILER_RT_MIPS_EL}")
+ set(COMPILER_RT_DEFAULT_TARGET_ARCH "${COMPILER_RT_DEFAULT_TARGET_ARCH}el")
+ endif()
+endif()
if ("${COMPILER_RT_DEFAULT_TARGET_TRIPLE}" MATCHES ".*android.*")
set(ANDROID 1)
string(REGEX MATCH "-target(=| +)[^ ]+android[a-z]*([0-9]+)" ANDROID_API_LEVEL "${CMAKE_C_FLAGS}")
pythonize_bool(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR)
-# We support running instrumented tests when we're not cross compiling
+# We support running instrumented tests when we're not cross-compiling
# and target a UNIX-like system or Windows.
# We can run tests on Android even when we are cross-compiling.
-if(("${CMAKE_HOST_SYSTEM}" STREQUAL "${CMAKE_SYSTEM}" AND (UNIX OR WIN32)) OR ANDROID
- OR COMPILER_RT_EMULATOR)
+if(("${CMAKE_HOST_SYSTEM_NAME}" STREQUAL "${CMAKE_SYSTEM_NAME}" AND (UNIX OR WIN32))
+ OR ANDROID OR COMPILER_RT_EMULATOR)
option(COMPILER_RT_CAN_EXECUTE_TESTS "Can we execute instrumented tests" ON)
else()
option(COMPILER_RT_CAN_EXECUTE_TESTS "Can we execute instrumented tests" OFF)
# This is either directly the C++ ABI library or the full C++ library
# which pulls in the ABI transitively.
+# TODO: Mark this as internal flag, most users should use COMPILER_RT_CXX_LIBRARY.
set(SANITIZER_CXX_ABI "default" CACHE STRING
"Specify C++ ABI library to use.")
set(CXXABIS none default libstdc++ libc++ libcxxabi)
handle_default_cxx_lib(SANITIZER_CXX_ABI)
# This needs to be a full C++ library for linking gtest and unit tests.
+# TODO: Mark this as internal flag, most users should use COMPILER_RT_CXX_LIBRARY.
set(SANITIZER_TEST_CXX "default" CACHE STRING
"Specify C++ library to use for tests.")
set(CXXLIBS none default libstdc++ libc++)
set_property(CACHE SANITIZER_TEST_CXX PROPERTY STRINGS ;${CXXLIBS})
handle_default_cxx_lib(SANITIZER_TEST_CXX)
+option(COMPILER_RT_USE_LLVM_UNWINDER "Use the LLVM unwinder." OFF)
+cmake_dependent_option(COMPILER_RT_ENABLE_STATIC_UNWINDER
+ "Statically link the LLVM unwinder." OFF
+ "COMPILER_RT_USE_LLVM_UNWINDER" OFF)
+
set(DEFAULT_SANITIZER_USE_STATIC_LLVM_UNWINDER OFF)
if (FUCHSIA)
set(DEFAULT_SANITIZER_USE_STATIC_LLVM_UNWINDER ON)
option(SANITIZER_USE_STATIC_LLVM_UNWINDER
"Use static LLVM unwinder." ${DEFAULT_SANITIZER_USE_STATIC_LLVM_UNWINDER})
+pythonize_bool(SANITIZER_USE_STATIC_LLVM_UNWINDER)
set(DEFAULT_SANITIZER_USE_STATIC_CXX_ABI OFF)
if (DEFINED LIBCXXABI_ENABLE_SHARED AND NOT LIBCXXABI_ENABLE_SHARED)
option(SANITIZER_USE_STATIC_CXX_ABI
"Use static libc++abi." ${DEFAULT_SANITIZER_USE_STATIC_CXX_ABI})
+pythonize_bool(SANITIZER_USE_STATIC_CXX_ABI)
+
+set(DEFAULT_SANITIZER_USE_STATIC_TEST_CXX OFF)
+if (DEFINED LIBCXX_ENABLE_SHARED AND NOT LIBCXX_ENABLE_SHARED)
+ set(DEFAULT_SANITIZER_USE_STATIC_TEST_CXX ON)
+endif()
+
+option(SANITIZER_USE_STATIC_TEST_CXX
+ "Use static libc++ for tests." ${DEFAULT_SANITIZER_USE_STATIC_TEST_CXX})
+pythonize_bool(SANITIZER_USE_STATIC_TEST_CXX)
+
+set(COMPILER_RT_SUPPORTED_CXX_LIBRARIES none default libcxx)
+set(COMPILER_RT_CXX_LIBRARY "default" CACHE STRING "Specify C++ library to use. Supported values are ${COMPILER_RT_SUPPORTED_CXX_LIBRARIES}.")
+if (NOT "${COMPILER_RT_CXX_LIBRARY}" IN_LIST COMPILER_RT_SUPPORTED_CXX_LIBRARIES)
+ message(FATAL_ERROR "Unsupported C++ library: '${COMPILER_RT_CXX_LIBRARY}'. Supported values are ${COMPILER_RT_SUPPORTED_CXX_LIBRARIES}.")
+endif()
+cmake_dependent_option(COMPILER_RT_STATIC_CXX_LIBRARY
+ "Statically link the C++ library." OFF
+ "COMPILER_RT_CXX_LIBRARY" OFF)
set(DEFAULT_COMPILER_RT_USE_BUILTINS_LIBRARY OFF)
if (FUCHSIA)
# Setup Compiler Flags
#================================
+# fcf-protection is a gcc/clang option for CET support on Linux platforms.
+# We need to handle MSVC CET option on Windows platforms.
+if (NOT MSVC)
+ if (COMPILER_RT_ENABLE_CET AND NOT COMPILER_RT_HAS_FCF_PROTECTION_FLAG)
+ message(FATAL_ERROR "Compiler used to build compiler-rt doesn't support CET!")
+ endif()
+endif()
+
if(MSVC)
# Override any existing /W flags with /W4. This is what LLVM does. Failing to
# remove other /W[0-4] flags will result in a warning about overriding a
append_string_if(COMPILER_RT_HAS_WX_FLAG /WX CMAKE_C_FLAGS CMAKE_CXX_FLAGS)
endif()
-append_string_if(COMPILER_RT_HAS_STD_CXX14_FLAG -std=c++14 CMAKE_CXX_FLAGS)
-
# Emulate C99 and C++11's __func__ for MSVC prior to 2013 CTP.
if(NOT COMPILER_RT_HAS_FUNC_SYMBOL)
add_definitions(-D__func__=__FUNCTION__)
endif()
if(LLVM_BUILD_INSTRUMENTED MATCHES IR AND COMPILER_RT_HAS_FNO_PROFILE_GENERATE_FLAG)
list(APPEND SANITIZER_COMMON_CFLAGS "-fno-profile-generate")
- elseif(LLVM_BUILD_INSTRUMENTED AND COMPILER_RT_HAS_FNO_PROFILE_INSTR_GENERATE_FLAG)
+ elseif((LLVM_BUILD_INSTRUMENTED OR LLVM_BUILD_INSTRUMENTED_COVERAGE) AND COMPILER_RT_HAS_FNO_PROFILE_INSTR_GENERATE_FLAG)
list(APPEND SANITIZER_COMMON_CFLAGS "-fno-profile-instr-generate")
+ if(LLVM_BUILD_INSTRUMENTED_COVERAGE AND COMPILER_RT_HAS_FNO_COVERAGE_MAPPING_FLAG)
+ list(APPEND SANITIZER_COMMON_CFLAGS "-fno-coverage-mapping")
+ endif()
endif()
endif()
append_list_if(COMPILER_RT_DEBUG -DSANITIZER_DEBUG=1 SANITIZER_COMMON_CFLAGS)
-if(CMAKE_CXX_COMPILER_ID MATCHES Clang)
- list(APPEND THREAD_SAFETY_FLAGS
- "-Werror=thread-safety"
- "-Werror=thread-safety-reference"
- "-Werror=thread-safety-beta"
- )
- list(APPEND SANITIZER_COMMON_CFLAGS ${THREAD_SAFETY_FLAGS})
- string(REPLACE ";" " " thread_safety_flags_space_sep "${THREAD_SAFETY_FLAGS}")
- string(APPEND COMPILER_RT_TEST_COMPILER_CFLAGS " ${thread_safety_flags_space_sep}")
-endif()
+append_list_if(COMPILER_RT_HAS_WTHREAD_SAFETY_FLAG -Wthread-safety THREAD_SAFETY_FLAGS)
+append_list_if(COMPILER_RT_HAS_WTHREAD_SAFETY_REFERENCE_FLAG -Wthread-safety-reference THREAD_SAFETY_FLAGS)
+append_list_if(COMPILER_RT_HAS_WTHREAD_SAFETY_BETA_FLAG -Wthread-safety-beta THREAD_SAFETY_FLAGS)
+list(APPEND SANITIZER_COMMON_CFLAGS ${THREAD_SAFETY_FLAGS})
+string(REPLACE ";" " " thread_safety_flags_space_sep "${THREAD_SAFETY_FLAGS}")
+string(APPEND COMPILER_RT_TEST_COMPILER_CFLAGS " ${thread_safety_flags_space_sep}")
# If we're using MSVC,
# always respect the optimization flags set by CMAKE_BUILD_TYPE instead.
# Build with optimization, unless we're in debug mode.
if(COMPILER_RT_DEBUG)
- list(APPEND SANITIZER_COMMON_CFLAGS -O0)
+ list(APPEND SANITIZER_COMMON_CFLAGS -O1)
else()
list(APPEND SANITIZER_COMMON_CFLAGS -O3)
endif()
string(REGEX REPLACE "(^| )/Z[i7I]($| )" " /Z7 "
"${var_to_update}" "${${var_to_update}}")
endforeach()
+elseif(APPLE)
+ # On Apple platforms use full debug info (i.e. not `-gline-tables-only`)
+ # for all build types so that the runtime can be debugged.
+ if(NOT COMPILER_RT_HAS_G_FLAG)
+ message(FATAL_ERROR "-g is not supported by host compiler")
+ endif()
+ list(APPEND SANITIZER_COMMON_CFLAGS -g)
elseif(COMPILER_RT_HAS_GLINE_TABLES_ONLY_FLAG AND NOT COMPILER_RT_DEBUG)
list(APPEND SANITIZER_COMMON_CFLAGS -gline-tables-only)
elseif(COMPILER_RT_HAS_G_FLAG)
append_list_if(COMPILER_RT_HAS_WGNU_FLAG -Wno-gnu SANITIZER_COMMON_CFLAGS)
append_list_if(COMPILER_RT_HAS_WVARIADIC_MACROS_FLAG -Wno-variadic-macros SANITIZER_COMMON_CFLAGS)
append_list_if(COMPILER_RT_HAS_WC99_EXTENSIONS_FLAG -Wno-c99-extensions SANITIZER_COMMON_CFLAGS)
+# format-pedantic warns about passing T* for %p, which is not useful.
append_list_if(COMPILER_RT_HAS_WD4146_FLAG /wd4146 SANITIZER_COMMON_CFLAGS)
append_list_if(COMPILER_RT_HAS_WD4291_FLAG /wd4291 SANITIZER_COMMON_CFLAGS)
append_list_if(COMPILER_RT_HAS_WD4391_FLAG /wd4391 SANITIZER_COMMON_CFLAGS)
append_list_if(MINGW -fms-extensions SANITIZER_COMMON_CFLAGS)
+# When lsan scans the stack for detecting reachable pointers, it's possible for
+# a leaked pointer, which was pushed to the stack on an earlier function call,
+# to still exist on the stack when doing a leak check if that part of the stack
+# was not overwritten. In particular, if there's any uninitialized data in the
+# lsan runtime, and the SP we start from is sufficiently deep into the runtime,
+# then a leaked pointer could be marked as reachable. Such instances could be
+# mitigated by clobbering any uninitialized data. Note that this won't cover
+# all possible uninitialized stack contents, such as those used for register
+# spill slots, unused portions for alignment, or even local variables not
+# yet in scope at a certain point in the function.
+#
+# Note that this type of issue was discovered with lsan, but can apply to other
+# sanitizers.
+append_list_if(COMPILER_RT_HAS_TRIVIAL_AUTO_INIT -ftrivial-auto-var-init=pattern SANITIZER_COMMON_CFLAGS)
+
# Set common link flags.
-append_list_if(COMPILER_RT_HAS_NODEFAULTLIBS_FLAG -nodefaultlibs SANITIZER_COMMON_LINK_FLAGS)
+# TODO: We should consider using the same model as libc++, that is use either
+# -nostdlib++ and --unwindlib=none if supported, or -nodefaultlibs otherwise.
+append_list_if(C_SUPPORTS_NODEFAULTLIBS_FLAG -nodefaultlibs SANITIZER_COMMON_LINK_FLAGS)
append_list_if(COMPILER_RT_HAS_Z_TEXT -Wl,-z,text SANITIZER_COMMON_LINK_FLAGS)
+# Only necessary for 32-bit SPARC. Solaris 11.2+ ld uses -z ignore/-z record
+# natively, but supports --as-needed/--no-as-needed for GNU ld compatibility.
+if("${COMPILER_RT_DEFAULT_TARGET_ARCH}" MATCHES "sparc")
+ list(APPEND SANITIZER_COMMON_LINK_LIBS -Wl,--as-needed atomic -Wl,--no-as-needed)
+endif()
+
if (COMPILER_RT_USE_BUILTINS_LIBRARY)
string(REPLACE "-Wl,-z,defs" "" CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS}")
else()
list(APPEND COMPILER_RT_COMMON_CFLAGS ${stdlib_flag})
list(APPEND COMPILER_RT_COMMON_LINK_FLAGS ${stdlib_flag})
-macro(append_libcxx_libs var)
- if (${var}_INTREE)
- if (SANITIZER_USE_STATIC_LLVM_UNWINDER AND (TARGET unwind_static OR HAVE_LIBUNWIND))
- list(APPEND ${var}_LIBRARIES unwind_static)
- elseif (TARGET unwind_shared OR HAVE_LIBUNWIND)
- list(APPEND ${var}_LIBRARIES unwind_shared)
- endif()
+# TODO: There's a lot of duplication across lib/*/tests/CMakeLists.txt files,
+# move some of the common flags to COMPILER_RT_UNITTEST_CFLAGS.
- if (SANITIZER_USE_STATIC_CXX_ABI AND (TARGET cxxabi_static OR HAVE_LIBCXXABI))
- list(APPEND ${var}_LIBRARIES cxxabi_static)
- elseif (TARGET cxxabi_shared OR HAVE_LIBCXXABI)
- list(APPEND ${var}_LIBRARIES cxxabi_shared)
- endif()
+# Unittests need access to C++ standard library.
+string(APPEND COMPILER_RT_TEST_COMPILER_CFLAGS " ${stdlib_flag}")
+
+# When cross-compiling, COMPILER_RT_TEST_COMPILER_CFLAGS help in compilation
+# and linking of unittests.
+string(REPLACE " " ";" COMPILER_RT_UNITTEST_CFLAGS "${COMPILER_RT_TEST_COMPILER_CFLAGS}")
+set(COMPILER_RT_UNITTEST_LINK_FLAGS ${COMPILER_RT_UNITTEST_CFLAGS})
+
+if(COMPILER_RT_USE_LLVM_UNWINDER)
+ # We're linking directly against the libunwind that we're building so don't
+ # try to link in the toolchain's default libunwind which may be missing.
+ append_list_if(CXX_SUPPORTS_UNWINDLIB_NONE_FLAG --unwindlib=none COMPILER_RT_COMMON_LINK_FLAGS)
+ append_list_if(CXX_SUPPORTS_UNWINDLIB_NONE_FLAG --unwindlib=none COMPILER_RT_UNITTEST_LINK_FLAGS)
+ if (COMPILER_RT_ENABLE_STATIC_UNWINDER)
+ list(APPEND COMPILER_RT_UNWINDER_LINK_LIBS "$<TARGET_LINKER_FILE:unwind_static>")
else()
- append_list_if(COMPILER_RT_HAS_LIBCXX c++ ${var}_LIBRARIES)
+ list(APPEND COMPILER_RT_UNWINDER_LINK_LIBS "$<TARGET_LINKER_FILE:$<IF:$<TARGET_EXISTS:unwind_shared>,unwind_shared,unwind_static>>")
endif()
-endmacro()
+endif()
+
+if (COMPILER_RT_CXX_LIBRARY STREQUAL "libcxx")
+ # We are using the in-tree libc++ so avoid including the default one.
+ append_list_if(COMPILER_RT_HAS_NOSTDINCXX_FLAG -nostdinc++ COMPILER_RT_COMMON_CFLAGS)
+ append_list_if(COMPILER_RT_HAS_NOSTDLIBXX_FLAG -nostdlib++ COMPILER_RT_COMMON_LINK_FLAGS)
+ # Use the in-tree libc++ through explicit include and library paths.
+ set(COMPILER_RT_CXX_CFLAGS "$<$<TARGET_EXISTS:cxx-headers>:$<IF:$<BOOL:${MSVC}>,/imsvc,-isystem>$<JOIN:$<TARGET_PROPERTY:cxx-headers,INTERFACE_INCLUDE_DIRECTORIES>,$<SEMICOLON>$<IF:$<BOOL:${MSVC}>,/imsvc,-isystem>>>")
+ if (COMPILER_RT_STATIC_CXX_LIBRARY)
+ set(COMPILER_RT_CXX_LINK_LIBS "$<TARGET_LINKER_FILE:cxx_static>")
+ else()
+ set(COMPILER_RT_CXX_LINK_LIBS "$<TARGET_LINKER_FILE:$<IF:$<TARGET_EXISTS:cxx_shared>,cxx_shared,cxx_static>>")
+ endif()
+elseif (COMPILER_RT_CXX_LIBRARY STREQUAL "none")
+ # We aren't using any C++ standard library so avoid including the default one.
+ append_list_if(COMPILER_RT_HAS_NOSTDINCXX_FLAG -nostdinc++ COMPILER_RT_COMMON_CFLAGS)
+ append_list_if(COMPILER_RT_HAS_NOSTDLIBXX_FLAG -nostdlib++ COMPILER_RT_COMMON_LINK_FLAGS)
+else()
+ # Nothing to be done for `default`.
+endif()
if (SANITIZER_CXX_ABI_LIBNAME STREQUAL "libc++")
- append_libcxx_libs(SANITIZER_CXX_ABI)
+ if (SANITIZER_CXX_ABI_INTREE)
+ # TODO: We don't need to add --unwindlib=none to SANITIZER_COMMON_LINK_FLAGS
+ # because we added -nodefaultlibs there earlier, and adding would result in
+ # a warning, but if we switch to -nostdlib++, we would need to add it here.
+ # append_list_if(CXX_SUPPORTS_UNWINDLIB_NONE_FLAG --unwindlib=none SANITIZER_COMMON_LINK_FLAGS)
+ if(SANITIZER_USE_STATIC_CXX_ABI)
+ if(TARGET libcxx-abi-static)
+ set(SANITIZER_CXX_ABI_LIBRARIES libcxx-abi-static)
+ endif()
+ else()
+ if(TARGET libcxx-abi-shared)
+ set(SANITIZER_CXX_ABI_LIBRARIES libcxx-abi-shared)
+ elseif(TARGET libcxx-abi-static)
+ set(SANITIZER_CXX_ABI_LIBRARIES libcxx-abi-static)
+ endif()
+ endif()
+ else()
+ append_list_if(COMPILER_RT_HAS_LIBCXX c++ SANITIZER_CXX_ABI_LIBRARIES)
+ endif()
elseif (SANITIZER_CXX_ABI_LIBNAME STREQUAL "libcxxabi")
list(APPEND SANITIZER_CXX_ABI_LIBRARIES "c++abi")
elseif (SANITIZER_CXX_ABI_LIBNAME STREQUAL "libstdc++")
endif()
if (SANITIZER_TEST_CXX_LIBNAME STREQUAL "libc++")
- append_libcxx_libs(SANITIZER_TEST_CXX)
+ if (SANITIZER_TEST_CXX_INTREE)
+ list(APPEND SANITIZER_TEST_CXX_CFLAGS "$<$<TARGET_EXISTS:cxx-headers>:$<IF:$<BOOL:${MSVC}>,/imsvc,-isystem>$<JOIN:$<TARGET_PROPERTY:cxx-headers,INTERFACE_INCLUDE_DIRECTORIES>,$<SEMICOLON>$<IF:$<BOOL:${MSVC}>,/imsvc,-isystem>>>")
+ if (SANITIZER_USE_STATIC_TEST_CXX)
+ list(APPEND SANITIZER_TEST_CXX_LIBRARIES "$<TARGET_LINKER_FILE:cxx_static>")
+ else()
+ list(APPEND SANITIZER_TEST_CXX_LIBRARIES "$<TARGET_LINKER_FILE:$<IF:$<TARGET_EXISTS:cxx_shared>,cxx_shared,cxx_static>>")
+ endif()
+ # We are using the in tree libc++ so avoid including the default one.
+ append_list_if(COMPILER_RT_HAS_NOSTDINCXX_FLAG -nostdinc++ COMPILER_RT_UNITTEST_CFLAGS)
+ append_list_if(COMPILER_RT_HAS_NOSTDLIBXX_FLAG -nostdlib++ COMPILER_RT_UNITTEST_LINK_FLAGS)
+ else()
+ append_list_if(COMPILER_RT_HAS_LIBCXX -lc++ SANITIZER_TEST_CXX_LIBRARIES)
+ endif()
elseif (SANITIZER_TEST_CXX_LIBNAME STREQUAL "libstdc++")
- append_list_if(COMPILER_RT_HAS_LIBSTDCXX stdc++ SANITIZER_TEST_CXX_LIBRARIES)
+ append_list_if(COMPILER_RT_HAS_LIBSTDCXX -lstdc++ SANITIZER_TEST_CXX_LIBRARIES)
endif()
-# TODO: There's a lot of duplication across lib/*/tests/CMakeLists.txt files,
-# move some of the common flags to COMPILER_RT_UNITTEST_CFLAGS.
-
-# Unittests need access to C++ standard library.
-string(APPEND COMPILER_RT_TEST_COMPILER_CFLAGS " ${stdlib_flag}")
-
-# When cross-compiling, COMPILER_RT_TEST_COMPILER_CFLAGS help in compilation
-# and linking of unittests.
-string(REPLACE " " ";" COMPILER_RT_UNITTEST_CFLAGS "${COMPILER_RT_TEST_COMPILER_CFLAGS}")
-set(COMPILER_RT_UNITTEST_LINK_FLAGS ${COMPILER_RT_UNITTEST_CFLAGS})
-
# Unittests support.
-set(COMPILER_RT_GTEST_PATH ${LLVM_MAIN_SRC_DIR}/utils/unittest/googletest)
+# FIXME: When compiler-rt is build using -DLLVM_BUILD_EXTERNAL_COMPILER_RT=ON, then
+# The LLVM_THIRD_PARTY_DIR variable is not set.
+if (NOT LLVM_THIRD_PARTY_DIR)
+ set(LLVM_THIRD_PARTY_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../third-party")
+endif()
+
+set(COMPILER_RT_GTEST_PATH ${LLVM_THIRD_PARTY_DIR}/unittest/googletest)
set(COMPILER_RT_GTEST_SOURCE ${COMPILER_RT_GTEST_PATH}/src/gtest-all.cc)
set(COMPILER_RT_GTEST_CFLAGS
-DGTEST_NO_LLVM_SUPPORT=1
-I${COMPILER_RT_GTEST_PATH}/include
-I${COMPILER_RT_GTEST_PATH}
)
-if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
- # FreeBSD has its pthread functions marked with thread safety annotations, but
- # googletest is not compatible with such annotations. Disable the thread
- # safety warnings-as-errors until googletest has been fixed.
- list(APPEND NO_THREAD_SAFETY_FLAGS ${THREAD_SAFETY_FLAGS})
- list(TRANSFORM NO_THREAD_SAFETY_FLAGS REPLACE "error=" "no-")
- list(APPEND COMPILER_RT_GTEST_CFLAGS ${NO_THREAD_SAFETY_FLAGS})
-endif()
# Mocking support.
-set(COMPILER_RT_GMOCK_PATH ${LLVM_MAIN_SRC_DIR}/utils/unittest/googlemock)
+set(COMPILER_RT_GMOCK_PATH ${LLVM_THIRD_PARTY_DIR}/unittest/googlemock)
set(COMPILER_RT_GMOCK_SOURCE ${COMPILER_RT_GMOCK_PATH}/src/gmock-all.cc)
set(COMPILER_RT_GMOCK_CFLAGS
-DGTEST_NO_LLVM_SUPPORT=1
-I${COMPILER_RT_GMOCK_PATH}
)
+if(COMPILER_RT_HAS_G_FLAG)
+ list(APPEND COMPILER_RT_UNITTEST_CFLAGS -g)
+endif()
append_list_if(COMPILER_RT_DEBUG -DSANITIZER_DEBUG=1 COMPILER_RT_UNITTEST_CFLAGS)
append_list_if(COMPILER_RT_HAS_WCOVERED_SWITCH_DEFAULT_FLAG -Wno-covered-switch-default COMPILER_RT_UNITTEST_CFLAGS)
append_list_if(COMPILER_RT_HAS_WSUGGEST_OVERRIDE_FLAG -Wno-suggest-override COMPILER_RT_UNITTEST_CFLAGS)
set(CMAKE_STATIC_LINKER_FLAGS "${CMAKE_STATIC_LINKER_FLAGS} /IGNORE:4221")
endif()
-if(${CMAKE_SYSTEM_NAME} MATCHES "AIX")
- set(CMAKE_C_ARCHIVE_CREATE "<CMAKE_AR> -X32_64 qc <TARGET> <LINK_FLAGS> <OBJECTS>")
- set(CMAKE_CXX_ARCHIVE_CREATE "<CMAKE_AR> -X32_64 qc <TARGET> <LINK_FLAGS> <OBJECTS>")
- set(CMAKE_C_ARCHIVE_APPEND "<CMAKE_AR> -X32_64 q <TARGET> <LINK_FLAGS> <OBJECTS>")
- set(CMAKE_CXX_ARCHIVE_APPEND "<CMAKE_AR> -X32_64 q <TARGET> <LINK_FLAGS> <OBJECTS>")
- set(CMAKE_C_ARCHIVE_FINISH "<CMAKE_RANLIB> -X32_64 <TARGET>")
- set(CMAKE_CXX_ARCHIVE_FINISH "<CMAKE_RANLIB> -X32_64 <TARGET>")
-endif()
-
add_subdirectory(include)
option(COMPILER_RT_USE_LIBCXX
set(COMPILER_RT_TEST_USE_LLD TRUE)
append_list_if(COMPILER_RT_HAS_FUSE_LD_LLD_FLAG -fuse-ld=lld SANITIZER_COMMON_LINK_FLAGS)
append_list_if(COMPILER_RT_HAS_LLD -fuse-ld=lld COMPILER_RT_UNITTEST_LINK_FLAGS)
- if(COMPILER_RT_HAS_FUSE_LD_LLD_FLAG)
- set(COMPILER_RT_HAS_GNU_VERSION_SCRIPT_COMPAT FALSE)
- endif()
endif()
pythonize_bool(COMPILER_RT_HAS_LLD)
pythonize_bool(COMPILER_RT_TEST_USE_LLD)
(W), PGP key ID and fingerprint (P), description (D), and snail-mail address
(S).
+N: Saleem Abdulrasool
+E: compnerd@compnerd.org
+D: builtins library
+
+N: Vitaly Buka
+E: vitalybuka@google.com
+D: Sanitizers
+
N: Peter Collingbourne
E: peter@pcc.me.uk
-D: DataFlowSanitizer
+D: CFI, SafeStack
-N: Daniel Dunbar
-E: daniel@zuster.org
-D: Makefile build
+N: Lang Hames
+E: lhames@gmail.com
+D: ORC
-N: Timur Iskhodzhanov
-E: timurrrr@google.com
-D: AddressSanitizer for Windows
+N: Petr Hosek
+E: phosek@google.com
+D: CRT, CMake build
-N: Howard Hinnant
-E: howard.hinnant@gmail.com
-D: builtins library
+N: Teresa Johnson
+E: tejohnson@google.com
+D: MemProf
+
+N: Mitch Phillips
+E: mitchp@google.com
+D: GWP ASAN
N: Alexander Potapenko
E: glider@google.com
-D: MacOS/iOS port of sanitizers
-
-N: Alexey Samsonov
-E: samsonov@google.com
-D: CMake build, test suite
+D: Sanitizers
N: Kostya Serebryany
E: kcc@google.com
-D: AddressSanitizer, sanitizer_common, porting sanitizers to another platforms, LeakSanitizer
+D: AddressSanitizer, sanitizer_common, LeakSanitizer, LibFuzzer
N: Richard Smith
E: richard-llvm@metafoo.co.uk
include(CompilerRTUtils)
include(HandleCompilerRT)
+# CMP0114: ExternalProject step targets fully adopt their steps.
+# New in CMake 3.19: https://cmake.org/cmake/help/latest/policy/CMP0114.html
+if(POLICY CMP0114)
+ cmake_policy(SET CMP0114 OLD)
+endif()
+
function(set_target_output_directories target output_dir)
# For RUNTIME_OUTPUT_DIRECTORY variable, Multi-configuration generators
# append a per-configuration subdirectory to the specified directory.
list(REMOVE_ITEM target_flags "-msse3")
endif()
+ # Build the macOS sanitizers with Mac Catalyst support.
+ if (APPLE AND
+ "${COMPILER_RT_ENABLE_MACCATALYST}" AND
+ "${libname}" MATCHES ".*\.osx.*")
+ foreach(arch ${LIB_ARCHS_${libname}})
+ list(APPEND target_flags
+ "SHELL:-target ${arch}-apple-macos${DARWIN_osx_MIN_VER} -darwin-target-variant ${arch}-apple-ios13.1-macabi")
+ endforeach()
+ endif()
+
set_target_compile_flags(${libname}
${extra_cflags_${libname}} ${target_flags})
set_property(TARGET ${libname} APPEND PROPERTY
if(COMPILER_RT_DEFAULT_TARGET_ONLY)
set(triple "${COMPILER_RT_DEFAULT_TARGET_TRIPLE}")
else()
- set(triple "${TARGET_TRIPLE}")
+ set(triple "${LLVM_TARGET_TRIPLE}")
endif()
- # When using arch-suffixed runtime library names, clang only looks for
- # libraries named "arm" or "armhf", see getArchNameForCompilerRTLib in
- # clang. Therefore, try to inspect both the arch name and the triple
- # if it seems like we're building an armhf target.
- if ("${arch}" MATCHES "hf$" OR "${triple}" MATCHES "hf$")
+ # Except for baremetal, when using arch-suffixed runtime library names,
+ # clang only looks for libraries named "arm" or "armhf", see
+ # getArchNameForCompilerRTLib in clang. Therefore, try to inspect both
+ # the arch name and the triple if it seems like we're building an armhf
+ # target.
+ if (COMPILER_RT_BAREMETAL_BUILD)
+ set(${output} "${name}-${arch}${COMPILER_RT_OS_SUFFIX}")
+ elseif ("${arch}" MATCHES "hf$" OR "${triple}" MATCHES "hf$")
set(${output} "${name}-armhf${COMPILER_RT_OS_SUFFIX}")
else()
set(${output} "${name}-arm${COMPILER_RT_OS_SUFFIX}")
endif()
if(LLVM_BUILD_INSTRUMENTED MATCHES IR AND COMPILER_RT_HAS_FNO_PROFILE_GENERATE_FLAG)
list(APPEND NO_PGO_FLAGS "-fno-profile-generate")
- elseif(LLVM_BUILD_INSTRUMENTED AND COMPILER_RT_HAS_FNO_PROFILE_INSTR_GENERATE_FLAG)
+ elseif((LLVM_BUILD_INSTRUMENTED OR LLVM_BUILD_INSTRUMENTED_COVERAGE) AND COMPILER_RT_HAS_FNO_PROFILE_INSTR_GENERATE_FLAG)
list(APPEND NO_PGO_FLAGS "-fno-profile-instr-generate")
+ if(LLVM_BUILD_INSTRUMENTED_COVERAGE AND COMPILER_RT_HAS_FNO_COVERAGE_MAPPING_FLAG)
+ list(APPEND NO_PGO_FLAGS "-fno-coverage-mapping")
+ endif()
endif()
endif()
get_compiler_rt_output_dir(${COMPILER_RT_DEFAULT_TARGET_ARCH} output_dir_${libname})
get_compiler_rt_install_dir(${COMPILER_RT_DEFAULT_TARGET_ARCH} install_dir_${libname})
endif()
+
+ # Build the macOS sanitizers with Mac Catalyst support.
+ if ("${COMPILER_RT_ENABLE_MACCATALYST}" AND
+ "${os}" MATCHES "^(osx)$")
+ foreach(arch ${LIB_ARCHS_${libname}})
+ list(APPEND extra_cflags_${libname}
+ "SHELL:-target ${arch}-apple-macos${DARWIN_osx_MIN_VER} -darwin-target-variant ${arch}-apple-ios13.1-macabi")
+ list(APPEND extra_link_flags_${libname}
+ "SHELL:-target ${arch}-apple-macos${DARWIN_osx_MIN_VER} -darwin-target-variant ${arch}-apple-ios13.1-macabi")
+ endforeach()
+ endif()
endforeach()
else()
foreach(arch ${LIB_ARCHS})
if(COMPILER_RT_USE_BUILTINS_LIBRARY AND NOT type STREQUAL "OBJECT" AND
NOT name STREQUAL "clang_rt.builtins")
get_compiler_rt_target(${arch} target)
- find_compiler_rt_library(builtins ${target} builtins_${libname})
+ find_compiler_rt_library(builtins builtins_${libname} TARGET ${target})
if(builtins_${libname} STREQUAL "NOTFOUND")
message(FATAL_ERROR "Cannot find builtins library for the target architecture")
endif()
target_link_libraries(${libname} PRIVATE ${builtins_${libname}})
endif()
if(${type} STREQUAL "SHARED")
- if(COMMAND llvm_setup_rpath)
- llvm_setup_rpath(${libname})
+ if(APPLE OR WIN32)
+ set_property(TARGET ${libname} PROPERTY BUILD_WITH_INSTALL_RPATH ON)
endif()
if(WIN32 AND NOT CYGWIN AND NOT MINGW)
set_target_properties(${libname} PROPERTIES IMPORT_PREFIX "")
set_target_properties(${libname} PROPERTIES IMPORT_SUFFIX ".lib")
endif()
- if(APPLE)
- # Ad-hoc sign the dylibs
- add_custom_command(TARGET ${libname}
- POST_BUILD
- COMMAND codesign --sign - $<TARGET_FILE:${libname}>
- WORKING_DIRECTORY ${COMPILER_RT_OUTPUT_LIBRARY_DIR}
+ if (APPLE AND NOT CMAKE_LINKER MATCHES ".*lld.*")
+ # Ad-hoc sign the dylibs when using Xcode versions older than 12.
+ # Xcode 12 shipped with ld64-609.
+ # FIXME: Remove whole conditional block once everything uses Xcode 12+.
+ set(LD_V_OUTPUT)
+ execute_process(
+ COMMAND sh -c "${CMAKE_LINKER} -v 2>&1 | head -1"
+ RESULT_VARIABLE HAD_ERROR
+ OUTPUT_VARIABLE LD_V_OUTPUT
)
+ if (HAD_ERROR)
+ message(FATAL_ERROR "${CMAKE_LINKER} failed with status ${HAD_ERROR}")
+ endif()
+ set(NEED_EXPLICIT_ADHOC_CODESIGN 1)
+ if ("${LD_V_OUTPUT}" MATCHES ".*ld64-([0-9.]+).*")
+ string(REGEX REPLACE ".*ld64-([0-9.]+).*" "\\1" HOST_LINK_VERSION ${LD_V_OUTPUT})
+ if (HOST_LINK_VERSION VERSION_GREATER_EQUAL 609)
+ set(NEED_EXPLICIT_ADHOC_CODESIGN 0)
+ endif()
+ endif()
+ if (NEED_EXPLICIT_ADHOC_CODESIGN)
+ add_custom_command(TARGET ${libname}
+ POST_BUILD
+ COMMAND codesign --sign - $<TARGET_FILE:${libname}>
+ WORKING_DIRECTORY ${COMPILER_RT_OUTPUT_LIBRARY_DIR}
+ )
+ endif()
endif()
endif()
endif()
add_custom_command(
OUTPUT "${output_bin}"
- COMMAND ${COMPILER_RT_TEST_COMPILER} ${TEST_OBJECTS} -o "${output_bin}"
+ COMMAND ${COMPILER_RT_TEST_CXX_COMPILER} ${TEST_OBJECTS} -o "${output_bin}"
${TEST_LINK_FLAGS}
DEPENDS ${TEST_DEPS}
)
-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER})
endif()
- set(STAMP_DIR ${prefix}-stamps/)
- set(BINARY_DIR ${prefix}-bins/)
-
add_custom_target(${name}-clear
- COMMAND ${CMAKE_COMMAND} -E remove_directory ${BINARY_DIR}
- COMMAND ${CMAKE_COMMAND} -E remove_directory ${STAMP_DIR}
- COMMENT "Clobbering ${name} build and stamp directories"
+ COMMAND ${CMAKE_COMMAND} -E remove_directory ${prefix}
+ COMMENT "Clobbering ${name} build directories"
USES_TERMINAL
)
set_target_properties(${name}-clear PROPERTIES FOLDER "Compiler-RT Misc")
add_custom_command(
OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${name}-clobber-stamp
DEPENDS ${LIBCXX_DEPS} ${toolchain_deps}
- COMMAND ${CMAKE_COMMAND} -E touch ${BINARY_DIR}/CMakeCache.txt
- COMMAND ${CMAKE_COMMAND} -E touch ${STAMP_DIR}/${name}-mkdir
+ COMMAND ${CMAKE_COMMAND} -E touch ${prefix}/CMakeCache.txt
COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/${name}-clobber-stamp
- COMMENT "Clobbering bootstrap build and stamp directories"
+ COMMENT "Clobbering bootstrap build directories"
)
add_custom_target(${name}-clobber
CMAKE_OBJCOPY
CMAKE_OBJDUMP
CMAKE_STRIP
+ CMAKE_READELF
CMAKE_SYSROOT
LIBCXX_HAS_MUSL_LIBC
+ LIBCXX_HAS_GCC_S_LIB
+ LIBCXX_HAS_PTHREAD_LIB
+ LIBCXX_HAS_RT_LIB
+ LIBCXX_USE_COMPILER_RT
+ LIBCXXABI_HAS_PTHREAD_LIB
PYTHON_EXECUTABLE
Python3_EXECUTABLE
Python2_EXECUTABLE
ExternalProject_Add(${name}
DEPENDS ${name}-clobber ${LIBCXX_DEPS}
- PREFIX ${prefix}
- SOURCE_DIR ${COMPILER_RT_SOURCE_DIR}/cmake/Modules/CustomLibcxx
- STAMP_DIR ${STAMP_DIR}
- BINARY_DIR ${BINARY_DIR}
+ PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${name}
+ SOURCE_DIR ${LLVM_MAIN_SRC_DIR}/../runtimes
+ BINARY_DIR ${prefix}
CMAKE_ARGS ${CMAKE_PASSTHROUGH_VARIABLES}
${compiler_args}
-DCMAKE_C_FLAGS=${LIBCXX_C_FLAGS}
-DCMAKE_BUILD_TYPE=Release
-DCMAKE_TRY_COMPILE_TARGET_TYPE=STATIC_LIBRARY
-DLLVM_PATH=${LLVM_MAIN_SRC_DIR}
- -DLLVM_BINARY_DIR=${prefix}
- -DLLVM_LIBRARY_OUTPUT_INTDIR=${prefix}/lib
- -DCOMPILER_RT_LIBCXX_PATH=${COMPILER_RT_LIBCXX_PATH}
- -DCOMPILER_RT_LIBCXXABI_PATH=${COMPILER_RT_LIBCXXABI_PATH}
+ -DLLVM_ENABLE_RUNTIMES=libcxx|libcxxabi
+ -DLIBCXXABI_ENABLE_SHARED=OFF
+ -DLIBCXXABI_HERMETIC_STATIC_LIBRARY=ON
+ -DLIBCXXABI_INCLUDE_TESTS=OFF
+ -DLIBCXX_CXX_ABI=libcxxabi
+ -DLIBCXX_ENABLE_SHARED=OFF
+ -DLIBCXX_HERMETIC_STATIC_LIBRARY=ON
+ -DLIBCXX_INCLUDE_BENCHMARKS=OFF
+ -DLIBCXX_INCLUDE_TESTS=OFF
+ -DLIBCXX_ENABLE_STATIC_ABI_LIBRARY=ON
${LIBCXX_CMAKE_ARGS}
INSTALL_COMMAND ""
STEP_TARGETS configure build
USES_TERMINAL_CONFIGURE 1
USES_TERMINAL_BUILD 1
USES_TERMINAL_INSTALL 1
+ LIST_SEPARATOR |
EXCLUDE_FROM_ALL TRUE
BUILD_BYPRODUCTS "${prefix}/lib/libc++.a" "${prefix}/lib/libc++abi.a"
)
if (CMAKE_GENERATOR MATCHES "Make")
- set(run_clean "$(MAKE)" "-C" "${BINARY_DIR}" "clean")
+ set(run_clean "$(MAKE)" "-C" "${prefix}" "clean")
else()
- set(run_clean ${CMAKE_COMMAND} --build ${BINARY_DIR} --target clean
+ set(run_clean ${CMAKE_COMMAND} --build ${prefix} --target clean
--config "$<CONFIG>")
endif()
COMMENT "Cleaning ${name}..."
DEPENDEES configure
${force_deps}
- WORKING_DIRECTORY ${BINARY_DIR}
+ WORKING_DIRECTORY ${prefix}
EXCLUDE_FROM_MAIN 1
USES_TERMINAL 1
)
--- /dev/null
+set(ARM64 aarch64)
+set(ARM32 arm armhf)
+set(HEXAGON hexagon)
+set(X86 i386)
+set(X86_64 x86_64)
+set(LOONGARCH64 loongarch64)
+set(MIPS32 mips mipsel)
+set(MIPS64 mips64 mips64el)
+set(PPC32 powerpc powerpcspe)
+set(PPC64 powerpc64 powerpc64le)
+set(RISCV32 riscv32)
+set(RISCV64 riscv64)
+set(S390X s390x)
+set(SPARC sparc)
+set(SPARCV9 sparcv9)
+set(WASM32 wasm32)
+set(WASM64 wasm64)
+set(VE ve)
+
+if(APPLE)
+ set(ARM64 arm64)
+ set(ARM32 armv7 armv7s armv7k)
+ set(X86_64 x86_64 x86_64h)
+endif()
+
+set(ALL_SANITIZER_COMMON_SUPPORTED_ARCH ${X86} ${X86_64} ${PPC64} ${RISCV64}
+ ${ARM32} ${ARM64} ${MIPS32} ${MIPS64} ${S390X} ${SPARC} ${SPARCV9}
+ ${HEXAGON} ${LOONGARCH64})
+set(ALL_ASAN_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64} ${RISCV64}
+ ${MIPS32} ${MIPS64} ${PPC64} ${S390X} ${SPARC} ${SPARCV9} ${HEXAGON}
+ ${LOONGARCH64})
+set(ALL_DFSAN_SUPPORTED_ARCH ${X86_64} ${MIPS64} ${ARM64})
+
+if(ANDROID)
+ set(OS_NAME "Android")
+else()
+ set(OS_NAME "${CMAKE_SYSTEM_NAME}")
+endif()
+
+if(OS_NAME MATCHES "Linux")
+ set(ALL_FUZZER_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64} ${S390X})
+elseif (OS_NAME MATCHES "Windows")
+ set(ALL_FUZZER_SUPPORTED_ARCH ${X86} ${X86_64})
+elseif(OS_NAME MATCHES "Android")
+ set(ALL_FUZZER_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64})
+else()
+ set(ALL_FUZZER_SUPPORTED_ARCH ${X86_64} ${ARM64})
+endif()
+
+set(ALL_GWP_ASAN_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64})
+if(APPLE)
+ set(ALL_LSAN_SUPPORTED_ARCH ${X86} ${X86_64} ${MIPS64} ${ARM64})
+else()
+ set(ALL_LSAN_SUPPORTED_ARCH ${X86} ${X86_64} ${MIPS64} ${ARM64} ${ARM32}
+ ${PPC64} ${S390X} ${RISCV64} ${HEXAGON} ${LOONGARCH64})
+endif()
+set(ALL_MSAN_SUPPORTED_ARCH ${X86_64} ${MIPS64} ${ARM64} ${PPC64} ${S390X})
+set(ALL_HWASAN_SUPPORTED_ARCH ${X86_64} ${ARM64} ${RISCV64})
+set(ALL_MEMPROF_SUPPORTED_ARCH ${X86_64})
+set(ALL_PROFILE_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64} ${PPC32} ${PPC64}
+ ${MIPS32} ${MIPS64} ${S390X} ${SPARC} ${SPARCV9} ${HEXAGON}
+ ${RISCV32} ${RISCV64})
+set(ALL_TSAN_SUPPORTED_ARCH ${X86_64} ${MIPS64} ${ARM64} ${PPC64} ${S390X}
+ ${LOONGARCH64})
+set(ALL_UBSAN_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64} ${RISCV64}
+ ${MIPS32} ${MIPS64} ${PPC64} ${S390X} ${SPARC} ${SPARCV9} ${HEXAGON}
+ ${LOONGARCH64})
+set(ALL_SAFESTACK_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM64} ${MIPS32} ${MIPS64}
+ ${HEXAGON} ${LOONGARCH64})
+set(ALL_CFI_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64} ${MIPS64}
+ ${HEXAGON})
+set(ALL_SCUDO_STANDALONE_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64}
+ ${MIPS32} ${MIPS64} ${PPC64} ${HEXAGON} ${LOONGARCH64})
+if(APPLE)
+set(ALL_XRAY_SUPPORTED_ARCH ${X86_64})
+else()
+set(ALL_XRAY_SUPPORTED_ARCH ${X86_64} ${ARM32} ${ARM64} ${MIPS32} ${MIPS64}
+ powerpc64le ${HEXAGON})
+endif()
+set(ALL_SHADOWCALLSTACK_SUPPORTED_ARCH ${ARM64})
+
+if (UNIX)
+set(ALL_ORC_SUPPORTED_ARCH ${X86_64} ${ARM64} ${ARM32})
+endif()
+
+if (WIN32)
+ set(ALL_ORC_SUPPORTED_ARCH ${X86_64})
+endif()
set(TRY_COMPILE_FLAGS "${ARG_FLAGS}")
if(CMAKE_C_COMPILER_ID MATCHES Clang AND CMAKE_C_COMPILER_TARGET)
- list(APPEND TRY_COMPILE_FLAGS "-target ${CMAKE_C_COMPILER_TARGET}")
+ list(APPEND TRY_COMPILE_FLAGS "--target=${CMAKE_C_COMPILER_TARGET}")
endif()
string(REPLACE ";" " " extra_flags "${TRY_COMPILE_FLAGS}")
# Strip quotes from the compile command, as the compiler is not expecting
# quoted arguments (see discussion on D62063 for when this can come up). If
- # the quotes were there for arugments with spaces in them, the quotes were
+ # the quotes were there for arguments with spaces in them, the quotes were
# not going to help since the string gets split on spaces below.
string(REPLACE "\"" "" test_compile_command "${test_compile_command}")
--- /dev/null
+function(check_section_exists section output)
+ cmake_parse_arguments(ARG "" "" "SOURCE;FLAGS" ${ARGN})
+ if(NOT ARG_SOURCE)
+ set(ARG_SOURCE "int main(void) { return 0; }\n")
+ endif()
+
+ string(RANDOM TARGET_NAME)
+ set(TARGET_NAME "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/cmTC_${TARGET_NAME}.dir")
+ file(MAKE_DIRECTORY ${TARGET_NAME})
+
+ file(WRITE "${TARGET_NAME}/CheckSectionExists.c" "${ARG_SOURCE}\n")
+
+ string(REGEX MATCHALL "<[A-Za-z0-9_]*>" substitutions
+ ${CMAKE_C_COMPILE_OBJECT})
+
+ set(try_compile_flags "${ARG_FLAGS}")
+ if(CMAKE_C_COMPILER_ID MATCHES Clang AND CMAKE_C_COMPILER_TARGET)
+ list(APPEND try_compile_flags "-target ${CMAKE_C_COMPILER_TARGET}")
+ endif()
+ append_list_if(COMPILER_RT_HAS_FNO_LTO_FLAG -fno-lto try_compile_flags)
+ if(NOT COMPILER_RT_ENABLE_PGO)
+ if(LLVM_PROFDATA_FILE AND COMPILER_RT_HAS_FNO_PROFILE_INSTR_USE_FLAG)
+ list(APPEND try_compile_flags "-fno-profile-instr-use")
+ endif()
+ if(LLVM_BUILD_INSTRUMENTED MATCHES IR AND COMPILER_RT_HAS_FNO_PROFILE_GENERATE_FLAG)
+ list(APPEND try_compile_flags "-fno-profile-generate")
+ elseif((LLVM_BUILD_INSTRUMENTED OR LLVM_BUILD_INSTRUMENTED_COVERAGE) AND COMPILER_RT_HAS_FNO_PROFILE_INSTR_GENERATE_FLAG)
+ list(APPEND try_compile_flags "-fno-profile-instr-generate")
+ if(LLVM_BUILD_INSTRUMENTED_COVERAGE AND COMPILER_RT_HAS_FNO_COVERAGE_MAPPING_FLAG)
+ list(APPEND try_compile_flags "-fno-coverage-mapping")
+ endif()
+ endif()
+ endif()
+
+ string(REPLACE ";" " " extra_flags "${try_compile_flags}")
+
+ set(test_compile_command "${CMAKE_C_COMPILE_OBJECT}")
+ foreach(substitution ${substitutions})
+ if(substitution STREQUAL "<CMAKE_C_COMPILER>")
+ string(REPLACE "<CMAKE_C_COMPILER>" "${CMAKE_C_COMPILER} ${CMAKE_C_COMPILER_ARG1}"
+ test_compile_command ${test_compile_command})
+ elseif(substitution STREQUAL "<OBJECT>")
+ string(REPLACE "<OBJECT>" "${TARGET_NAME}/CheckSectionExists.o"
+ test_compile_command ${test_compile_command})
+ elseif(substitution STREQUAL "<SOURCE>")
+ string(REPLACE "<SOURCE>" "${TARGET_NAME}/CheckSectionExists.c"
+ test_compile_command ${test_compile_command})
+ elseif(substitution STREQUAL "<FLAGS>")
+ string(REPLACE "<FLAGS>" "${CMAKE_C_FLAGS} ${extra_flags}"
+ test_compile_command ${test_compile_command})
+ else()
+ string(REPLACE "${substitution}" "" test_compile_command
+ ${test_compile_command})
+ endif()
+ endforeach()
+
+ # Strip quotes from the compile command, as the compiler is not expecting
+ # quoted arguments (potential quotes added from D62063).
+ string(REPLACE "\"" "" test_compile_command "${test_compile_command}")
+
+ string(REPLACE " " ";" test_compile_command "${test_compile_command}")
+
+ execute_process(
+ COMMAND ${test_compile_command}
+ RESULT_VARIABLE TEST_RESULT
+ OUTPUT_VARIABLE TEST_OUTPUT
+ ERROR_VARIABLE TEST_ERROR
+ )
+
+ # Explicitly throw a fatal error message if test_compile_command fails.
+ if(TEST_RESULT)
+ message(FATAL_ERROR "${TEST_ERROR}")
+ return()
+ endif()
+
+ execute_process(
+ COMMAND ${CMAKE_OBJDUMP} -h "${TARGET_NAME}/CheckSectionExists.o"
+ RESULT_VARIABLE CHECK_RESULT
+ OUTPUT_VARIABLE CHECK_OUTPUT
+ ERROR_VARIABLE CHECK_ERROR
+ )
+ string(FIND "${CHECK_OUTPUT}" "${section}" SECTION_FOUND)
+
+ if(NOT SECTION_FOUND EQUAL -1)
+ set(${output} TRUE PARENT_SCOPE)
+ else()
+ set(${output} FALSE PARENT_SCOPE)
+ endif()
+
+ file(REMOVE_RECURSE ${TARGET_NAME})
+endfunction()
include(CompilerRTUtils)
function(get_aix_libatomic_default_link_flags link_flags export_list)
- set(linkopts
- "-Wl,-H512 -Wl,-D0 \
- -Wl,-T512 -Wl,-bhalt:4 -Wl,-bernotok \
- -Wl,-bnoentry -Wl,-bexport:${export_list} \
- -Wl,-bmodtype:SRE -Wl,-lc")
+set(linkopts
+ -Wl,-H512 -Wl,-D0
+ -Wl,-T512 -Wl,-bhalt:4 -Wl,-bernotok
+ -Wl,-bnoentry -Wl,-bexport:${export_list}
+ -Wl,-bmodtype:SRE -Wl,-lc)
# Add `-Wl,-G`. Quoted from release notes of cmake-3.16.0
# > On AIX, runtime linking is no longer enabled by default.
# See https://cmake.org/cmake/help/latest/release/3.16.html
if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.16.0")
- set(linkopts "-Wl,-G" "${linkopts}")
+ set(linkopts -Wl,-G ${linkopts})
endif()
set(${link_flags} ${linkopts} PARENT_SCOPE)
endfunction()
endif()
endfunction()
-macro(archive_aix_libatomic name)
+macro(archive_aix_libatomic name libname)
cmake_parse_arguments(LIB
""
""
"ARCHS;PARENT_TARGET"
${ARGN})
- set(shared_libraries_to_archive "")
+ set(objects_to_archive "")
foreach (arch ${LIB_ARCHS})
if(CAN_TARGET_${arch})
- set(output_dir "${CMAKE_CURRENT_BINARY_DIR}/libatomic-${arch}.dir")
+ set(output_dir "${CMAKE_CURRENT_BINARY_DIR}/${libname}-${arch}.dir")
# FIXME: Target name should be kept consistent with definition
# in AddCompilerRT.cmake added by
# add_compiler_rt_runtime(<name> SHARED ...)
COMMAND ${CMAKE_STRIP} -X32_64 -E
"${output_dir}/libatomic.so.1"
DEPENDS ${target})
- list(APPEND shared_libraries_to_archive "${output_dir}/libatomic.so.1")
+ list(APPEND objects_to_archive "${output_dir}/libatomic.so.1")
endif()
endif()
endforeach()
- if(shared_libraries_to_archive)
+ if(objects_to_archive)
set(output_dir "")
set(install_dir "")
# If LLVM defines top level library directory, we want to deliver
get_compiler_rt_output_dir(${COMPILER_RT_DEFAULT_TARGET_ARCH} output_dir)
get_compiler_rt_install_dir(${COMPILER_RT_DEFAULT_TARGET_ARCH} install_dir)
endif()
- add_custom_command(OUTPUT "${output_dir}/libatomic.a"
- COMMAND ${CMAKE_AR} -X32_64 r "${output_dir}/libatomic.a"
- ${shared_libraries_to_archive}
- DEPENDS ${shared_libraries_to_archive})
- install(FILES "${output_dir}/libatomic.a"
+ add_custom_command(OUTPUT "${output_dir}/${libname}.a"
+ COMMAND ${CMAKE_AR} -X32_64 r "${output_dir}/${libname}.a"
+ ${objects_to_archive}
+ DEPENDS ${objects_to_archive})
+ install(FILES "${output_dir}/${libname}.a"
DESTINATION ${install_dir})
- add_custom_target(aix-libatomic
- DEPENDS "${output_dir}/libatomic.a")
+ add_custom_target(aix-${libname}
+ DEPENDS "${output_dir}/${libname}.a")
+ add_dependencies(${LIB_PARENT_TARGET} aix-${libname})
endif()
- add_dependencies(${LIB_PARENT_TARGET} aix-libatomic)
endmacro()
-o "${object_file}"
${source_rpath}
MAIN_DEPENDENCY ${source}
- DEPENDS ${SOURCE_DEPS})
+ DEPENDS ${SOURCE_DEPS}
+ COMMAND_EXPAND_LISTS)
endfunction()
# On Darwin, there are no system-wide C++ headers and the just-built clang is
if(NOT TEST_COMPILE_ONLY)
message(STATUS "Finding valid architectures for ${os}...")
set(SIMPLE_C ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/src.c)
- file(WRITE ${SIMPLE_C} "#include <stdio.h>\nint main() { printf(__FILE__); return 0; }\n")
+ file(WRITE ${SIMPLE_C} "#include <stdio.h>\nint main(void) { printf(__FILE__); return 0; }\n")
set(os_linker_flags)
foreach(flag ${DARWIN_${os}_LINK_FLAGS})
list(REMOVE_ITEM archs "x86_64h")
endif()
+ if(${os} MATCHES "iossim")
+ message(STATUS "Disabling i386 slice for iossim")
+ list(REMOVE_ITEM archs "i386")
+ endif()
+
set(working_archs)
foreach(arch ${archs})
if(ARM_HOST)
list(REMOVE_ITEM tmp_var i386)
+ list(REMOVE_ITEM tmp_var x86_64)
+ list(REMOVE_ITEM tmp_var x86_64h)
else()
list(REMOVE_ITEM tmp_var arm64)
list(REMOVE_ITEM tmp_var arm64e)
-target "${LIB_ARCH}-apple-${base_os}${DARWIN_${LIBOS}_BUILTIN_MIN_VER}-simulator")
endif()
+ if ("${COMPILER_RT_ENABLE_MACCATALYST}" AND
+ "${LIB_OS}" MATCHES "^osx$")
+ # Build the macOS builtins with Mac Catalyst support.
+ list(APPEND builtin_cflags
+ "SHELL:-target ${LIB_ARCH}-apple-macos${DARWIN_osx_BUILTIN_MIN_VER} -darwin-target-variant ${LIB_ARCH}-apple-ios13.1-macabi")
+ endif()
+
set_target_compile_flags(${libname}
${sysroot_flag}
${DARWIN_${LIB_OS}_BUILTIN_MIN_VER_FLAG}
macro(darwin_add_builtin_libraries)
set(DARWIN_EXCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/Darwin-excludes)
- set(CFLAGS "-fPIC -O3 -fvisibility=hidden -DVISIBILITY_HIDDEN -Wall -fomit-frame-pointer")
+ set(CFLAGS -fPIC -O3 -fvisibility=hidden -DVISIBILITY_HIDDEN -Wall -fomit-frame-pointer)
set(CMAKE_C_FLAGS "")
set(CMAKE_CXX_FLAGS "")
set(CMAKE_ASM_FLAGS "")
- append_string_if(COMPILER_RT_HAS_ASM_LSE " -DHAS_ASM_LSE" CFLAGS)
+ append_list_if(COMPILER_RT_HAS_ASM_LSE -DHAS_ASM_LSE CFLAGS)
set(PROFILE_SOURCES ../profile/InstrProfiling.c
../profile/InstrProfilingBuffer.c
../profile/InstrProfilingInternal.c
../profile/InstrProfilingVersionVar.c)
foreach (os ${ARGN})
+ set(macosx_sdk_version 99999)
+ if ("${os}" STREQUAL "osx")
+ find_darwin_sdk_version(macosx_sdk_version "macosx")
+ endif()
+ add_security_warnings(CFLAGS ${macosx_sdk_version})
+
list_intersect(DARWIN_BUILTIN_ARCHS DARWIN_${os}_BUILTIN_ARCHS BUILTIN_SUPPORTED_ARCH)
if((arm64 IN_LIST DARWIN_BUILTIN_ARCHS OR arm64e IN_LIST DARWIN_BUILTIN_ARCHS) AND NOT TARGET lse_builtin_symlinks)
set(MACHO_SYM_DIR ${CMAKE_CURRENT_SOURCE_DIR}/macho_embedded)
- set(CFLAGS "-Oz -Wall -fomit-frame-pointer -ffreestanding")
+ set(CFLAGS -Oz -Wall -fomit-frame-pointer -ffreestanding)
set(CMAKE_C_FLAGS "")
set(CMAKE_CXX_FLAGS "")
set(CMAKE_ASM_FLAGS "")
set(DARWIN_macho_embedded_LIBRARY_INSTALL_DIR
${COMPILER_RT_INSTALL_LIBRARY_DIR}/macho_embedded)
- set(CFLAGS_armv7 "-target thumbv7-apple-darwin-eabi")
- set(CFLAGS_i386 "-march=pentium")
+ set(CFLAGS_armv7 -target thumbv7-apple-darwin-eabi)
+ set(CFLAGS_i386 -march=pentium)
darwin_read_list_from_file(common_FUNCTIONS ${MACHO_SYM_DIR}/common.txt)
darwin_read_list_from_file(thumb2_FUNCTIONS ${MACHO_SYM_DIR}/thumb2.txt)
endmacro()
macro(compiler_rt_mock_llvm_cmake_config_set_cmake_path)
- # Point `LLVM_CMAKE_PATH` at the source tree in the monorepo.
- set(LLVM_CMAKE_PATH "${LLVM_MAIN_SRC_DIR}/cmake/modules")
- if (NOT EXISTS "${LLVM_CMAKE_PATH}")
- message(FATAL_ERROR "LLVM_CMAKE_PATH (${LLVM_CMAKE_PATH}) does not exist")
+ # Point `LLVM_CMAKE_DIR` at the source tree in the monorepo.
+ set(LLVM_CMAKE_DIR "${LLVM_MAIN_SRC_DIR}/cmake/modules")
+ if (NOT EXISTS "${LLVM_CMAKE_DIR}")
+ message(FATAL_ERROR "LLVM_CMAKE_DIR (${LLVM_CMAKE_DIR}) does not exist")
endif()
- list(APPEND CMAKE_MODULE_PATH "${LLVM_CMAKE_PATH}")
- message(STATUS "LLVM_CMAKE_PATH: \"${LLVM_CMAKE_PATH}\"")
+ list(APPEND CMAKE_MODULE_PATH "${LLVM_CMAKE_DIR}")
+ message(STATUS "LLVM_CMAKE_DIR: \"${LLVM_CMAKE_DIR}\"")
endmacro()
function(compiler_rt_mock_llvm_cmake_config_set_target_triple)
- # Various bits of compiler-rt depend on the `TARGET_TRIPLE`variable being
- # defined. This function tries to set a sensible value for the variable.
- # This is a function rather than a macro to avoid polluting the variable
- # namespace.
+ # Various bits of compiler-rt depend on the `LLVM_TARGET_TRIPLE` variable
+ # being defined. This function tries to set a sensible value for the
+ # variable. This is a function rather than a macro to avoid polluting the
+ # variable namespace.
set(COMPILER_OUTPUT "")
# If the user provides `COMPILER_RT_DEFAULT_TARGET_ONLY` and `CMAKE_C_COMPILER_TARGET`
if (COMPILER_RT_DEFAULT_TARGET_ONLY)
if (NOT "${CMAKE_C_COMPILER_TARGET}" STREQUAL "")
message(STATUS
- "Using CMAKE_C_COMPILER_TARGET (${CMAKE_C_COMPILER_TARGET}) as TARGET_TRIPLE")
+ "Using CMAKE_C_COMPILER_TARGET (${CMAKE_C_COMPILER_TARGET}) as LLVM_TARGET_TRIPLE")
endif()
set(COMPILER_OUTPUT "${CMAKE_C_COMPILER_TARGET}")
endif()
if (HAD_ERROR)
message(FATAL_ERROR "Fetching target triple from compiler failed")
endif()
- set(TARGET_TRIPLE "${COMPILER_OUTPUT}")
- message(STATUS "TARGET_TRIPLE: \"${TARGET_TRIPLE}\"")
- if ("${TARGET_TRIPLE}" STREQUAL "")
+ set(LLVM_TARGET_TRIPLE "${COMPILER_OUTPUT}")
+ message(STATUS "TARGET_TRIPLE: \"${LLVM_TARGET_TRIPLE}\"")
+ if ("${LLVM_TARGET_TRIPLE}" STREQUAL "")
message(FATAL_ERROR "TARGET_TRIPLE cannot be empty")
endif()
- set(TARGET_TRIPLE "${TARGET_TRIPLE}" PARENT_SCOPE)
+ set(LLVM_TARGET_TRIPLE "${LLVM_TARGET_TRIPLE}" PARENT_SCOPE)
endfunction()
macro(compiler_rt_mock_llvm_cmake_config_include_cmake_files)
# Some compiler-rt CMake code needs to call code in this file.
- include("${LLVM_CMAKE_PATH}/AddLLVM.cmake")
+ include("${LLVM_CMAKE_DIR}/AddLLVM.cmake")
endmacro()
# define a handy helper function for it. The compile flags setting in CMake
# has serious issues that make its syntax challenging at best.
function(set_target_compile_flags target)
- set(argstring "")
- foreach(arg ${ARGN})
- set(argstring "${argstring} ${arg}")
- endforeach()
- set_property(TARGET ${target} PROPERTY COMPILE_FLAGS "${argstring}")
+ set_property(TARGET ${target} PROPERTY COMPILE_OPTIONS ${ARGN})
endfunction()
function(set_target_link_flags target)
- set(argstring "")
- foreach(arg ${ARGN})
- set(argstring "${argstring} ${arg}")
- endforeach()
- set_property(TARGET ${target} PROPERTY LINK_FLAGS "${argstring}")
+ set_property(TARGET ${target} PROPERTY LINK_OPTIONS ${ARGN})
endfunction()
# Set the variable var_PYBOOL to True if var holds a true-ish string,
if(NOT HAS_${arch}_DEF)
set(CAN_TARGET_${arch} FALSE)
elseif(TEST_COMPILE_ONLY)
- try_compile_only(CAN_TARGET_${arch} FLAGS ${TARGET_${arch}_CFLAGS})
+ try_compile_only(CAN_TARGET_${arch}
+ SOURCE "#include <limits.h>\nint foo(int x, int y) { return x + y; }\n"
+ FLAGS ${TARGET_${arch}_CFLAGS})
else()
set(FLAG_NO_EXCEPTIONS "")
if(COMPILER_RT_HAS_FNO_EXCEPTIONS_FLAG)
macro(detect_target_arch)
check_symbol_exists(__arm__ "" __ARM)
+ check_symbol_exists(__AVR__ "" __AVR)
check_symbol_exists(__aarch64__ "" __AARCH64)
check_symbol_exists(__x86_64__ "" __X86_64)
check_symbol_exists(__i386__ "" __I386)
+ check_symbol_exists(__loongarch__ "" __LOONGARCH)
check_symbol_exists(__mips__ "" __MIPS)
check_symbol_exists(__mips64__ "" __MIPS64)
check_symbol_exists(__powerpc__ "" __PPC)
check_symbol_exists(__ve__ "" __VE)
if(__ARM)
add_default_target_arch(arm)
+ elseif(__AVR)
+ add_default_target_arch(avr)
elseif(__AARCH64)
add_default_target_arch(aarch64)
elseif(__X86_64)
endif()
elseif(__I386)
add_default_target_arch(i386)
+ elseif(__LOONGARCH)
+ if(CMAKE_SIZEOF_VOID_P EQUAL "4")
+ add_default_target_arch(loongarch32)
+ elseif(CMAKE_SIZEOF_VOID_P EQUAL "8")
+ add_default_target_arch(loongarch64)
+ else()
+ message(FATAL_ERROR "Unsupported pointer size for LoongArch")
+ endif()
elseif(__MIPS64) # must be checked before __MIPS
add_default_target_arch(mips64)
elseif(__MIPS)
# Compiler-RT Builtins standalone build.
# `llvm-project/compiler-rt/lib/builtins`
set(PATH_TO_COMPILER_RT_SOURCE_ROOT "${CompilerRTBuiltins_SOURCE_DIR}/../../")
+ elseif (DEFINED CompilerRTCRT_SOURCE_DIR)
+ # Compiler-RT CRT standalone build.
+ # `llvm-project/compiler-rt/lib/crt`
+ set(PATH_TO_COMPILER_RT_SOURCE_ROOT "${CompilerRTCRT_SOURCE_DIR}/../../")
elseif(DEFINED CompilerRT_SOURCE_DIR)
# Compiler-RT standalone build.
# `llvm-project/compiler-rt`
endfunction()
macro(load_llvm_config)
- if (NOT LLVM_CONFIG_PATH)
- find_program(LLVM_CONFIG_PATH "llvm-config"
- DOC "Path to llvm-config binary")
- if (NOT LLVM_CONFIG_PATH)
- message(WARNING "UNSUPPORTED COMPILER-RT CONFIGURATION DETECTED: "
- "llvm-config not found.\n"
- "Reconfigure with -DLLVM_CONFIG_PATH=path/to/llvm-config.")
- endif()
+ if (LLVM_CONFIG_PATH AND NOT LLVM_CMAKE_DIR)
+ message(WARNING
+ "LLVM_CONFIG_PATH is deprecated, please use LLVM_CMAKE_DIR instead")
+ # Compute the path to the LLVM install prefix and pass it as LLVM_CMAKE_DIR,
+ # CMake will locate the appropriate lib*/cmake subdirectory from there.
+ # For example. for -DLLVM_CONFIG_PATH=/usr/lib/llvm/16/bin/llvm-config
+ # this will yield LLVM_CMAKE_DIR=/usr/lib/llvm/16.
+ get_filename_component(LLVM_CMAKE_DIR "${LLVM_CONFIG_PATH}" DIRECTORY)
+ get_filename_component(LLVM_CMAKE_DIR "${LLVM_CMAKE_DIR}" DIRECTORY)
endif()
# Compute path to LLVM sources assuming the monorepo layout.
"You are not using the monorepo layout. This configuration is DEPRECATED.")
endif()
- set(FOUND_LLVM_CMAKE_PATH FALSE)
- if (LLVM_CONFIG_PATH)
- execute_process(
- COMMAND ${LLVM_CONFIG_PATH} "--obj-root" "--bindir" "--libdir" "--src-root" "--includedir"
- RESULT_VARIABLE HAD_ERROR
- OUTPUT_VARIABLE CONFIG_OUTPUT)
- if (HAD_ERROR)
- message(FATAL_ERROR "llvm-config failed with status ${HAD_ERROR}")
- endif()
- string(REGEX REPLACE "[ \t]*[\r\n]+[ \t]*" ";" CONFIG_OUTPUT ${CONFIG_OUTPUT})
- list(GET CONFIG_OUTPUT 0 BINARY_DIR)
- list(GET CONFIG_OUTPUT 1 TOOLS_BINARY_DIR)
- list(GET CONFIG_OUTPUT 2 LIBRARY_DIR)
- list(GET CONFIG_OUTPUT 3 MAIN_SRC_DIR)
- list(GET CONFIG_OUTPUT 4 INCLUDE_DIR)
-
- set(LLVM_BINARY_DIR ${BINARY_DIR} CACHE PATH "Path to LLVM build tree")
- set(LLVM_LIBRARY_DIR ${LIBRARY_DIR} CACHE PATH "Path to llvm/lib")
- set(LLVM_TOOLS_BINARY_DIR ${TOOLS_BINARY_DIR} CACHE PATH "Path to llvm/bin")
- set(LLVM_INCLUDE_DIR ${INCLUDE_DIR} CACHE PATH "Paths to LLVM headers")
-
- if (NOT EXISTS "${LLVM_MAIN_SRC_DIR_DEFAULT}")
- # TODO(dliew): Remove this legacy fallback path.
- message(WARNING
- "Consulting llvm-config for the LLVM source path "
- "as a fallback. This behavior will be removed in the future.")
- # We don't set `LLVM_MAIN_SRC_DIR` directly to avoid overriding a user
- # provided CMake cache value.
- set(LLVM_MAIN_SRC_DIR_DEFAULT "${MAIN_SRC_DIR}")
- message(STATUS "Using LLVM source path (${LLVM_MAIN_SRC_DIR_DEFAULT}) from llvm-config")
- endif()
-
- # Detect if we have the LLVMXRay and TestingSupport library installed and
- # available from llvm-config.
- execute_process(
- COMMAND ${LLVM_CONFIG_PATH} "--ldflags" "--libs" "xray"
- RESULT_VARIABLE HAD_ERROR
- OUTPUT_VARIABLE CONFIG_OUTPUT
- ERROR_QUIET)
- if (HAD_ERROR)
- message(WARNING "llvm-config finding xray failed with status ${HAD_ERROR}")
+ find_package(LLVM HINTS "${LLVM_CMAKE_DIR}")
+ if (NOT LLVM_FOUND)
+ message(WARNING "UNSUPPORTED COMPILER-RT CONFIGURATION DETECTED: "
+ "LLVM cmake package not found.\n"
+ "Reconfigure with -DLLVM_CMAKE_DIR=/path/to/llvm.")
+ else()
+ list(APPEND CMAKE_MODULE_PATH "${LLVM_DIR}")
+ # Turn into CACHE PATHs for overwritting
+ set(LLVM_BINARY_DIR "${LLVM_BINARY_DIR}" CACHE PATH "Path to LLVM build tree")
+ set(LLVM_LIBRARY_DIR "${LLVM_LIBRARY_DIR}" CACHE PATH "Path to llvm/lib")
+ set(LLVM_TOOLS_BINARY_DIR "${LLVM_TOOLS_BINARY_DIR}" CACHE PATH "Path to llvm/bin")
+ set(LLVM_INCLUDE_DIR ${LLVM_INCLUDE_DIRS} CACHE PATH "Path to llvm/include and any other header dirs needed")
+
+ list(FIND LLVM_AVAILABLE_LIBS LLVMXRay XRAY_INDEX)
+ set(COMPILER_RT_HAS_LLVMXRAY TRUE)
+ if (XRAY_INDEX EQUAL -1)
+ message(WARNING "LLVMXRay not found in LLVM_AVAILABLE_LIBS")
set(COMPILER_RT_HAS_LLVMXRAY FALSE)
- else()
- string(REGEX REPLACE "[ \t]*[\r\n]+[ \t]*" ";" CONFIG_OUTPUT ${CONFIG_OUTPUT})
- list(GET CONFIG_OUTPUT 0 LDFLAGS)
- list(GET CONFIG_OUTPUT 1 LIBLIST)
- file(TO_CMAKE_PATH "${LDFLAGS}" LDFLAGS)
- file(TO_CMAKE_PATH "${LIBLIST}" LIBLIST)
- set(LLVM_XRAY_LDFLAGS ${LDFLAGS} CACHE STRING "Linker flags for LLVMXRay library")
- set(LLVM_XRAY_LIBLIST ${LIBLIST} CACHE STRING "Library list for LLVMXRay")
- set(COMPILER_RT_HAS_LLVMXRAY TRUE)
endif()
- set(COMPILER_RT_HAS_LLVMTESTINGSUPPORT FALSE)
- execute_process(
- COMMAND ${LLVM_CONFIG_PATH} "--ldflags" "--libs" "testingsupport"
- RESULT_VARIABLE HAD_ERROR
- OUTPUT_VARIABLE CONFIG_OUTPUT
- ERROR_QUIET)
- if (HAD_ERROR)
- message(WARNING "llvm-config finding testingsupport failed with status ${HAD_ERROR}")
- elseif(COMPILER_RT_INCLUDE_TESTS)
- string(REGEX REPLACE "[ \t]*[\r\n]+[ \t]*" ";" CONFIG_OUTPUT ${CONFIG_OUTPUT})
- list(GET CONFIG_OUTPUT 0 LDFLAGS)
- list(GET CONFIG_OUTPUT 1 LIBLIST)
- if (LIBLIST STREQUAL "")
- message(WARNING "testingsupport library not installed, some tests will be skipped")
- else()
- file(TO_CMAKE_PATH "${LDFLAGS}" LDFLAGS)
- file(TO_CMAKE_PATH "${LIBLIST}" LIBLIST)
- set(LLVM_TESTINGSUPPORT_LDFLAGS ${LDFLAGS} CACHE STRING "Linker flags for LLVMTestingSupport library")
- set(LLVM_TESTINGSUPPORT_LIBLIST ${LIBLIST} CACHE STRING "Library list for LLVMTestingSupport")
- set(COMPILER_RT_HAS_LLVMTESTINGSUPPORT TRUE)
- endif()
- endif()
-
- # Make use of LLVM CMake modules.
- # --cmakedir is supported since llvm r291218 (4.0 release)
- execute_process(
- COMMAND ${LLVM_CONFIG_PATH} --cmakedir
- RESULT_VARIABLE HAD_ERROR
- OUTPUT_VARIABLE CONFIG_OUTPUT)
- if(NOT HAD_ERROR)
- string(STRIP "${CONFIG_OUTPUT}" LLVM_CMAKE_PATH_FROM_LLVM_CONFIG)
- file(TO_CMAKE_PATH ${LLVM_CMAKE_PATH_FROM_LLVM_CONFIG} LLVM_CMAKE_PATH)
- else()
- file(TO_CMAKE_PATH ${LLVM_BINARY_DIR} LLVM_BINARY_DIR_CMAKE_STYLE)
- set(LLVM_CMAKE_PATH "${LLVM_BINARY_DIR_CMAKE_STYLE}/lib${LLVM_LIBDIR_SUFFIX}/cmake/llvm")
- endif()
-
- set(LLVM_CMAKE_INCLUDE_FILE "${LLVM_CMAKE_PATH}/LLVMConfig.cmake")
- if (EXISTS "${LLVM_CMAKE_INCLUDE_FILE}")
- list(APPEND CMAKE_MODULE_PATH "${LLVM_CMAKE_PATH}")
- # Get some LLVM variables from LLVMConfig.
- include("${LLVM_CMAKE_INCLUDE_FILE}")
- set(FOUND_LLVM_CMAKE_PATH TRUE)
- else()
- set(FOUND_LLVM_CMAKE_PATH FALSE)
- message(WARNING "LLVM CMake path (${LLVM_CMAKE_INCLUDE_FILE}) reported by llvm-config does not exist")
+ list(FIND LLVM_AVAILABLE_LIBS LLVMTestingSupport TESTINGSUPPORT_INDEX)
+ set(COMPILER_RT_HAS_LLVMTESTINGSUPPORT TRUE)
+ if (TESTINGSUPPORT_INDEX EQUAL -1)
+ message(WARNING "LLVMTestingSupport not found in LLVM_AVAILABLE_LIBS")
+ set(COMPILER_RT_HAS_LLVMTESTINGSUPPORT FALSE)
endif()
- unset(LLVM_CMAKE_INCLUDE_FILE)
-
- set(LLVM_LIBRARY_OUTPUT_INTDIR
- ${LLVM_BINARY_DIR}/${CMAKE_CFG_INTDIR}/lib${LLVM_LIBDIR_SUFFIX})
endif()
- # Finally set the cache variable now that `llvm-config` has also had a chance
- # to set `LLVM_MAIN_SRC_DIR_DEFAULT`.
+ set(LLVM_LIBRARY_OUTPUT_INTDIR
+ ${LLVM_BINARY_DIR}/${CMAKE_CFG_INTDIR}/lib${LLVM_LIBDIR_SUFFIX})
+
set(LLVM_MAIN_SRC_DIR "${LLVM_MAIN_SRC_DIR_DEFAULT}" CACHE PATH "Path to LLVM source tree")
message(STATUS "LLVM_MAIN_SRC_DIR: \"${LLVM_MAIN_SRC_DIR}\"")
if (NOT EXISTS "${LLVM_MAIN_SRC_DIR}")
"This will be treated as error in the future.")
endif()
- if (NOT FOUND_LLVM_CMAKE_PATH)
+ if (NOT LLVM_FOUND)
# This configuration tries to configure without the prescence of `LLVMConfig.cmake`. It is
# intended for testing purposes (generating the lit test suites) and will likely not support
# a build of the runtimes in compiler-rt.
endif()
set(COMPILER_RT_DEFAULT_TARGET_TRIPLE ${CMAKE_C_COMPILER_TARGET})
else()
- set(COMPILER_RT_DEFAULT_TARGET_TRIPLE ${TARGET_TRIPLE} CACHE STRING
+ set(COMPILER_RT_DEFAULT_TARGET_TRIPLE ${LLVM_TARGET_TRIPLE} CACHE STRING
"Default triple for which compiler-rt runtimes will be built.")
endif()
- if(DEFINED COMPILER_RT_TEST_TARGET_TRIPLE)
- # Backwards compatibility: this variable used to be called
- # COMPILER_RT_TEST_TARGET_TRIPLE.
- set(COMPILER_RT_DEFAULT_TARGET_TRIPLE ${COMPILER_RT_TEST_TARGET_TRIPLE})
- endif()
-
- string(REPLACE "-" ";" TARGET_TRIPLE_LIST ${COMPILER_RT_DEFAULT_TARGET_TRIPLE})
- list(GET TARGET_TRIPLE_LIST 0 COMPILER_RT_DEFAULT_TARGET_ARCH)
+ string(REPLACE "-" ";" LLVM_TARGET_TRIPLE_LIST ${COMPILER_RT_DEFAULT_TARGET_TRIPLE})
+ list(GET LLVM_TARGET_TRIPLE_LIST 0 COMPILER_RT_DEFAULT_TARGET_ARCH)
# Map various forms of the architecture names to the canonical forms
# (as they are used by clang, see getArchNameForCompilerRTLib).
# Determine if test target triple is specified explicitly, and doesn't match the
# default.
- if(NOT COMPILER_RT_DEFAULT_TARGET_TRIPLE STREQUAL TARGET_TRIPLE)
+ if(NOT COMPILER_RT_DEFAULT_TARGET_TRIPLE STREQUAL LLVM_TARGET_TRIPLE)
set(COMPILER_RT_HAS_EXPLICIT_DEFAULT_TARGET_TRIPLE TRUE)
else()
set(COMPILER_RT_HAS_EXPLICIT_DEFAULT_TARGET_TRIPLE FALSE)
function(get_compiler_rt_target arch variable)
string(FIND ${COMPILER_RT_DEFAULT_TARGET_TRIPLE} "-" dash_index)
string(SUBSTRING ${COMPILER_RT_DEFAULT_TARGET_TRIPLE} ${dash_index} -1 triple_suffix)
+ string(SUBSTRING ${COMPILER_RT_DEFAULT_TARGET_TRIPLE} 0 ${dash_index} triple_cpu)
if(COMPILER_RT_DEFAULT_TARGET_ONLY)
# Use exact spelling when building only for the target specified to CMake.
set(target "${COMPILER_RT_DEFAULT_TARGET_TRIPLE}")
elseif(ANDROID AND ${arch} STREQUAL "i386")
set(target "i686${triple_suffix}")
+ elseif(${arch} STREQUAL "amd64")
+ set(target "x86_64${triple_suffix}")
+ elseif(${arch} STREQUAL "sparc64")
+ set(target "sparcv9${triple_suffix}")
+ elseif("${arch}" MATCHES "mips64|mips64el")
+ string(REGEX REPLACE "-gnu.*" "-gnuabi64" triple_suffix_gnu "${triple_suffix}")
+ string(REGEX REPLACE "mipsisa32" "mipsisa64" triple_cpu_mips "${triple_cpu}")
+ string(REGEX REPLACE "^mips$" "mips64" triple_cpu_mips "${triple_cpu_mips}")
+ string(REGEX REPLACE "^mipsel$" "mips64el" triple_cpu_mips "${triple_cpu_mips}")
+ set(target "${triple_cpu_mips}${triple_suffix_gnu}")
+ elseif("${arch}" MATCHES "mips|mipsel")
+ string(REGEX REPLACE "-gnuabi.*" "-gnu" triple_suffix_gnu "${triple_suffix}")
+ string(REGEX REPLACE "mipsisa64" "mipsisa32" triple_cpu_mips "${triple_cpu}")
+ string(REGEX REPLACE "mips64" "mips" triple_cpu_mips "${triple_cpu_mips}")
+ set(target "${triple_cpu_mips}${triple_suffix_gnu}")
+ elseif("${arch}" MATCHES "^arm")
+ # Arch is arm, armhf, armv6m (anything else would come from using
+ # COMPILER_RT_DEFAULT_TARGET_ONLY, which is checked above).
+ if (${arch} STREQUAL "armhf")
+ # If we are building for hard float but our ABI is soft float.
+ if ("${triple_suffix}" MATCHES ".*eabi$")
+ # Change "eabi" -> "eabihf"
+ set(triple_suffix "${triple_suffix}hf")
+ endif()
+ # ABI is already set in the triple, don't repeat it in the architecture.
+ set(arch "arm")
+ else ()
+ # If we are building for soft float, but the triple's ABI is hard float.
+ if ("${triple_suffix}" MATCHES ".*eabihf$")
+ # Change "eabihf" -> "eabi"
+ string(REGEX REPLACE "hf$" "" triple_suffix "${triple_suffix}")
+ endif()
+ endif()
+ set(target "${arch}${triple_suffix}")
else()
set(target "${arch}${triple_suffix}")
endif()
endif()
endif()
endfunction()
+
+# Add warnings to catch potential errors that can lead to security
+# vulnerabilities.
+function(add_security_warnings out_flags macosx_sdk_version)
+ set(flags "${${out_flags}}")
+
+ append_list_if(COMPILER_RT_HAS_ARRAY_BOUNDS_FLAG -Werror=array-bounds flags)
+ append_list_if(COMPILER_RT_HAS_UNINITIALIZED_FLAG -Werror=uninitialized flags)
+ append_list_if(COMPILER_RT_HAS_SHADOW_FLAG -Werror=shadow flags)
+ append_list_if(COMPILER_RT_HAS_EMPTY_BODY_FLAG -Werror=empty-body flags)
+ append_list_if(COMPILER_RT_HAS_SIZEOF_POINTER_MEMACCESS_FLAG -Werror=sizeof-pointer-memaccess flags)
+ append_list_if(COMPILER_RT_HAS_SIZEOF_ARRAY_ARGUMENT_FLAG -Werror=sizeof-array-argument flags)
+ append_list_if(COMPILER_RT_HAS_SUSPICIOUS_MEMACCESS_FLAG -Werror=suspicious-memaccess flags)
+ append_list_if(COMPILER_RT_HAS_BUILTIN_MEMCPY_CHK_SIZE_FLAG -Werror=builtin-memcpy-chk-size flags)
+ append_list_if(COMPILER_RT_HAS_ARRAY_BOUNDS_POINTER_ARITHMETIC_FLAG -Werror=array-bounds-pointer-arithmetic flags)
+ append_list_if(COMPILER_RT_HAS_RETURN_STACK_ADDRESS_FLAG -Werror=return-stack-address flags)
+ append_list_if(COMPILER_RT_HAS_SIZEOF_ARRAY_DECAY_FLAG -Werror=sizeof-array-decay flags)
+ append_list_if(COMPILER_RT_HAS_FORMAT_INSUFFICIENT_ARGS_FLAG -Werror=format-insufficient-args flags)
+ append_list_if(COMPILER_RT_HAS_BUILTIN_FORMAL_SECURITY_FLAG -Werror=format-security flags)
+ append_list_if(COMPILER_RT_HAS_SIZEOF_ARRAY_DIV_FLAG -Werror=sizeof-array-div)
+ append_list_if(COMPILER_RT_HAS_SIZEOF_POINTER_DIV_FLAG -Werror=sizeof-pointer-div)
+
+ # Add -Wformat-nonliteral only if we can avoid adding the definition of
+ # eprintf. On Apple platforms, eprintf is needed only on macosx and only if
+ # its version is older than 10.7.
+ if ("${macosx_sdk_version}" VERSION_GREATER_EQUAL 10.7)
+ list(APPEND flags -Werror=format-nonliteral -DDONT_DEFINE_EPRINTF)
+ endif()
+
+ set(${out_flags} "${flags}" PARENT_SCOPE)
+endfunction()
set(SANITIZER_GEN_DYNAMIC_LIST
${COMPILER_RT_SOURCE_DIR}/lib/sanitizer_common/scripts/gen_dynamic_list.py)
-set(SANITIZER_LINT_SCRIPT
- ${COMPILER_RT_SOURCE_DIR}/lib/sanitizer_common/scripts/check_lint.sh)
-
if(CMAKE_NM)
set(SANITIZER_NM "${CMAKE_NM}")
else()
add_custom_target(${name}-version-list ALL
DEPENDS ${vers})
endmacro()
-
-# Add target to check code style for sanitizer runtimes.
-if(CMAKE_HOST_UNIX AND NOT OS_NAME MATCHES "OpenBSD")
- add_custom_target(SanitizerLintCheck
- COMMAND env LLVM_CHECKOUT=${LLVM_MAIN_SRC_DIR} SILENT=1 TMPDIR=
- PYTHON_EXECUTABLE=${Python3_EXECUTABLE}
- COMPILER_RT=${COMPILER_RT_SOURCE_DIR}
- ${SANITIZER_LINT_SCRIPT}
- DEPENDS ${SANITIZER_LINT_SCRIPT}
- COMMENT "Running lint check for sanitizer sources..."
- VERBATIM)
-else()
- add_custom_target(SanitizerLintCheck
- COMMAND echo "No lint check")
-endif()
-set_target_properties(SanitizerLintCheck
- PROPERTIES FOLDER "Compiler-RT Misc")
# .o files. This is particularly useful in producing larger, more complex
# runtime libraries.
+include(BuiltinTests)
include(CheckIncludeFile)
include(CheckCXXSourceCompiles)
+include(GNUInstallDirs)
+include(ExtendPath)
+include(CompilerRTDarwinUtils)
check_include_file(unwind.h HAVE_UNWIND_H)
if (LLVM_TREE_AVAILABLE)
# Compute the Clang version from the LLVM version.
- # FIXME: We should be able to reuse CLANG_VERSION variable calculated
+ # FIXME: We should be able to reuse CLANG_VERSION_MAJOR variable calculated
# in Clang cmake files, instead of copying the rules here.
- string(REGEX MATCH "[0-9]+\\.[0-9]+(\\.[0-9]+)?" CLANG_VERSION
+ string(REGEX MATCH "^[0-9]+" CLANG_VERSION_MAJOR
${PACKAGE_VERSION})
# Setup the paths where compiler-rt runtimes and headers should be stored.
- set(COMPILER_RT_OUTPUT_DIR ${LLVM_LIBRARY_OUTPUT_INTDIR}/clang/${CLANG_VERSION})
+ set(COMPILER_RT_OUTPUT_DIR ${LLVM_LIBRARY_OUTPUT_INTDIR}/clang/${CLANG_VERSION_MAJOR})
set(COMPILER_RT_EXEC_OUTPUT_DIR ${LLVM_RUNTIME_OUTPUT_INTDIR})
- set(COMPILER_RT_INSTALL_PATH lib${LLVM_LIBDIR_SUFFIX}/clang/${CLANG_VERSION})
+ set(COMPILER_RT_INSTALL_PATH lib${LLVM_LIBDIR_SUFFIX}/clang/${CLANG_VERSION_MAJOR})
option(COMPILER_RT_INCLUDE_TESTS "Generate and build compiler-rt unit tests."
${LLVM_INCLUDE_TESTS})
option(COMPILER_RT_ENABLE_WERROR "Fail and stop if warning is triggered"
set(COMPILER_RT_TEST_COMPILER_ID GNU)
endif()
-function(extend_install_path joined_path current_segment)
- if("${current_segment}" STREQUAL "")
- set(temp_path "${COMPILER_RT_INSTALL_PATH}")
- elseif("${COMPILER_RT_INSTALL_PATH}" STREQUAL "")
- set(temp_path "${current_segment}")
- elseif(IS_ABSOLUTE "${current_segment}")
- message(WARNING "Since \"${current_segment}\" is absolute, it overrides COMPILER_RT_INSTALL_PATH: \"${COMPILER_RT_INSTALL_PATH}\".")
- set(temp_path "${current_segment}")
+if(NOT DEFINED COMPILER_RT_OS_DIR)
+ if(ANDROID)
+ # The CMAKE_SYSTEM_NAME for Android is Android, but the OS is Linux and the
+ # driver will search for compiler-rt libraries in the "linux" directory.
+ set(COMPILER_RT_OS_DIR linux)
else()
- set(temp_path "${COMPILER_RT_INSTALL_PATH}/${current_segment}")
+ string(TOLOWER ${CMAKE_SYSTEM_NAME} COMPILER_RT_OS_DIR)
endif()
- set(${joined_path} "${temp_path}" PARENT_SCOPE)
-endfunction()
-
-if(NOT DEFINED COMPILER_RT_OS_DIR)
- string(TOLOWER ${CMAKE_SYSTEM_NAME} COMPILER_RT_OS_DIR)
endif()
if(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR AND NOT APPLE)
set(COMPILER_RT_OUTPUT_LIBRARY_DIR
${COMPILER_RT_OUTPUT_DIR}/lib)
- extend_install_path(default_install_path lib)
+ extend_path(default_install_path "${COMPILER_RT_INSTALL_PATH}" lib)
set(COMPILER_RT_INSTALL_LIBRARY_DIR "${default_install_path}" CACHE PATH
"Path where built compiler-rt libraries should be installed.")
else(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR AND NOT APPLE)
set(COMPILER_RT_OUTPUT_LIBRARY_DIR
${COMPILER_RT_OUTPUT_DIR}/lib/${COMPILER_RT_OS_DIR})
- extend_install_path(default_install_path "lib/${COMPILER_RT_OS_DIR}")
+ extend_path(default_install_path "${COMPILER_RT_INSTALL_PATH}" "lib/${COMPILER_RT_OS_DIR}")
set(COMPILER_RT_INSTALL_LIBRARY_DIR "${default_install_path}" CACHE PATH
"Path where built compiler-rt libraries should be installed.")
endif()
-extend_install_path(default_install_path bin)
+extend_path(default_install_path "${COMPILER_RT_INSTALL_PATH}" "${CMAKE_INSTALL_BINDIR}")
set(COMPILER_RT_INSTALL_BINARY_DIR "${default_install_path}" CACHE PATH
"Path where built compiler-rt executables should be installed.")
-extend_install_path(default_install_path include)
+extend_path(default_install_path "${COMPILER_RT_INSTALL_PATH}" "${CMAKE_INSTALL_INCLUDEDIR}")
set(COMPILER_RT_INSTALL_INCLUDE_DIR "${default_install_path}" CACHE PATH
"Path where compiler-rt headers should be installed.")
-extend_install_path(default_install_path share)
+extend_path(default_install_path "${COMPILER_RT_INSTALL_PATH}" "${CMAKE_INSTALL_DATADIR}")
set(COMPILER_RT_INSTALL_DATA_DIR "${default_install_path}" CACHE PATH
"Path where compiler-rt data files should be installed.")
set(OSX_SYSROOT_FLAG "")
endif()
- option(COMPILER_RT_ENABLE_IOS "Enable building for iOS" On)
+ try_compile_only(COMPILER_RT_HAS_DARWIN_TARGET_VARIANT_FLAG
+ FLAGS
+ "-target" "x86_64-apple-macos10.15"
+ "-darwin-target-variant" "x86_64-apple-ios13.1-macabi"
+ "-Werror")
+ option(COMPILER_RT_ENABLE_MACCATALYST "Enable building for Mac Catalyst" ${COMPILER_RT_HAS_DARWIN_TARGET_VARIANT_FLAG})
+
+ # Don't enable COMPILER_RT_ENABLE_IOS if we can't find the sdk dir.
+ # This can happen when you only have the commandline tools installed
+ # which doesn't come with the iOS SDK.
+ find_darwin_sdk_dir(HAS_IOS_SDK "iphoneos")
+ set(COMPILER_RT_ENABLE_IOS_DEFAULT On)
+ if("${HAS_IOS_SDK}" STREQUAL "")
+ message(WARNING "iOS SDK not found! Building compiler-rt without iOS support.")
+ set(COMPILER_RT_ENABLE_IOS_DEFAULT Off)
+ endif()
+ option(COMPILER_RT_ENABLE_IOS "Enable building for iOS" ${COMPILER_RT_ENABLE_IOS_DEFAULT})
+
option(COMPILER_RT_ENABLE_WATCHOS "Enable building for watchOS - Experimental" Off)
option(COMPILER_RT_ENABLE_TVOS "Enable building for tvOS - Experimental" Off)
test_target_arch(x86_64 "" "")
endif()
endif()
- elseif("${COMPILER_RT_DEFAULT_TARGET_ARCH}" MATCHES "powerpc64le")
+ elseif("${COMPILER_RT_DEFAULT_TARGET_ARCH}" MATCHES "loongarch64")
+ test_target_arch(loongarch64 "" "")
+ elseif("${COMPILER_RT_DEFAULT_TARGET_ARCH}" MATCHES "powerpc64le|ppc64le")
test_target_arch(powerpc64le "" "-m64")
elseif("${COMPILER_RT_DEFAULT_TARGET_ARCH}" MATCHES "powerpc")
- if(CMAKE_SYSTEM_NAME MATCHES "AIX")
- test_target_arch(powerpc "" "-m32")
- endif()
+ test_target_arch(powerpc "" "-m32")
test_target_arch(powerpc64 "" "-m64")
elseif("${COMPILER_RT_DEFAULT_TARGET_ARCH}" MATCHES "s390x")
test_target_arch(s390x "" "")
elseif("${COMPILER_RT_DEFAULT_TARGET_ARCH}" MATCHES "sparc")
test_target_arch(sparc "" "-m32")
test_target_arch(sparcv9 "" "-m64")
- elseif("${COMPILER_RT_DEFAULT_TARGET_ARCH}" MATCHES "mipsel|mips64el")
- # Gcc doesn't accept -m32/-m64 so we do the next best thing and use
- # -mips32r2/-mips64r2. We don't use -mips1/-mips3 because we want to match
- # clang's default CPU's. In the 64-bit case, we must also specify the ABI
- # since the default ABI differs between gcc and clang.
- # FIXME: Ideally, we would build the N32 library too.
- test_target_arch(mipsel "" "-mips32r2" "-mabi=32" "-D_LARGEFILE_SOURCE" "-D_FILE_OFFSET_BITS=64")
- test_target_arch(mips64el "" "-mips64r2" "-mabi=64")
elseif("${COMPILER_RT_DEFAULT_TARGET_ARCH}" MATCHES "mips")
- test_target_arch(mips "" "-mips32r2" "-mabi=32" "-D_LARGEFILE_SOURCE" "-D_FILE_OFFSET_BITS=64")
- test_target_arch(mips64 "" "-mips64r2" "-mabi=64")
+ # FIXME: Ideally, we would build the N32 library too.
+ if("${COMPILER_RT_MIPS_EL}" AND ("${COMPILER_RT_MIPS32R6}" OR "${COMPILER_RT_MIPS64R6}"))
+ test_target_arch(mipsel "" "-mips32r6" "-mabi=32" "-D_LARGEFILE_SOURCE" "-D_FILE_OFFSET_BITS=64")
+ test_target_arch(mips64el "" "-mips64r6" "-mabi=64")
+ elseif("${COMPILER_RT_MIPS_EL}")
+ test_target_arch(mipsel "" "-mips32r2" "-mabi=32" "-D_LARGEFILE_SOURCE" "-D_FILE_OFFSET_BITS=64")
+ test_target_arch(mips64el "" "-mips64r2" "-mabi=64")
+ elseif("${COMPILER_RT_MIPS32R6}" OR "${COMPILER_RT_MIPS64R6}")
+ test_target_arch(mips "" "-mips32r6" "-mabi=32" "-D_LARGEFILE_SOURCE" "-D_FILE_OFFSET_BITS=64")
+ test_target_arch(mips64 "" "-mips64r6" "-mabi=64")
+ else()
+ test_target_arch(mips "" "-mips32r2" "-mabi=32" "-D_LARGEFILE_SOURCE" "-D_FILE_OFFSET_BITS=64")
+ test_target_arch(mips64 "" "-mips64r2" "-mabi=64")
+ endif()
elseif("${COMPILER_RT_DEFAULT_TARGET_ARCH}" MATCHES "arm")
if(WIN32)
test_target_arch(arm "" "" "")
else()
+ test_target_arch(armv4t "" "-march=armv4t" "-mfloat-abi=soft")
+ test_target_arch(armv6m "" "-march=armv6m" "-mfloat-abi=soft")
test_target_arch(arm "" "-march=armv7-a" "-mfloat-abi=soft")
test_target_arch(armhf "" "-march=armv7-a" "-mfloat-abi=hard")
- test_target_arch(armv6m "" "-march=armv6m" "-mfloat-abi=soft")
endif()
+ elseif("${COMPILER_RT_DEFAULT_TARGET_ARCH}" MATCHES "avr")
+ test_target_arch(avr "__AVR__" "--target=avr")
elseif("${COMPILER_RT_DEFAULT_TARGET_ARCH}" MATCHES "aarch32")
test_target_arch(aarch32 "" "-march=armv8-a")
elseif("${COMPILER_RT_DEFAULT_TARGET_ARCH}" MATCHES "aarch64")
builtin_check_c_compiler_flag(-fno-builtin COMPILER_RT_HAS_FNO_BUILTIN_FLAG)
builtin_check_c_compiler_flag(-std=c11 COMPILER_RT_HAS_STD_C11_FLAG)
builtin_check_c_compiler_flag(-fvisibility=hidden COMPILER_RT_HAS_VISIBILITY_HIDDEN_FLAG)
-builtin_check_c_compiler_flag(-ffreestanding COMPILER_RT_HAS_FREESTANDING_FLAG)
+builtin_check_c_compiler_flag(-fomit-frame-pointer COMPILER_RT_HAS_OMIT_FRAME_POINTER_FLAG)
+builtin_check_c_compiler_flag(-ffreestanding COMPILER_RT_HAS_FFREESTANDING_FLAG)
builtin_check_c_compiler_flag(-fxray-instrument COMPILER_RT_HAS_XRAY_COMPILER_FLAG)
builtin_check_c_compiler_source(COMPILER_RT_HAS_ATOMIC_KEYWORD
}
")
-builtin_check_c_compiler_source(COMPILER_RT_HAS_FLOAT16
-"
-_Float16 foo(_Float16 x) {
- return x;
-}
-"
-)
-
builtin_check_c_compiler_source(COMPILER_RT_HAS_ASM_LSE
"
asm(\".arch armv8-a+lse\");
")
set(ARM64 aarch64)
-set(ARM32 arm armhf armv6m armv7m armv7em armv7 armv7s armv7k armv8m.main armv8.1m.main)
+set(ARM32 arm armhf armv4t armv5te armv6 armv6m armv7m armv7em armv7 armv7s armv7k armv8m.main armv8.1m.main)
+set(AVR avr)
set(HEXAGON hexagon)
set(X86 i386)
set(X86_64 x86_64)
+set(LOONGARCH64 loongarch64)
set(MIPS32 mips mipsel)
set(MIPS64 mips64 mips64el)
-set(PPC32 powerpc)
+set(PPC32 powerpc powerpcspe)
set(PPC64 powerpc64 powerpc64le)
set(RISCV32 riscv32)
set(RISCV64 riscv64)
endif()
set(ALL_BUILTIN_SUPPORTED_ARCH
- ${X86} ${X86_64} ${ARM32} ${ARM64}
+ ${X86} ${X86_64} ${ARM32} ${ARM64} ${AVR}
${HEXAGON} ${MIPS32} ${MIPS64} ${PPC32} ${PPC64}
${RISCV32} ${RISCV64} ${SPARC} ${SPARCV9}
- ${WASM32} ${WASM64} ${VE})
+ ${WASM32} ${WASM64} ${VE} ${LOONGARCH64})
include(CompilerRTUtils)
include(CompilerRTDarwinUtils)
execute_process(COMMAND
/usr/libexec/PlistBuddy -c "Print :SupportedTargets:${os}:Archs" ${sdk_path}/SDKSettings.plist
OUTPUT_VARIABLE SDK_SUPPORTED_ARCHS
- RESULT_VARIABLE PLIST_ERROR)
+ RESULT_VARIABLE PLIST_ERROR
+ ERROR_QUIET)
if (PLIST_ERROR EQUAL 0 AND
SDK_SUPPORTED_ARCHS MATCHES " ${arch}\n")
message(STATUS "Found ${arch} support in ${sdk_path}/SDKSettings.plist")
endfunction()
set(DARWIN_EMBEDDED_PLATFORMS)
- set(DARWIN_osx_BUILTIN_MIN_VER 10.5)
+ set(DARWIN_osx_BUILTIN_MIN_VER 10.7)
set(DARWIN_osx_BUILTIN_MIN_VER_FLAG
-mmacosx-version-min=${DARWIN_osx_BUILTIN_MIN_VER})
set(DARWIN_osx_BUILTIN_ALL_POSSIBLE_ARCHS ${X86} ${X86_64})
${DARWIN_ios_MIN_VER_FLAG}=${DARWIN_ios_BUILTIN_MIN_VER})
set(DARWIN_ios_BUILTIN_ALL_POSSIBLE_ARCHS ${ARM64} ${ARM32})
set(DARWIN_iossim_BUILTIN_ALL_POSSIBLE_ARCHS ${X86} ${X86_64})
+ find_darwin_sdk_version(iossim_sdk_version "iphonesimulator")
+ if ("${iossim_sdk_version}" VERSION_GREATER 14.0 OR "${iossim_sdk_version}" VERSION_EQUAL 14.0)
+ list(APPEND DARWIN_iossim_BUILTIN_ALL_POSSIBLE_ARCHS arm64)
+ endif()
endif()
if(COMPILER_RT_ENABLE_WATCHOS)
list(APPEND DARWIN_EMBEDDED_PLATFORMS watchos)
${DARWIN_watchos_MIN_VER_FLAG}=${DARWIN_watchos_BUILTIN_MIN_VER})
set(DARWIN_watchos_BUILTIN_ALL_POSSIBLE_ARCHS armv7 armv7k arm64_32)
set(DARWIN_watchossim_BUILTIN_ALL_POSSIBLE_ARCHS ${X86})
+ find_darwin_sdk_version(watchossim_sdk_version "watchsimulator")
+ if ("${watchossim_sdk_version}" VERSION_GREATER 7.0 OR "${watchossim_sdk_version}" VERSION_EQUAL 7.0)
+ list(APPEND DARWIN_watchossim_BUILTIN_ALL_POSSIBLE_ARCHS arm64)
+ endif()
endif()
if(COMPILER_RT_ENABLE_TVOS)
list(APPEND DARWIN_EMBEDDED_PLATFORMS tvos)
${DARWIN_tvos_MIN_VER_FLAG}=${DARWIN_tvos_BUILTIN_MIN_VER})
set(DARWIN_tvos_BUILTIN_ALL_POSSIBLE_ARCHS armv7 arm64)
set(DARWIN_tvossim_BUILTIN_ALL_POSSIBLE_ARCHS ${X86} ${X86_64})
+ find_darwin_sdk_version(tvossim_sdk_version "appletvsimulator")
+ if ("${tvossim_sdk_version}" VERSION_GREATER 14.0 OR "${tvossim_sdk_version}" VERSION_EQUAL 14.0)
+ list(APPEND DARWIN_tvossim_BUILTIN_ALL_POSSIBLE_ARCHS arm64)
+ endif()
endif()
set(BUILTIN_SUPPORTED_OS osx)
include(CMakePushCheckState)
+include(LLVMCheckCompilerLinkerFlag)
include(CheckCCompilerFlag)
include(CheckCXXCompilerFlag)
include(CheckIncludeFiles)
include(CheckLibraryExists)
+include(LLVMCheckCompilerLinkerFlag)
include(CheckSymbolExists)
include(TestBigEndian)
-function(compiler_rt_check_linker_flag flag out_var)
- cmake_push_check_state()
- set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${flag}")
- check_cxx_compiler_flag("" ${out_var})
- cmake_pop_check_state()
-endfunction()
+# The compiler driver may be implicitly trying to link against libunwind.
+# This is normally ok (libcxx relies on an unwinder), but if libunwind is
+# built in the same cmake invocation as compiler-rt and we're using the
+# in tree version of runtimes, we'd be linking against the just-built
+# libunwind (and the compiler implicit -lunwind wouldn't succeed as the newly
+# built libunwind isn't installed yet). For those cases, it'd be good to
+# link with --uwnindlib=none. Check if that option works.
+llvm_check_compiler_linker_flag(C "--unwindlib=none" CXX_SUPPORTS_UNWINDLIB_NONE_FLAG)
check_library_exists(c fopen "" COMPILER_RT_HAS_LIBC)
if (COMPILER_RT_USE_BUILTINS_LIBRARY)
include(HandleCompilerRT)
- find_compiler_rt_library(builtins "" COMPILER_RT_BUILTINS_LIBRARY)
+ find_compiler_rt_library(builtins COMPILER_RT_BUILTINS_LIBRARY
+ FLAGS ${SANITIZER_COMMON_FLAGS})
+ # TODO(PR51389): We should check COMPILER_RT_BUILTINS_LIBRARY and report an
+ # error if the value is NOTFOUND rather than silenty continuing but we first
+ # need to fix find_compiler_rt_library on Darwin.
else()
if (ANDROID)
check_library_exists(gcc __gcc_personality_v0 "" COMPILER_RT_HAS_GCC_LIB)
endif()
endif()
-check_c_compiler_flag(-nodefaultlibs COMPILER_RT_HAS_NODEFAULTLIBS_FLAG)
-if (COMPILER_RT_HAS_NODEFAULTLIBS_FLAG)
+check_c_compiler_flag(-nodefaultlibs C_SUPPORTS_NODEFAULTLIBS_FLAG)
+if (C_SUPPORTS_NODEFAULTLIBS_FLAG)
set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -nodefaultlibs")
if (COMPILER_RT_HAS_LIBC)
list(APPEND CMAKE_REQUIRED_LIBRARIES c)
endif ()
if (COMPILER_RT_USE_BUILTINS_LIBRARY)
- list(APPEND CMAKE_REQUIRED_LIBRARIES "${COMPILER_RT_BUILTINS_LIBRARY}")
+ # TODO: remote this check once we address PR51389.
+ if (${COMPILER_RT_BUILTINS_LIBRARY})
+ list(APPEND CMAKE_REQUIRED_LIBRARIES "${COMPILER_RT_BUILTINS_LIBRARY}")
+ endif()
elseif (COMPILER_RT_HAS_GCC_S_LIB)
list(APPEND CMAKE_REQUIRED_LIBRARIES gcc_s)
elseif (COMPILER_RT_HAS_GCC_LIB)
check_c_compiler_flag(-ffreestanding COMPILER_RT_HAS_FFREESTANDING_FLAG)
check_c_compiler_flag(-fomit-frame-pointer COMPILER_RT_HAS_OMIT_FRAME_POINTER_FLAG)
check_c_compiler_flag(-std=c11 COMPILER_RT_HAS_STD_C11_FLAG)
+check_c_compiler_flag(-fcf-protection=full COMPILER_RT_HAS_FCF_PROTECTION_FLAG)
check_cxx_compiler_flag(-fPIC COMPILER_RT_HAS_FPIC_FLAG)
check_cxx_compiler_flag(-fPIE COMPILER_RT_HAS_FPIE_FLAG)
check_cxx_compiler_flag(-fno-builtin COMPILER_RT_HAS_FNO_BUILTIN_FLAG)
check_cxx_compiler_flag(-frtti COMPILER_RT_HAS_FRTTI_FLAG)
check_cxx_compiler_flag(-fno-rtti COMPILER_RT_HAS_FNO_RTTI_FLAG)
check_cxx_compiler_flag("-Werror -fno-function-sections" COMPILER_RT_HAS_FNO_FUNCTION_SECTIONS_FLAG)
-check_cxx_compiler_flag(-std=c++14 COMPILER_RT_HAS_STD_CXX14_FLAG)
check_cxx_compiler_flag(-ftls-model=initial-exec COMPILER_RT_HAS_FTLS_MODEL_INITIAL_EXEC)
check_cxx_compiler_flag(-fno-lto COMPILER_RT_HAS_FNO_LTO_FLAG)
check_cxx_compiler_flag(-fno-profile-generate COMPILER_RT_HAS_FNO_PROFILE_GENERATE_FLAG)
check_cxx_compiler_flag(-fno-profile-instr-generate COMPILER_RT_HAS_FNO_PROFILE_INSTR_GENERATE_FLAG)
check_cxx_compiler_flag(-fno-profile-instr-use COMPILER_RT_HAS_FNO_PROFILE_INSTR_USE_FLAG)
+check_cxx_compiler_flag(-fno-coverage-mapping COMPILER_RT_HAS_FNO_COVERAGE_MAPPING_FLAG)
+check_cxx_compiler_flag("-Werror -mcrc32" COMPILER_RT_HAS_MCRC32_FLAG)
check_cxx_compiler_flag("-Werror -msse3" COMPILER_RT_HAS_MSSE3_FLAG)
check_cxx_compiler_flag("-Werror -msse4.2" COMPILER_RT_HAS_MSSE4_2_FLAG)
check_cxx_compiler_flag(--sysroot=. COMPILER_RT_HAS_SYSROOT_FLAG)
check_cxx_compiler_flag("-Werror -mcrc" COMPILER_RT_HAS_MCRC_FLAG)
check_cxx_compiler_flag(-fno-partial-inlining COMPILER_RT_HAS_FNO_PARTIAL_INLINING_FLAG)
+check_cxx_compiler_flag(-Werror -ftrivial-auto-var-init=pattern COMPILER_RT_HAS_TRIVIAL_AUTO_INIT)
if(NOT WIN32 AND NOT CYGWIN)
# MinGW warns if -fvisibility-inlines-hidden is used.
check_cxx_compiler_flag("-Werror -Wunused-parameter" COMPILER_RT_HAS_WUNUSED_PARAMETER_FLAG)
check_cxx_compiler_flag("-Werror -Wcovered-switch-default" COMPILER_RT_HAS_WCOVERED_SWITCH_DEFAULT_FLAG)
check_cxx_compiler_flag("-Werror -Wsuggest-override" COMPILER_RT_HAS_WSUGGEST_OVERRIDE_FLAG)
+check_cxx_compiler_flag("-Werror -Wthread-safety" COMPILER_RT_HAS_WTHREAD_SAFETY_FLAG)
+check_cxx_compiler_flag("-Werror -Wthread-safety-reference" COMPILER_RT_HAS_WTHREAD_SAFETY_REFERENCE_FLAG)
+check_cxx_compiler_flag("-Werror -Wthread-safety-beta" COMPILER_RT_HAS_WTHREAD_SAFETY_BETA_FLAG)
check_cxx_compiler_flag(-Wno-pedantic COMPILER_RT_HAS_WNO_PEDANTIC)
+check_cxx_compiler_flag(-Wno-format COMPILER_RT_HAS_WNO_FORMAT)
+check_cxx_compiler_flag(-Wno-format-pedantic COMPILER_RT_HAS_WNO_FORMAT_PEDANTIC)
+
+check_cxx_compiler_flag("/experimental:external /external:W0" COMPILER_RT_HAS_EXTERNAL_FLAG)
check_cxx_compiler_flag(/W4 COMPILER_RT_HAS_W4_FLAG)
check_cxx_compiler_flag(/WX COMPILER_RT_HAS_WX_FLAG)
check_cxx_compiler_flag(/wd4146 COMPILER_RT_HAS_WD4146_FLAG)
+check_cxx_compiler_flag(/wd4206 COMPILER_RT_HAS_WD4206_FLAG)
check_cxx_compiler_flag(/wd4291 COMPILER_RT_HAS_WD4291_FLAG)
check_cxx_compiler_flag(/wd4221 COMPILER_RT_HAS_WD4221_FLAG)
check_cxx_compiler_flag(/wd4391 COMPILER_RT_HAS_WD4391_FLAG)
check_cxx_compiler_flag(/wd4722 COMPILER_RT_HAS_WD4722_FLAG)
check_cxx_compiler_flag(/wd4800 COMPILER_RT_HAS_WD4800_FLAG)
+check_cxx_compiler_flag(-Werror -Warray-bounds COMPILER_RT_HAS_ARRAY_BOUNDS_FLAG)
+check_cxx_compiler_flag(-Werror -Wuninitialized COMPILER_RT_HAS_UNINITIALIZED_FLAG)
+check_cxx_compiler_flag(-Werror -Wshadow COMPILER_RT_HAS_SHADOW_FLAG)
+check_cxx_compiler_flag(-Werror -Wempty-body COMPILER_RT_HAS_EMPTY_BODY_FLAG)
+check_cxx_compiler_flag(-Werror -Wsizeof-pointer-memaccess COMPILER_RT_HAS_SIZEOF_POINTER_MEMACCESS_FLAG)
+check_cxx_compiler_flag(-Werror -Wsizeof-array-argument COMPILER_RT_HAS_SIZEOF_ARRAY_ARGUMENT_FLAG)
+check_cxx_compiler_flag(-Werror -Wsuspicious-memaccess COMPILER_RT_HAS_SUSPICIOUS_MEMACCESS_FLAG)
+check_cxx_compiler_flag(-Werror -Wbuiltin-memcpy-chk-size COMPILER_RT_HAS_BUILTIN_MEMCPY_CHK_SIZE_FLAG)
+check_cxx_compiler_flag(-Werror -Warray-bounds-pointer-arithmetic COMPILER_RT_HAS_ARRAY_BOUNDS_POINTER_ARITHMETIC_FLAG)
+check_cxx_compiler_flag(-Werror -Wreturn-stack-address COMPILER_RT_HAS_RETURN_STACK_ADDRESS_FLAG)
+check_cxx_compiler_flag(-Werror -Wsizeof-array-decay COMPILER_RT_HAS_SIZEOF_ARRAY_DECAY_FLAG)
+check_cxx_compiler_flag(-Werror -Wformat-insufficient-args COMPILER_RT_HAS_FORMAT_INSUFFICIENT_ARGS_FLAG)
+check_cxx_compiler_flag(-Werror -Wformat-security COMPILER_RT_HAS_BUILTIN_FORMAL_SECURITY_FLAG)
+check_cxx_compiler_flag(-Werror -Wsizeof-array-div COMPILER_RT_HAS_SIZEOF_ARRAY_DIV_FLAG)
+check_cxx_compiler_flag(-Werror -Wsizeof-pointer-div COMPILER_RT_HAS_SIZEOF_POINTER_DIV_FLAG)
+
# Symbols.
check_symbol_exists(__func__ "" COMPILER_RT_HAS_FUNC_SYMBOL)
check_library_exists(stdc++ __cxa_throw "" COMPILER_RT_HAS_LIBSTDCXX)
# Linker flags.
-compiler_rt_check_linker_flag("-Wl,-z,text" COMPILER_RT_HAS_Z_TEXT)
-compiler_rt_check_linker_flag("-fuse-ld=lld" COMPILER_RT_HAS_FUSE_LD_LLD_FLAG)
+llvm_check_compiler_linker_flag(C "-Wl,-z,text" COMPILER_RT_HAS_Z_TEXT)
+llvm_check_compiler_linker_flag(C "-fuse-ld=lld" COMPILER_RT_HAS_FUSE_LD_LLD_FLAG)
-set(VERS_COMPAT_OPTION "-Wl,-z,gnu-version-script-compat")
-compiler_rt_check_linker_flag("${VERS_COMPAT_OPTION}" COMPILER_RT_HAS_GNU_VERSION_SCRIPT_COMPAT)
+if(${CMAKE_SYSTEM_NAME} MATCHES "SunOS")
+ set(VERS_COMPAT_OPTION "-Wl,-z,gnu-version-script-compat")
+ llvm_check_compiler_linker_flag(C "${VERS_COMPAT_OPTION}" COMPILER_RT_HAS_GNU_VERSION_SCRIPT_COMPAT)
+endif()
set(DUMMY_VERS ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/dummy.vers)
file(WRITE ${DUMMY_VERS} "{};")
# -z gnu-version-script-compat.
string(APPEND VERS_OPTION " ${VERS_COMPAT_OPTION}")
endif()
-compiler_rt_check_linker_flag("${VERS_OPTION}" COMPILER_RT_HAS_VERSION_SCRIPT)
+llvm_check_compiler_linker_flag(C "${VERS_OPTION}" COMPILER_RT_HAS_VERSION_SCRIPT)
if(ANDROID)
- compiler_rt_check_linker_flag("-Wl,-z,global" COMPILER_RT_HAS_Z_GLOBAL)
+ llvm_check_compiler_linker_flag(C "-Wl,-z,global" COMPILER_RT_HAS_Z_GLOBAL)
check_library_exists(log __android_log_write "" COMPILER_RT_HAS_LIBLOG)
endif()
# runtime libraries supported by our current compilers cross-compiling
# abilities.
set(SIMPLE_SOURCE ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/simple.cc)
-file(WRITE ${SIMPLE_SOURCE} "#include <stdlib.h>\n#include <stdio.h>\nint main() { printf(\"hello, world\"); }\n")
+file(WRITE ${SIMPLE_SOURCE} "#include <stdlib.h>\n#include <stdio.h>\nint main(void) { printf(\"hello, world\"); }\n")
# Detect whether the current target platform is 32-bit or 64-bit, and setup
# the correct commandline flags needed to attempt to target 32-bit and 64-bit.
+# AVR and MSP430 are omitted since they have 16-bit pointers.
if (NOT CMAKE_SIZEOF_VOID_P EQUAL 4 AND
- NOT CMAKE_SIZEOF_VOID_P EQUAL 8)
+ NOT CMAKE_SIZEOF_VOID_P EQUAL 8 AND
+ NOT ${arch} MATCHES "avr|msp430")
message(FATAL_ERROR "Please use architecture with 4 or 8 byte pointers.")
endif()
endif()
endfunction()
+# Returns a list of architecture specific target ldflags in @out_var list.
+function(get_target_link_flags_for_arch arch out_var)
+ list(FIND COMPILER_RT_SUPPORTED_ARCH ${arch} ARCH_INDEX)
+ if(ARCH_INDEX EQUAL -1)
+ message(FATAL_ERROR "Unsupported architecture: ${arch}")
+ else()
+ # Workaround for direct calls to __tls_get_addr on Solaris/amd64.
+ if(OS_NAME MATCHES "SunOS" AND ${arch} MATCHES x86_64)
+ set(${out_var} "-Wl,-z,relax=transtls" PARENT_SCOPE)
+ endif()
+ endif()
+endfunction()
+
# Returns a compiler and CFLAGS that should be used to run tests for the
# specific architecture. When cross-compiling, this is controled via
# COMPILER_RT_TEST_COMPILER and COMPILER_RT_TEST_COMPILER_CFLAGS.
macro(get_test_cc_for_arch arch cc_out cflags_out)
- if(ANDROID OR ${arch} MATCHES "arm|aarch64|riscv32|riscv64")
+ if (NOT ${ARGC} EQUAL 3)
+ message(FATAL_ERROR "got too many args. expected 3, got ${ARGC} (namely: ${ARGV})")
+ endif()
+ if(ANDROID OR (NOT APPLE AND ${arch} MATCHES "arm|aarch64|riscv32|riscv64"))
# This is only true if we are cross-compiling.
# Build all tests with host compiler and use host tools.
set(${cc_out} ${COMPILER_RT_TEST_COMPILER})
endif()
set(test_cflags "")
get_target_flags_for_arch(${arch} test_cflags)
- list(APPEND test_cflags ${DARWIN_${platform}_CFLAGS})
+
+ if (NOT "${arch}" STREQUAL "arm64e")
+ list(APPEND test_cflags ${DARWIN_${platform}_CFLAGS})
+ else()
+ # arm64e is not currently ABI stable so we need to build for the
+ # OS version being tested. Rather than querying the device under test
+ # we use the SDK version which "should" be the same as the
+ # device under test (it is a configuration error for these not to match).
+ # FIXME(dliew): We can remove this if we build the runtimes with the appropriate
+ # deployment target for arm64e.
+ foreach (flag ${DARWIN_${platform}_CFLAGS})
+ if ("${flag}" MATCHES "^${DARWIN_${platform}_MIN_VER_FLAG}=.+")
+ # Find the SDK version
+ get_xcrun_platform_from_apple_platform("${platform}" xcrun_platform_name)
+ # TODO(dliew): Remove this check once get_xcrun_platform_from_apple_platform
+ # emits a fatal error for unrecognised platforms.
+ if (NOT "${xcrun_platform_name}" STREQUAL "")
+ find_darwin_sdk_version(platform_sdk_version "${xcrun_platform_name}")
+ # Patch flag with correct deployment target
+ set(replacement_flag "${DARWIN_${platform}_MIN_VER_FLAG}=${platform_sdk_version}")
+ list(APPEND test_cflags "${replacement_flag}")
+ endif()
+ else()
+ # Copy through
+ list(APPEND test_cflags "${flag}")
+ endif()
+ endforeach()
+ endif()
+
string(REPLACE ";" " " test_cflags_str "${test_cflags}")
string(APPEND test_cflags_str "${COMPILER_RT_TEST_COMPILER_CFLAGS}")
set(${cflags_out} "${test_cflags_str}" PARENT_SCOPE)
set(${is_valid_out} ${is_valid} PARENT_SCOPE)
endfunction()
-set(ARM64 aarch64)
-set(ARM32 arm armhf)
-set(HEXAGON hexagon)
-set(X86 i386)
-set(X86_64 x86_64)
-set(MIPS32 mips mipsel)
-set(MIPS64 mips64 mips64el)
-set(PPC32 powerpc)
-set(PPC64 powerpc64 powerpc64le)
-set(RISCV32 riscv32)
-set(RISCV64 riscv64)
-set(S390X s390x)
-set(SPARC sparc)
-set(SPARCV9 sparcv9)
-set(WASM32 wasm32)
-set(WASM64 wasm64)
-set(VE ve)
-
-if(APPLE)
- set(ARM64 arm64)
- set(ARM32 armv7 armv7s armv7k)
- set(X86_64 x86_64 x86_64h)
-endif()
-
-set(ALL_SANITIZER_COMMON_SUPPORTED_ARCH ${X86} ${X86_64} ${PPC64} ${RISCV64}
- ${ARM32} ${ARM64} ${MIPS32} ${MIPS64} ${S390X} ${SPARC} ${SPARCV9})
-set(ALL_ASAN_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64} ${RISCV64}
- ${MIPS32} ${MIPS64} ${PPC64} ${S390X} ${SPARC} ${SPARCV9})
-set(ALL_CRT_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64} ${RISCV32} ${RISCV64} ${VE})
-set(ALL_DFSAN_SUPPORTED_ARCH ${X86_64} ${MIPS64} ${ARM64})
-
-if(ANDROID)
- set(OS_NAME "Android")
-else()
- set(OS_NAME "${CMAKE_SYSTEM_NAME}")
-endif()
-
-if(OS_NAME MATCHES "Linux")
- set(ALL_FUZZER_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM64} ${S390X})
-elseif (OS_NAME MATCHES "Windows")
- set(ALL_FUZZER_SUPPORTED_ARCH ${X86} ${X86_64})
-elseif(OS_NAME MATCHES "Android")
- set(ALL_FUZZER_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64})
-else()
- set(ALL_FUZZER_SUPPORTED_ARCH ${X86_64} ${ARM64})
-endif()
-
-set(ALL_GWP_ASAN_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64})
-if(APPLE)
- set(ALL_LSAN_SUPPORTED_ARCH ${X86} ${X86_64} ${MIPS64} ${ARM64})
-else()
- set(ALL_LSAN_SUPPORTED_ARCH ${X86} ${X86_64} ${MIPS64} ${ARM64} ${ARM32} ${PPC64} ${S390X} ${RISCV64})
-endif()
-set(ALL_MSAN_SUPPORTED_ARCH ${X86_64} ${MIPS64} ${ARM64} ${PPC64} ${S390X})
-set(ALL_HWASAN_SUPPORTED_ARCH ${X86_64} ${ARM64})
-set(ALL_MEMPROF_SUPPORTED_ARCH ${X86_64})
-set(ALL_PROFILE_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64} ${PPC32} ${PPC64}
- ${MIPS32} ${MIPS64} ${S390X} ${SPARC} ${SPARCV9})
-set(ALL_TSAN_SUPPORTED_ARCH ${X86_64} ${MIPS64} ${ARM64} ${PPC64} ${S390X})
-set(ALL_UBSAN_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64} ${RISCV64}
- ${MIPS32} ${MIPS64} ${PPC64} ${S390X} ${SPARC} ${SPARCV9})
-set(ALL_SAFESTACK_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM64} ${MIPS32} ${MIPS64})
-set(ALL_CFI_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64} ${MIPS64})
-set(ALL_SCUDO_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64} ${MIPS32} ${MIPS64} ${PPC64})
-set(ALL_SCUDO_STANDALONE_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64} ${MIPS32} ${MIPS64} ${PPC64})
-if(APPLE)
-set(ALL_XRAY_SUPPORTED_ARCH ${X86_64})
-else()
-set(ALL_XRAY_SUPPORTED_ARCH ${X86_64} ${ARM32} ${ARM64} ${MIPS32} ${MIPS64} powerpc64le)
-endif()
-set(ALL_SHADOWCALLSTACK_SUPPORTED_ARCH ${ARM64})
+# Maps the Apple platform name used in Compiler-rt's CMake code
+# to the name recognised by xcrun's `--sdk` argument
+function(get_xcrun_platform_from_apple_platform platform out_var)
+ set(xcrun_platform "")
+ if ("${platform}" STREQUAL "osx")
+ set(xcrun_platform "macosx")
+ elseif ("${platform}" STREQUAL "iossim")
+ set(xcrun_platform "iphonesimulator")
+ elseif ("${platform}" STREQUAL "ios")
+ set(xcrun_platform "iphoneos")
+ elseif ("${platform}" STREQUAL "watchossim")
+ set(xcrun_platform "watchsimulator")
+ elseif ("${platform}" STREQUAL "watchos")
+ set(xcrun_platform "watchos")
+ elseif ("${platform}" STREQUAL "tvossim")
+ set(xcrun_platform "appletvsimulator")
+ elseif ("${platform}" STREQUAL "tvos")
+ set(xcrun_platform "appletvos")
+ else()
+ # TODO(dliew): Make this an error.
+ message(WARNING "\"${platform}\" is not a handled apple platform")
+ endif()
+ set(${out_var} ${xcrun_platform} PARENT_SCOPE)
+endfunction()
-if (UNIX)
-set(ALL_ORC_SUPPORTED_ARCH ${X86_64})
-endif()
+include(AllSupportedArchDefs)
if(APPLE)
include(CompilerRTDarwinUtils)
set(XRAY_SUPPORTED_OS osx)
set(FUZZER_SUPPORTED_OS osx)
set(ORC_SUPPORTED_OS osx)
+ set(UBSAN_SUPPORTED_OS osx)
+ set(LSAN_SUPPORTED_OS osx)
+ set(STATS_SUPPORTED_OS osx)
# Note: In order to target x86_64h on OS X the minimum deployment target must
# be 10.8 or higher.
-lc++
-lc++abi)
- compiler_rt_check_linker_flag("-fapplication-extension" COMPILER_RT_HAS_APP_EXTENSION)
+ llvm_check_compiler_linker_flag(C "-fapplication-extension" COMPILER_RT_HAS_APP_EXTENSION)
if(COMPILER_RT_HAS_APP_EXTENSION)
list(APPEND DARWIN_COMMON_LINK_FLAGS "-fapplication-extension")
endif()
list(APPEND PROFILE_SUPPORTED_OS ${platform}sim)
list(APPEND TSAN_SUPPORTED_OS ${platform}sim)
list(APPEND FUZZER_SUPPORTED_OS ${platform}sim)
+ list(APPEND ORC_SUPPORTED_OS ${platform}sim)
+ list(APPEND UBSAN_SUPPORTED_OS ${platform}sim)
+ list(APPEND LSAN_SUPPORTED_OS ${platform}sim)
+ list(APPEND STATS_SUPPORTED_OS ${platform}sim)
endif()
foreach(arch ${DARWIN_${platform}sim_ARCHS})
list(APPEND COMPILER_RT_SUPPORTED_ARCH ${arch})
list(APPEND TSAN_SUPPORTED_OS ${platform})
endif()
list(APPEND FUZZER_SUPPORTED_OS ${platform})
+ list(APPEND ORC_SUPPORTED_OS ${platform})
+ list(APPEND UBSAN_SUPPORTED_OS ${platform})
+ list(APPEND LSAN_SUPPORTED_OS ${platform})
+ list(APPEND STATS_SUPPORTED_OS ${platform})
endif()
foreach(arch ${DARWIN_${platform}_ARCHS})
list(APPEND COMPILER_RT_SUPPORTED_ARCH ${arch})
endforeach()
endif()
- # Explictly disable unsupported Sanitizer configurations.
+ # Explicitly disable unsupported Sanitizer configurations.
list(REMOVE_ITEM FUZZER_SUPPORTED_OS "watchos")
list(REMOVE_ITEM FUZZER_SUPPORTED_OS "watchossim")
list_intersect(CFI_SUPPORTED_ARCH
ALL_CFI_SUPPORTED_ARCH
SANITIZER_COMMON_SUPPORTED_ARCH)
- list_intersect(SCUDO_SUPPORTED_ARCH
- ALL_SCUDO_SUPPORTED_ARCH
- SANITIZER_COMMON_SUPPORTED_ARCH)
list_intersect(SCUDO_STANDALONE_SUPPORTED_ARCH
ALL_SCUDO_STANDALONE_SUPPORTED_ARCH
SANITIZER_COMMON_SUPPORTED_ARCH)
SANITIZER_COMMON_SUPPORTED_ARCH)
else()
- filter_available_targets(CRT_SUPPORTED_ARCH ${ALL_CRT_SUPPORTED_ARCH})
# Architectures supported by compiler-rt libraries.
filter_available_targets(SANITIZER_COMMON_SUPPORTED_ARCH
${ALL_SANITIZER_COMMON_SUPPORTED_ARCH})
filter_available_targets(SAFESTACK_SUPPORTED_ARCH
${ALL_SAFESTACK_SUPPORTED_ARCH})
filter_available_targets(CFI_SUPPORTED_ARCH ${ALL_CFI_SUPPORTED_ARCH})
- filter_available_targets(SCUDO_SUPPORTED_ARCH ${ALL_SCUDO_SUPPORTED_ARCH})
filter_available_targets(SCUDO_STANDALONE_SUPPORTED_ARCH ${ALL_SCUDO_STANDALONE_SUPPORTED_ARCH})
filter_available_targets(XRAY_SUPPORTED_ARCH ${ALL_XRAY_SUPPORTED_ARCH})
filter_available_targets(SHADOWCALLSTACK_SUPPORTED_ARCH
endif()
if (MSVC)
+ # Allow setting clang-cl's /winsysroot flag.
+ set(LLVM_WINSYSROOT "" CACHE STRING
+ "If set, argument to clang-cl's /winsysroot")
+
+ if (LLVM_WINSYSROOT)
+ set(MSVC_DIA_SDK_DIR "${LLVM_WINSYSROOT}/DIA SDK" CACHE PATH
+ "Path to the DIA SDK")
+ else()
+ set(MSVC_DIA_SDK_DIR "$ENV{VSINSTALLDIR}DIA SDK" CACHE PATH
+ "Path to the DIA SDK")
+ endif()
+
# See if the DIA SDK is available and usable.
- set(MSVC_DIA_SDK_DIR "$ENV{VSINSTALLDIR}DIA SDK")
if (IS_DIRECTORY ${MSVC_DIA_SDK_DIR})
set(CAN_SYMBOLIZE 1)
else()
endif()
message(STATUS "Compiler-RT supported architectures: ${COMPILER_RT_SUPPORTED_ARCH}")
-set(ALL_SANITIZERS asan;dfsan;msan;hwasan;tsan;safestack;cfi;scudo;ubsan_minimal;gwp_asan)
+set(ALL_SANITIZERS asan;dfsan;msan;hwasan;tsan;safestack;cfi;scudo_standalone;ubsan_minimal;gwp_asan)
set(COMPILER_RT_SANITIZERS_TO_BUILD all CACHE STRING
"sanitizers to build if supported on the target (all;${ALL_SANITIZERS})")
list_replace(COMPILER_RT_SANITIZERS_TO_BUILD all "${ALL_SANITIZERS}")
# TODO: Add builtins support.
-if (CRT_SUPPORTED_ARCH AND OS_NAME MATCHES "Linux" AND NOT LLVM_USE_SANITIZER)
- set(COMPILER_RT_HAS_CRT TRUE)
-else()
- set(COMPILER_RT_HAS_CRT FALSE)
-endif()
-
if (COMPILER_RT_HAS_SANITIZER_COMMON AND DFSAN_SUPPORTED_ARCH AND
OS_NAME MATCHES "Linux")
set(COMPILER_RT_HAS_DFSAN TRUE)
endif()
if (COMPILER_RT_HAS_SANITIZER_COMMON AND HWASAN_SUPPORTED_ARCH AND
- OS_NAME MATCHES "Linux|Android")
+ OS_NAME MATCHES "Linux|Android|Fuchsia")
set(COMPILER_RT_HAS_HWASAN TRUE)
else()
set(COMPILER_RT_HAS_HWASAN FALSE)
set(COMPILER_RT_HAS_PROFILE FALSE)
endif()
-if (COMPILER_RT_HAS_SANITIZER_COMMON AND TSAN_SUPPORTED_ARCH AND
- OS_NAME MATCHES "Darwin|Linux|FreeBSD|Android|NetBSD")
- set(COMPILER_RT_HAS_TSAN TRUE)
+if (COMPILER_RT_HAS_SANITIZER_COMMON AND TSAN_SUPPORTED_ARCH)
+ if (OS_NAME MATCHES "Linux|Darwin|FreeBSD|NetBSD")
+ set(COMPILER_RT_HAS_TSAN TRUE)
+ elseif (OS_NAME MATCHES "Android" AND ANDROID_PLATFORM_LEVEL GREATER 23)
+ set(COMPILER_RT_HAS_TSAN TRUE)
+ else()
+ set(COMPILER_RT_HAS_TSAN FALSE)
+ endif()
else()
set(COMPILER_RT_HAS_TSAN FALSE)
endif()
+if (OS_NAME MATCHES "Linux|FreeBSD|Windows|NetBSD|SunOS")
+ set(COMPILER_RT_TSAN_HAS_STATIC_RUNTIME TRUE)
+else()
+ set(COMPILER_RT_TSAN_HAS_STATIC_RUNTIME FALSE)
+endif()
+
if (COMPILER_RT_HAS_SANITIZER_COMMON AND UBSAN_SUPPORTED_ARCH AND
OS_NAME MATCHES "Darwin|Linux|FreeBSD|NetBSD|Windows|Android|Fuchsia|SunOS")
set(COMPILER_RT_HAS_UBSAN TRUE)
endif()
#TODO(kostyak): add back Android & Fuchsia when the code settles a bit.
-if (SCUDO_STANDALONE_SUPPORTED_ARCH AND OS_NAME MATCHES "Linux" AND
+if (SCUDO_STANDALONE_SUPPORTED_ARCH AND
+ COMPILER_RT_BUILD_SANITIZERS AND
+ "scudo_standalone" IN_LIST COMPILER_RT_SANITIZERS_TO_BUILD AND
+ OS_NAME MATCHES "Linux" AND
COMPILER_RT_HAS_AUXV)
set(COMPILER_RT_HAS_SCUDO_STANDALONE TRUE)
else()
set(COMPILER_RT_HAS_SCUDO_STANDALONE FALSE)
endif()
-if (COMPILER_RT_HAS_SANITIZER_COMMON AND SCUDO_SUPPORTED_ARCH AND
- OS_NAME MATCHES "Linux|Android|Fuchsia")
- set(COMPILER_RT_HAS_SCUDO TRUE)
-else()
- set(COMPILER_RT_HAS_SCUDO FALSE)
-endif()
-
if (COMPILER_RT_HAS_SANITIZER_COMMON AND XRAY_SUPPORTED_ARCH AND
OS_NAME MATCHES "Darwin|Linux|FreeBSD|NetBSD|Fuchsia")
set(COMPILER_RT_HAS_XRAY TRUE)
# calling malloc on first use.
# TODO(hctim): Enable this on Android again. Looks like it's causing a SIGSEGV
# for Scudo and GWP-ASan, further testing needed.
-if (COMPILER_RT_HAS_SANITIZER_COMMON AND GWP_ASAN_SUPPORTED_ARCH AND
+if (GWP_ASAN_SUPPORTED_ARCH AND
+ COMPILER_RT_BUILD_GWP_ASAN AND
+ COMPILER_RT_BUILD_SANITIZERS AND
+ "gwp_asan" IN_LIST COMPILER_RT_SANITIZERS_TO_BUILD AND
OS_NAME MATCHES "Linux")
set(COMPILER_RT_HAS_GWP_ASAN TRUE)
else()
--- /dev/null
+include(BuiltinTests)
+include(CheckCSourceCompiles)
+
+# Make all the tests only check the compiler
+set(TEST_COMPILE_ONLY On)
+
+builtin_check_c_compiler_flag(-fPIC COMPILER_RT_HAS_FPIC_FLAG)
+builtin_check_c_compiler_flag(-std=c11 COMPILER_RT_HAS_STD_C11_FLAG)
+builtin_check_c_compiler_flag(-Wno-pedantic COMPILER_RT_HAS_WNO_PEDANTIC)
+builtin_check_c_compiler_flag(-fno-lto COMPILER_RT_HAS_FNO_LTO_FLAG)
+builtin_check_c_compiler_flag(-fno-profile-generate COMPILER_RT_HAS_FNO_PROFILE_GENERATE_FLAG)
+builtin_check_c_compiler_flag(-fno-profile-instr-generate COMPILER_RT_HAS_FNO_PROFILE_INSTR_GENERATE_FLAG)
+builtin_check_c_compiler_flag(-fno-profile-instr-use COMPILER_RT_HAS_FNO_PROFILE_INSTR_USE_FLAG)
+
+if(ANDROID)
+ set(OS_NAME "Android")
+else()
+ set(OS_NAME "${CMAKE_SYSTEM_NAME}")
+endif()
+
+set(ARM64 aarch64)
+set(ARM32 arm armhf)
+set(HEXAGON hexagon)
+set(X86 i386)
+set(X86_64 x86_64)
+set(LOONGARCH64 loongarch64)
+set(PPC32 powerpc powerpcspe)
+set(PPC64 powerpc64 powerpc64le)
+set(RISCV32 riscv32)
+set(RISCV64 riscv64)
+set(VE ve)
+
+set(ALL_CRT_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64} ${PPC32}
+ ${PPC64} ${RISCV32} ${RISCV64} ${VE} ${HEXAGON} ${LOONGARCH64})
+
+include(CompilerRTUtils)
+
+if(NOT APPLE)
+ if(COMPILER_RT_CRT_STANDALONE_BUILD)
+ test_targets()
+ endif()
+ # Architectures supported by compiler-rt crt library.
+ filter_available_targets(CRT_SUPPORTED_ARCH ${ALL_CRT_SUPPORTED_ARCH})
+ message(STATUS "Supported architectures for crt: ${CRT_SUPPORTED_ARCH}")
+endif()
+
+if (CRT_SUPPORTED_ARCH AND OS_NAME MATCHES "Linux" AND NOT LLVM_USE_SANITIZER)
+ set(COMPILER_RT_HAS_CRT TRUE)
+else()
+ set(COMPILER_RT_HAS_CRT FALSE)
+endif()
Sometimes it is necessary to restrict a test to a specific target or mark it as
an "expected fail" or XFAIL. This is normally achieved using ``REQUIRES:`` or
-``XFAIL:`` with a substring of LLVM's default target triple. Unfortunately, the
+``XFAIL:`` and the ``target=<target-triple>`` feature, typically with a regular
+expression matching an appropriate substring of the triple. Unfortunately, the
behaviour of this is somewhat quirky in compiler-rt. There are two main
pitfalls to avoid.
-The first pitfall is that these directives perform a substring match on the
-triple and as such ``XFAIL: mips`` affects more triples than expected. For
-example, ``mips-linux-gnu``, ``mipsel-linux-gnu``, ``mips64-linux-gnu``, and
-``mips64el-linux-gnu`` will all match a ``XFAIL: mips`` directive. Including a
-trailing ``-`` such as in ``XFAIL: mips-`` can help to mitigate this quirk but
-even that has issues as described below.
+The first pitfall is that these regular expressions may inadvertently match
+more triples than expected. For example, ``XFAIL: target=mips{{.*}}`` matches
+``mips-linux-gnu``, ``mipsel-linux-gnu``, ``mips64-linux-gnu``, and
+``mips64el-linux-gnu``. Including a trailing ``-`` such as in
+``XFAIL: target=mips-{{.*}}`` can help to mitigate this quirk but even that has
+issues as described below.
The second pitfall is that the default target triple is often inappropriate for
compiler-rt tests since compiler-rt tests may be compiled for multiple targets.
For example, a typical build on an ``x86_64-linux-gnu`` host will often run the
-tests for both x86_64 and i386. In this situation ``XFAIL: x86_64`` will mark
-both the x86_64 and i386 tests as an expected failure while ``XFAIL: i386``
-will have no effect at all.
+tests for both x86_64 and i386. In this situation ``XFAIL: target=x86_64{{{.*}}``
+will mark both the x86_64 and i386 tests as an expected failure while
+``XFAIL: target=i386{{.*}}`` will have no effect at all.
To remedy both pitfalls, compiler-rt tests provide a feature string which can
be used to specify a single target. This string is of the form
if (COMPILER_RT_BUILD_MEMPROF)
set(MEMPROF_HEADERS
sanitizer/memprof_interface.h
+ profile/MemProfData.inc
)
endif(COMPILER_RT_BUILD_MEMPROF)
)
endif(COMPILER_RT_BUILD_XRAY)
+if (COMPILER_RT_BUILD_ORC)
+ set(ORC_HEADERS
+ orc_rt/c_api.h
+ )
+endif(COMPILER_RT_BUILD_ORC)
+
if (COMPILER_RT_BUILD_PROFILE)
set(PROFILE_HEADERS
profile/InstrProfData.inc
${FUZZER_HEADERS}
${MEMPROF_HEADERS}
${XRAY_HEADERS}
+ ${ORC_HEADERS}
${PROFILE_HEADERS})
set(output_dir ${COMPILER_RT_OUTPUT_DIR}/include)
COMPONENT compiler-rt-headers
PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ
DESTINATION ${COMPILER_RT_INSTALL_INCLUDE_DIR}/fuzzer)
+# Install memprof headers.
+if (COMPILER_RT_BUILD_MEMPROF)
+ install(FILES sanitizer/memprof_interface.h
+ COMPONENT compiler-rt-headers
+ PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ
+ DESTINATION ${COMPILER_RT_INSTALL_INCLUDE_DIR}/sanitizer)
+endif(COMPILER_RT_BUILD_MEMPROF)
# Install xray headers.
install(FILES ${XRAY_HEADERS}
COMPONENT compiler-rt-headers
PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ
DESTINATION ${COMPILER_RT_INSTALL_INCLUDE_DIR}/xray)
+# Install ORC headers.
+install(FILES ${ORC_HEADERS}
+ COMPONENT compiler-rt-headers
+ PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ
+ DESTINATION ${COMPILER_RT_INSTALL_INCLUDE_DIR}/orc)
# Install profile headers.
install(FILES ${PROFILE_HEADERS}
COMPONENT compiler-rt-headers
--- /dev/null
+/*===- c_api.h - C API for the ORC runtime ------------------------*- C -*-===*\
+|* *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM *|
+|* Exceptions. *|
+|* See https://llvm.org/LICENSE.txt for license information. *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception *|
+|* *|
+|*===----------------------------------------------------------------------===*|
+|* *|
+|* This file defines the C API for the ORC runtime *|
+|* *|
+\*===----------------------------------------------------------------------===*/
+
+#ifndef ORC_RT_C_API_H
+#define ORC_RT_C_API_H
+
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+/* Helper to suppress strict prototype warnings. */
+#ifdef __clang__
+#define ORC_RT_C_STRICT_PROTOTYPES_BEGIN \
+ _Pragma("clang diagnostic push") \
+ _Pragma("clang diagnostic error \"-Wstrict-prototypes\"")
+#define ORC_RT_C_STRICT_PROTOTYPES_END _Pragma("clang diagnostic pop")
+#else
+#define ORC_RT_C_STRICT_PROTOTYPES_BEGIN
+#define ORC_RT_C_STRICT_PROTOTYPES_END
+#endif
+
+/* Helper to wrap C code for C++ */
+#ifdef __cplusplus
+#define ORC_RT_C_EXTERN_C_BEGIN \
+ extern "C" { \
+ ORC_RT_C_STRICT_PROTOTYPES_BEGIN
+#define ORC_RT_C_EXTERN_C_END \
+ ORC_RT_C_STRICT_PROTOTYPES_END \
+ }
+#else
+#define ORC_RT_C_EXTERN_C_BEGIN ORC_RT_C_STRICT_PROTOTYPES_BEGIN
+#define ORC_RT_C_EXTERN_C_END ORC_RT_C_STRICT_PROTOTYPES_END
+#endif
+
+ORC_RT_C_EXTERN_C_BEGIN
+
+typedef union {
+ char *ValuePtr;
+ char Value[sizeof(char *)];
+} __orc_rt_CWrapperFunctionResultDataUnion;
+
+/**
+ * __orc_rt_CWrapperFunctionResult is a kind of C-SmallVector with an
+ * out-of-band error state.
+ *
+ * If Size == 0 and Data.ValuePtr is non-zero then the value is in the
+ * 'out-of-band error' state, and Data.ValuePtr points at a malloc-allocated,
+ * null-terminated string error message.
+ *
+ * If Size <= sizeof(__orc_rt_CWrapperFunctionResultData) then the value is in
+ * the 'small' state and the content is held in the first Size bytes of
+ * Data.Value.
+ *
+ * If Size > sizeof(OrtRTCWrapperFunctionResultData) then the value is in the
+ * 'large' state and the content is held in the first Size bytes of the
+ * memory pointed to by Data.ValuePtr. This memory must have been allocated by
+ * malloc, and will be freed with free when this value is destroyed.
+ */
+typedef struct {
+ __orc_rt_CWrapperFunctionResultDataUnion Data;
+ size_t Size;
+} __orc_rt_CWrapperFunctionResult;
+
+typedef struct __orc_rt_CSharedOpaqueJITProcessControl
+ *__orc_rt_SharedJITProcessControlRef;
+
+/**
+ * Zero-initialize an __orc_rt_CWrapperFunctionResult.
+ */
+static inline void
+__orc_rt_CWrapperFunctionResultInit(__orc_rt_CWrapperFunctionResult *R) {
+ R->Size = 0;
+ R->Data.ValuePtr = 0;
+}
+
+/**
+ * Create an __orc_rt_CWrapperFunctionResult with an uninitialized buffer of
+ * size Size. The buffer is returned via the DataPtr argument.
+ */
+static inline __orc_rt_CWrapperFunctionResult
+__orc_rt_CWrapperFunctionResultAllocate(size_t Size) {
+ __orc_rt_CWrapperFunctionResult R;
+ R.Size = Size;
+ // If Size is 0 ValuePtr must be 0 or it is considered an out-of-band error.
+ R.Data.ValuePtr = 0;
+ if (Size > sizeof(R.Data.Value))
+ R.Data.ValuePtr = (char *)malloc(Size);
+ return R;
+}
+
+/**
+ * Create an __orc_rt_WrapperFunctionResult from the given data range.
+ */
+static inline __orc_rt_CWrapperFunctionResult
+__orc_rt_CreateCWrapperFunctionResultFromRange(const char *Data, size_t Size) {
+ __orc_rt_CWrapperFunctionResult R;
+ R.Size = Size;
+ if (R.Size > sizeof(R.Data.Value)) {
+ char *Tmp = (char *)malloc(Size);
+ memcpy(Tmp, Data, Size);
+ R.Data.ValuePtr = Tmp;
+ } else
+ memcpy(R.Data.Value, Data, Size);
+ return R;
+}
+
+/**
+ * Create an __orc_rt_CWrapperFunctionResult by copying the given string,
+ * including the null-terminator.
+ *
+ * This function copies the input string. The client is responsible for freeing
+ * the ErrMsg arg.
+ */
+static inline __orc_rt_CWrapperFunctionResult
+__orc_rt_CreateCWrapperFunctionResultFromString(const char *Source) {
+ return __orc_rt_CreateCWrapperFunctionResultFromRange(Source,
+ strlen(Source) + 1);
+}
+
+/**
+ * Create an __orc_rt_CWrapperFunctionResult representing an out-of-band
+ * error.
+ *
+ * This function copies the input string. The client is responsible for freeing
+ * the ErrMsg arg.
+ */
+static inline __orc_rt_CWrapperFunctionResult
+__orc_rt_CreateCWrapperFunctionResultFromOutOfBandError(const char *ErrMsg) {
+ __orc_rt_CWrapperFunctionResult R;
+ R.Size = 0;
+ char *Tmp = (char *)malloc(strlen(ErrMsg) + 1);
+ strcpy(Tmp, ErrMsg);
+ R.Data.ValuePtr = Tmp;
+ return R;
+}
+
+/**
+ * This should be called to destroy __orc_rt_CWrapperFunctionResult values
+ * regardless of their state.
+ */
+static inline void
+__orc_rt_DisposeCWrapperFunctionResult(__orc_rt_CWrapperFunctionResult *R) {
+ if (R->Size > sizeof(R->Data.Value) ||
+ (R->Size == 0 && R->Data.ValuePtr))
+ free(R->Data.ValuePtr);
+}
+
+/**
+ * Get a pointer to the data contained in the given
+ * __orc_rt_CWrapperFunctionResult.
+ */
+static inline char *
+__orc_rt_CWrapperFunctionResultData(__orc_rt_CWrapperFunctionResult *R) {
+ assert((R->Size != 0 || R->Data.ValuePtr == NULL) &&
+ "Cannot get data for out-of-band error value");
+ return R->Size > sizeof(R->Data.Value) ? R->Data.ValuePtr : R->Data.Value;
+}
+
+/**
+ * Safely get the size of the given __orc_rt_CWrapperFunctionResult.
+ *
+ * Asserts that we're not trying to access the size of an error value.
+ */
+static inline size_t
+__orc_rt_CWrapperFunctionResultSize(const __orc_rt_CWrapperFunctionResult *R) {
+ assert((R->Size != 0 || R->Data.ValuePtr == NULL) &&
+ "Cannot get size for out-of-band error value");
+ return R->Size;
+}
+
+/**
+ * Returns 1 if this value is equivalent to a value just initialized by
+ * __orc_rt_CWrapperFunctionResultInit, 0 otherwise.
+ */
+static inline size_t
+__orc_rt_CWrapperFunctionResultEmpty(const __orc_rt_CWrapperFunctionResult *R) {
+ return R->Size == 0 && R->Data.ValuePtr == 0;
+}
+
+/**
+ * Returns a pointer to the out-of-band error string for this
+ * __orc_rt_CWrapperFunctionResult, or null if there is no error.
+ *
+ * The __orc_rt_CWrapperFunctionResult retains ownership of the error
+ * string, so it should be copied if the caller wishes to preserve it.
+ */
+static inline const char *__orc_rt_CWrapperFunctionResultGetOutOfBandError(
+ const __orc_rt_CWrapperFunctionResult *R) {
+ return R->Size == 0 ? R->Data.ValuePtr : 0;
+}
+
+ORC_RT_C_EXTERN_C_END
+
+#endif /* ORC_RT_C_API_H */
INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
Inc->getHash()->getZExtValue()))
-INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt64PtrTy(Ctx), CounterPtr, \
- ConstantExpr::getBitCast(CounterPtr, \
- llvm::Type::getInt64PtrTy(Ctx)))
+INSTR_PROF_DATA(const IntPtrT, IntPtrTy, CounterPtr, RelativeCounterPtr)
/* This is used to map function pointers for the indirect call targets to
* function name hashes during the conversion from raw to merged profile
* data.
INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic())
INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version())
INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL))
+/* FIXME: A more accurate name is NumData */
INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize)
INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters)
+/* FIXME: A more accurate name is NumCounters */
INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize)
INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters)
INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize)
-INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin)
+INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta,
+ (uintptr_t)CountersBegin - (uintptr_t)DataBegin)
INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)
INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last)
#undef INSTR_PROF_RAW_HEADER
(uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \
(uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129
+/* FIXME: Please remedy the fixme in the header before bumping the version. */
/* Raw profile format version (start from 1). */
-#define INSTR_PROF_RAW_VERSION 7
+#define INSTR_PROF_RAW_VERSION 8
/* Indexed profile format version (start from 1). */
-#define INSTR_PROF_INDEX_VERSION 7
+#define INSTR_PROF_INDEX_VERSION 9
/* Coverage mapping format version (start from 0). */
#define INSTR_PROF_COVMAP_VERSION 5
/* Profile version is always of type uint64_t. Reserve the upper 8 bits in the
* version for other variants of profile. We set the lowest bit of the upper 8
- * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentaiton
+ * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentation
* generated profile, and 0 if this is a Clang FE generated profile.
* 1 in bit 57 indicates there are context-sensitive records in the profile.
+ * The 59th bit indicates whether to use debug info to correlate profiles.
+ * The 60th bit indicates single byte coverage instrumentation.
+ * The 61st bit indicates function entry instrumentation only.
+ * The 62nd bit indicates whether memory profile information is present.
*/
#define VARIANT_MASKS_ALL 0xff00000000000000ULL
#define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL)
#define VARIANT_MASK_IR_PROF (0x1ULL << 56)
#define VARIANT_MASK_CSIR_PROF (0x1ULL << 57)
#define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58)
+#define VARIANT_MASK_DBG_CORRELATE (0x1ULL << 59)
+#define VARIANT_MASK_BYTE_COVERAGE (0x1ULL << 60)
+#define VARIANT_MASK_FUNCTION_ENTRY_ONLY (0x1ULL << 61)
+#define VARIANT_MASK_MEMPROF (0x1ULL << 62)
#define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version
#define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime
#define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias
--- /dev/null
+/*===-- MemEntryDef.inc - MemProf profiling runtime macros -*- C++ -*-======== *\
+|*
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+|* See https://llvm.org/LICENSE.txt for license information.
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+|*
+\*===----------------------------------------------------------------------===*/
+/*
+ * This file defines the macros for memprof profiling data structures.
+ * Eg. usage to define the memprof meminfoblock struct:
+ *
+ * struct MemInfoBlock {
+ * #define MIBEntryDef(NameTag, Name, Type) Type Name;
+ * #include MIBEntryDef.inc
+ * #undef MIBEntryDef
+ * };
+ *
+ * This file has two identical copies. The primary copy lives in LLVM and
+ * the other one sits in compiler-rt/include/profile directory. To make changes
+ * in this file, first modify the primary copy and copy it over to compiler-rt.
+ * Testing of any change in this file can start only after the two copies are
+ * synced up.
+ *
+\*===----------------------------------------------------------------------===*/
+#ifndef MIBEntryDef
+#define MIBEntryDef(NameTag, Name, Type)
+#endif
+
+MIBEntryDef(AllocCount = 1, AllocCount, uint32_t)
+MIBEntryDef(TotalAccessCount = 2, TotalAccessCount, uint64_t)
+MIBEntryDef(MinAccessCount = 3, MinAccessCount, uint64_t)
+MIBEntryDef(MaxAccessCount = 4, MaxAccessCount, uint64_t)
+MIBEntryDef(TotalSize = 5, TotalSize, uint64_t)
+MIBEntryDef(MinSize = 6, MinSize, uint32_t)
+MIBEntryDef(MaxSize = 7, MaxSize, uint32_t)
+MIBEntryDef(AllocTimestamp = 8, AllocTimestamp, uint32_t)
+MIBEntryDef(DeallocTimestamp = 9, DeallocTimestamp, uint32_t)
+MIBEntryDef(TotalLifetime = 10, TotalLifetime, uint64_t)
+MIBEntryDef(MinLifetime = 11, MinLifetime, uint32_t)
+MIBEntryDef(MaxLifetime = 12, MaxLifetime, uint32_t)
+MIBEntryDef(AllocCpuId = 13, AllocCpuId, uint32_t)
+MIBEntryDef(DeallocCpuId = 14, DeallocCpuId, uint32_t)
+MIBEntryDef(NumMigratedCpu = 15, NumMigratedCpu, uint32_t)
+MIBEntryDef(NumLifetimeOverlaps = 16, NumLifetimeOverlaps, uint32_t)
+MIBEntryDef(NumSameAllocCpu = 17, NumSameAllocCpu, uint32_t)
+MIBEntryDef(NumSameDeallocCpu = 18, NumSameDeallocCpu, uint32_t)
+MIBEntryDef(DataTypeId = 19, DataTypeId, uint64_t)
+MIBEntryDef(TotalAccessDensity = 20, TotalAccessDensity, uint64_t)
+MIBEntryDef(MinAccessDensity = 21, MinAccessDensity, uint32_t)
+MIBEntryDef(MaxAccessDensity = 22, MaxAccessDensity, uint32_t)
+MIBEntryDef(TotalLifetimeAccessDensity = 23, TotalLifetimeAccessDensity, uint64_t)
+MIBEntryDef(MinLifetimeAccessDensity = 24, MinLifetimeAccessDensity, uint32_t)
+MIBEntryDef(MaxLifetimeAccessDensity = 25, MaxLifetimeAccessDensity, uint32_t)
--- /dev/null
+#ifndef MEMPROF_DATA_INC
+#define MEMPROF_DATA_INC
+/*===-- MemProfData.inc - MemProf profiling runtime structures -*- C++ -*-=== *\
+|*
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+|* See https://llvm.org/LICENSE.txt for license information.
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+|*
+\*===----------------------------------------------------------------------===*/
+/*
+ * This is the main file that defines all the data structure, signature,
+ * constant literals that are shared across profiling runtime library,
+ * and host tools (reader/writer).
+ *
+ * This file has two identical copies. The primary copy lives in LLVM and
+ * the other one sits in compiler-rt/include/profile directory. To make changes
+ * in this file, first modify the primary copy and copy it over to compiler-rt.
+ * Testing of any change in this file can start only after the two copies are
+ * synced up.
+ *
+\*===----------------------------------------------------------------------===*/
+
+#ifdef _MSC_VER
+#define PACKED(...) __pragma(pack(push,1)) __VA_ARGS__ __pragma(pack(pop))
+#else
+#define PACKED(...) __VA_ARGS__ __attribute__((__packed__))
+#endif
+
+// A 64-bit magic number to uniquely identify the raw binary memprof profile file.
+#define MEMPROF_RAW_MAGIC_64 \
+ ((uint64_t)255 << 56 | (uint64_t)'m' << 48 | (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | \
+ (uint64_t)'o' << 24 | (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129)
+
+// The version number of the raw binary format.
+#define MEMPROF_RAW_VERSION 2ULL
+
+namespace llvm {
+namespace memprof {
+// A struct describing the header used for the raw binary memprof profile format.
+PACKED(struct Header {
+ uint64_t Magic;
+ uint64_t Version;
+ uint64_t TotalSize;
+ uint64_t SegmentOffset;
+ uint64_t MIBOffset;
+ uint64_t StackOffset;
+});
+
+
+// A struct describing the information necessary to describe a /proc/maps
+// segment entry for a particular binary/library identified by its build id.
+PACKED(struct SegmentEntry {
+ uint64_t Start;
+ uint64_t End;
+ uint64_t Offset;
+ // This field is unused until sanitizer procmaps support for build ids for
+ // Linux-Elf is implemented.
+ uint8_t BuildId[32] = {0};
+
+ SegmentEntry(uint64_t S, uint64_t E, uint64_t O) :
+ Start(S), End(E), Offset(O) {}
+
+ SegmentEntry(const SegmentEntry& S) {
+ Start = S.Start;
+ End = S.End;
+ Offset = S.Offset;
+ }
+
+ SegmentEntry& operator=(const SegmentEntry& S) {
+ Start = S.Start;
+ End = S.End;
+ Offset = S.Offset;
+ return *this;
+ }
+
+ bool operator==(const SegmentEntry& S) const {
+ return Start == S.Start &&
+ End == S.End &&
+ Offset == S.Offset;
+ }
+});
+
+// Packed struct definition for MSVC. We can't use the PACKED macro defined in
+// MemProfData.inc since it would mean we are embedding a directive (the
+// #include for MIBEntryDef) into the macros which is undefined behaviour.
+#ifdef _MSC_VER
+__pragma(pack(push,1))
+#endif
+
+// A struct representing the heap allocation characteristics of a particular
+// runtime context. This struct is shared between the compiler-rt runtime and
+// the raw profile reader. The indexed format uses a separate, self-describing
+// backwards compatible format.
+struct MemInfoBlock{
+
+#define MIBEntryDef(NameTag, Name, Type) Type Name;
+#include "MIBEntryDef.inc"
+#undef MIBEntryDef
+
+bool operator==(const MemInfoBlock& Other) const {
+ bool IsEqual = true;
+#define MIBEntryDef(NameTag, Name, Type) \
+ IsEqual = (IsEqual && Name == Other.Name);
+#include "MIBEntryDef.inc"
+#undef MIBEntryDef
+ return IsEqual;
+}
+
+MemInfoBlock() {
+#define MIBEntryDef(NameTag, Name, Type) Name = Type();
+#include "MIBEntryDef.inc"
+#undef MIBEntryDef
+}
+
+MemInfoBlock(uint32_t Size, uint64_t AccessCount, uint32_t AllocTs,
+ uint32_t DeallocTs, uint32_t AllocCpu, uint32_t DeallocCpu)
+ : MemInfoBlock() {
+ AllocCount = 1U;
+ TotalAccessCount = AccessCount;
+ MinAccessCount = AccessCount;
+ MaxAccessCount = AccessCount;
+ TotalSize = Size;
+ MinSize = Size;
+ MaxSize = Size;
+ AllocTimestamp = AllocTs;
+ DeallocTimestamp = DeallocTs;
+ TotalLifetime = DeallocTimestamp - AllocTimestamp;
+ MinLifetime = TotalLifetime;
+ MaxLifetime = TotalLifetime;
+ // Access density is accesses per byte. Multiply by 100 to include the
+ // fractional part.
+ TotalAccessDensity = AccessCount * 100 / Size;
+ MinAccessDensity = TotalAccessDensity;
+ MaxAccessDensity = TotalAccessDensity;
+ // Lifetime access density is the access density per second of lifetime.
+ // Multiply by 1000 to convert denominator lifetime to seconds (using a
+ // minimum lifetime of 1ms to avoid divide by 0. Do the multiplication first
+ // to reduce truncations to 0.
+ TotalLifetimeAccessDensity =
+ TotalAccessDensity * 1000 / (TotalLifetime ? TotalLifetime : 1);
+ MinLifetimeAccessDensity = TotalLifetimeAccessDensity;
+ MaxLifetimeAccessDensity = TotalLifetimeAccessDensity;
+ AllocCpuId = AllocCpu;
+ DeallocCpuId = DeallocCpu;
+ NumMigratedCpu = AllocCpuId != DeallocCpuId;
+}
+
+void Merge(const MemInfoBlock &newMIB) {
+ AllocCount += newMIB.AllocCount;
+
+ TotalAccessCount += newMIB.TotalAccessCount;
+ MinAccessCount = newMIB.MinAccessCount < MinAccessCount ? newMIB.MinAccessCount : MinAccessCount;
+ MaxAccessCount = newMIB.MaxAccessCount > MaxAccessCount ? newMIB.MaxAccessCount : MaxAccessCount;
+
+ TotalSize += newMIB.TotalSize;
+ MinSize = newMIB.MinSize < MinSize ? newMIB.MinSize : MinSize;
+ MaxSize = newMIB.MaxSize > MaxSize ? newMIB.MaxSize : MaxSize;
+
+ TotalLifetime += newMIB.TotalLifetime;
+ MinLifetime = newMIB.MinLifetime < MinLifetime ? newMIB.MinLifetime : MinLifetime;
+ MaxLifetime = newMIB.MaxLifetime > MaxLifetime ? newMIB.MaxLifetime : MaxLifetime;
+
+ TotalAccessDensity += newMIB.TotalAccessDensity;
+ MinAccessDensity = newMIB.MinAccessDensity < MinAccessDensity
+ ? newMIB.MinAccessDensity
+ : MinAccessDensity;
+ MaxAccessDensity = newMIB.MaxAccessDensity > MaxAccessDensity
+ ? newMIB.MaxAccessDensity
+ : MaxAccessDensity;
+
+ TotalLifetimeAccessDensity += newMIB.TotalLifetimeAccessDensity;
+ MinLifetimeAccessDensity =
+ newMIB.MinLifetimeAccessDensity < MinLifetimeAccessDensity
+ ? newMIB.MinLifetimeAccessDensity
+ : MinLifetimeAccessDensity;
+ MaxLifetimeAccessDensity =
+ newMIB.MaxLifetimeAccessDensity > MaxLifetimeAccessDensity
+ ? newMIB.MaxLifetimeAccessDensity
+ : MaxLifetimeAccessDensity;
+
+ // We know newMIB was deallocated later, so just need to check if it was
+ // allocated before last one deallocated.
+ NumLifetimeOverlaps += newMIB.AllocTimestamp < DeallocTimestamp;
+ AllocTimestamp = newMIB.AllocTimestamp;
+ DeallocTimestamp = newMIB.DeallocTimestamp;
+
+ NumSameAllocCpu += AllocCpuId == newMIB.AllocCpuId;
+ NumSameDeallocCpu += DeallocCpuId == newMIB.DeallocCpuId;
+ AllocCpuId = newMIB.AllocCpuId;
+ DeallocCpuId = newMIB.DeallocCpuId;
+}
+
+#ifdef _MSC_VER
+} __pragma(pack(pop));
+#else
+} __attribute__((__packed__));
+#endif
+
+} // namespace memprof
+} // namespace llvm
+
+#endif
void __asan_handle_no_return(void);
/// Update allocation stack trace for the given allocation to the current stack
-/// trace. Returns 1 if successfull, 0 if not.
+/// trace. Returns 1 if successful, 0 if not.
int __asan_update_allocation_context(void* addr);
#ifdef __cplusplus
// Enable sandbox support in sanitizer coverage.
int coverage_sandboxed;
// File descriptor to write coverage data to. If -1 is passed, a file will
- // be pre-opened by __sanitizer_sandobx_on_notify(). This field has no
+ // be pre-opened by __sanitizer_sandbox_on_notify(). This field has no
// effect if coverage_sandboxed == 0.
intptr_t coverage_fd;
// If non-zero, split the coverage data into well-formed blocks. This is
const void *old_mid,
const void *new_mid);
+/// Similar to <c>__sanitizer_annotate_contiguous_container</c>.
+///
+/// Annotates the current state of a contiguous container memory,
+/// such as <c>std::deque</c>'s single chunk, when the boundries are moved.
+///
+/// A contiguous chunk is a chunk that keeps all of its elements
+/// in a contiguous region of memory. The container owns the region of memory
+/// <c>[storage_beg, storage_end)</c>; the memory <c>[container_beg,
+/// container_end)</c> is used to store the current elements, and the memory
+/// <c>[storage_beg, container_beg), [container_end, storage_end)</c> is
+/// reserved for future elements (<c>storage_beg <= container_beg <=
+/// container_end <= storage_end</c>). For example, in <c> std::deque </c>:
+/// - chunk with a frist deques element will have container_beg equal to address
+/// of the first element.
+/// - in every next chunk with elements, true is <c> container_beg ==
+/// storage_beg </c>.
+///
+/// Argument requirements:
+/// During unpoisoning memory of empty container (before first element is
+/// added):
+/// - old_container_beg_p == old_container_end_p
+/// During poisoning after last element was removed:
+/// - new_container_beg_p == new_container_end_p
+/// \param storage_beg Beginning of memory region.
+/// \param storage_end End of memory region.
+/// \param old_container_beg Old beginning of used region.
+/// \param old_container_end End of used region.
+/// \param new_container_beg New beginning of used region.
+/// \param new_container_end New end of used region.
+void __sanitizer_annotate_double_ended_contiguous_container(
+ const void *storage_beg, const void *storage_end,
+ const void *old_container_beg, const void *old_container_end,
+ const void *new_container_beg, const void *new_container_end);
+
/// Returns true if the contiguous container <c>[beg, end)</c> is properly
/// poisoned.
///
int __sanitizer_verify_contiguous_container(const void *beg, const void *mid,
const void *end);
+/// Returns true if the double ended contiguous
+/// container <c>[storage_beg, storage_end)</c> is properly poisoned.
+///
+/// Proper poisoning could occur, for example, with
+/// <c>__sanitizer_annotate_double_ended_contiguous_container</c>), that is, if
+/// <c>[storage_beg, container_beg)</c> is not addressable, <c>[container_beg,
+/// container_end)</c> is addressable and <c>[container_end, end)</c> is
+/// unaddressable. Full verification requires O (<c>storage_end -
+/// storage_beg</c>) time; this function tries to avoid such complexity by
+/// touching only parts of the container around <c><i>storage_beg</i></c>,
+/// <c><i>container_beg</i></c>, <c><i>container_end</i></c>, and
+/// <c><i>storage_end</i></c>.
+///
+/// \param storage_beg Beginning of memory region.
+/// \param container_beg Beginning of used region.
+/// \param container_end End of used region.
+/// \param storage_end End of memory region.
+///
+/// \returns True if the double-ended contiguous container <c>[storage_beg,
+/// container_beg, container_end, end)</c> is properly poisoned - only
+/// [container_beg; container_end) is addressable.
+int __sanitizer_verify_double_ended_contiguous_container(
+ const void *storage_beg, const void *container_beg,
+ const void *container_end, const void *storage_end);
+
/// Similar to <c>__sanitizer_verify_contiguous_container()</c> but also
/// returns the address of the first improperly poisoned byte.
///
const void *mid,
const void *end);
+/// returns the address of the first improperly poisoned byte.
+///
+/// Returns NULL if the area is poisoned properly.
+///
+/// \param storage_beg Beginning of memory region.
+/// \param container_beg Beginning of used region.
+/// \param container_end End of used region.
+/// \param storage_end End of memory region.
+///
+/// \returns The bad address or NULL.
+const void *__sanitizer_double_ended_contiguous_container_find_bad_address(
+ const void *storage_beg, const void *container_beg,
+ const void *container_end, const void *storage_end);
+
/// Prints the stack trace leading to this call (useful for calling from the
/// debugger).
void __sanitizer_print_stack_trace(void);
// Same as __sanitizer_symbolize_pc, but for data section (i.e. globals).
void __sanitizer_symbolize_global(void *data_ptr, const char *fmt,
char *out_buf, size_t out_buf_size);
+// Determine the return address.
+#if !defined(_MSC_VER) || defined(__clang__)
+#define __sanitizer_return_address() \
+ __builtin_extract_return_addr(__builtin_return_address(0))
+#else
+extern "C" void *_ReturnAddress(void);
+#pragma intrinsic(_ReturnAddress)
+#define __sanitizer_return_address() _ReturnAddress()
+#endif
/// Sets the callback to be called immediately before death on error.
///
/// Signature of the callback argument to dfsan_set_write_callback().
typedef void (*dfsan_write_callback_t)(int fd, const void *buf, size_t count);
+/// Signature of the callback argument to dfsan_set_conditional_callback().
+typedef void (*dfsan_conditional_callback_t)(dfsan_label label,
+ dfsan_origin origin);
+
+/// Signature of the callback argument to dfsan_set_reaches_function_callback().
+/// The description is intended to hold the name of the variable.
+typedef void (*dfsan_reaches_function_callback_t)(dfsan_label label,
+ dfsan_origin origin,
+ const char *file,
+ unsigned int line,
+ const char *function);
+
/// Computes the union of \c l1 and \c l2, resulting in a union label.
dfsan_label dfsan_union(dfsan_label l1, dfsan_label l2);
/// Retrieves the label associated with the data at the given address.
dfsan_label dfsan_read_label(const void *addr, size_t size);
+/// Return the origin associated with the first taint byte in the size bytes
+/// from the address addr.
+dfsan_origin dfsan_read_origin_of_first_taint(const void *addr, size_t size);
+
/// Returns whether the given label label contains the label elem.
int dfsan_has_label(dfsan_label label, dfsan_label elem);
/// callback executes. Pass in NULL to remove any callback.
void dfsan_set_write_callback(dfsan_write_callback_t labeled_write_callback);
+/// Sets a callback to be invoked on any conditional expressions which have a
+/// taint label set. This can be used to find where tainted data influences
+/// the behavior of the program.
+/// These callbacks will only be added when -dfsan-conditional-callbacks=true.
+void dfsan_set_conditional_callback(dfsan_conditional_callback_t callback);
+
+/// Conditional expressions occur during signal handlers.
+/// Making callbacks that handle signals well is tricky, so when
+/// -dfsan-conditional-callbacks=true, conditional expressions used in signal
+/// handlers will add the labels they see into a global (bitwise-or together).
+/// This function returns all label bits seen in signal handler conditions.
+dfsan_label dfsan_get_labels_in_signal_conditional();
+
+/// Sets a callback to be invoked when tainted data reaches a function.
+/// This could occur at function entry, or at a load instruction.
+/// These callbacks will only be added if -dfsan-reaches-function-callbacks=1.
+void dfsan_set_reaches_function_callback(
+ dfsan_reaches_function_callback_t callback);
+
+/// Making callbacks that handle signals well is tricky, so when
+/// -dfsan-reaches-function-callbacks=true, functions reached in signal
+/// handlers will add the labels they see into a global (bitwise-or together).
+/// This function returns all label bits seen during signal handlers.
+dfsan_label dfsan_get_labels_in_signal_reaches_function();
+
/// Interceptor hooks.
/// Whenever a dfsan's custom function is called the corresponding
/// hook is called it non-zero. The hooks should be defined by the user.
/// prints description at the beginning of the trace. If origin tracking is not
/// on, or the address is not labeled, it prints nothing.
void dfsan_print_origin_trace(const void *addr, const char *description);
+/// As above, but use an origin id from dfsan_get_origin() instead of address.
+/// Does not include header line with taint label and address information.
+void dfsan_print_origin_id_trace(dfsan_origin origin);
/// Prints the origin trace of the label at the address \p addr to a
/// pre-allocated output buffer. If origin tracking is not on, or the address is
/// return value is not less than \p out_buf_size.
size_t dfsan_sprint_origin_trace(const void *addr, const char *description,
char *out_buf, size_t out_buf_size);
+/// As above, but use an origin id from dfsan_get_origin() instead of address.
+/// Does not include header line with taint label and address information.
+size_t dfsan_sprint_origin_id_trace(dfsan_origin origin, char *out_buf,
+ size_t out_buf_size);
/// Prints the stack trace leading to this call to a pre-allocated output
/// buffer.
#ifdef __cplusplus
} // extern "C"
-template <typename T>
-void dfsan_set_label(dfsan_label label, T &data) { // NOLINT
+template <typename T> void dfsan_set_label(dfsan_label label, T &data) {
dfsan_set_label(label, (void *)&data, sizeof(T));
}
#ifndef SANITIZER_LINUX_SYSCALL_HOOKS_H
#define SANITIZER_LINUX_SYSCALL_HOOKS_H
-#define __sanitizer_syscall_pre_time(tloc) \
+#define __sanitizer_syscall_pre_time(tloc) \
__sanitizer_syscall_pre_impl_time((long)(tloc))
-#define __sanitizer_syscall_post_time(res, tloc) \
+#define __sanitizer_syscall_post_time(res, tloc) \
__sanitizer_syscall_post_impl_time(res, (long)(tloc))
-#define __sanitizer_syscall_pre_stime(tptr) \
+#define __sanitizer_syscall_pre_stime(tptr) \
__sanitizer_syscall_pre_impl_stime((long)(tptr))
-#define __sanitizer_syscall_post_stime(res, tptr) \
+#define __sanitizer_syscall_post_stime(res, tptr) \
__sanitizer_syscall_post_impl_stime(res, (long)(tptr))
-#define __sanitizer_syscall_pre_gettimeofday(tv, tz) \
+#define __sanitizer_syscall_pre_gettimeofday(tv, tz) \
__sanitizer_syscall_pre_impl_gettimeofday((long)(tv), (long)(tz))
-#define __sanitizer_syscall_post_gettimeofday(res, tv, tz) \
+#define __sanitizer_syscall_post_gettimeofday(res, tv, tz) \
__sanitizer_syscall_post_impl_gettimeofday(res, (long)(tv), (long)(tz))
-#define __sanitizer_syscall_pre_settimeofday(tv, tz) \
+#define __sanitizer_syscall_pre_settimeofday(tv, tz) \
__sanitizer_syscall_pre_impl_settimeofday((long)(tv), (long)(tz))
-#define __sanitizer_syscall_post_settimeofday(res, tv, tz) \
+#define __sanitizer_syscall_post_settimeofday(res, tv, tz) \
__sanitizer_syscall_post_impl_settimeofday(res, (long)(tv), (long)(tz))
-#define __sanitizer_syscall_pre_adjtimex(txc_p) \
+#define __sanitizer_syscall_pre_adjtimex(txc_p) \
__sanitizer_syscall_pre_impl_adjtimex((long)(txc_p))
-#define __sanitizer_syscall_post_adjtimex(res, txc_p) \
+#define __sanitizer_syscall_post_adjtimex(res, txc_p) \
__sanitizer_syscall_post_impl_adjtimex(res, (long)(txc_p))
-#define __sanitizer_syscall_pre_times(tbuf) \
+#define __sanitizer_syscall_pre_times(tbuf) \
__sanitizer_syscall_pre_impl_times((long)(tbuf))
-#define __sanitizer_syscall_post_times(res, tbuf) \
+#define __sanitizer_syscall_post_times(res, tbuf) \
__sanitizer_syscall_post_impl_times(res, (long)(tbuf))
#define __sanitizer_syscall_pre_gettid() __sanitizer_syscall_pre_impl_gettid()
-#define __sanitizer_syscall_post_gettid(res) \
+#define __sanitizer_syscall_post_gettid(res) \
__sanitizer_syscall_post_impl_gettid(res)
-#define __sanitizer_syscall_pre_nanosleep(rqtp, rmtp) \
+#define __sanitizer_syscall_pre_nanosleep(rqtp, rmtp) \
__sanitizer_syscall_pre_impl_nanosleep((long)(rqtp), (long)(rmtp))
-#define __sanitizer_syscall_post_nanosleep(res, rqtp, rmtp) \
+#define __sanitizer_syscall_post_nanosleep(res, rqtp, rmtp) \
__sanitizer_syscall_post_impl_nanosleep(res, (long)(rqtp), (long)(rmtp))
-#define __sanitizer_syscall_pre_alarm(seconds) \
+#define __sanitizer_syscall_pre_alarm(seconds) \
__sanitizer_syscall_pre_impl_alarm((long)(seconds))
-#define __sanitizer_syscall_post_alarm(res, seconds) \
+#define __sanitizer_syscall_post_alarm(res, seconds) \
__sanitizer_syscall_post_impl_alarm(res, (long)(seconds))
#define __sanitizer_syscall_pre_getpid() __sanitizer_syscall_pre_impl_getpid()
-#define __sanitizer_syscall_post_getpid(res) \
+#define __sanitizer_syscall_post_getpid(res) \
__sanitizer_syscall_post_impl_getpid(res)
#define __sanitizer_syscall_pre_getppid() __sanitizer_syscall_pre_impl_getppid()
-#define __sanitizer_syscall_post_getppid(res) \
+#define __sanitizer_syscall_post_getppid(res) \
__sanitizer_syscall_post_impl_getppid(res)
#define __sanitizer_syscall_pre_getuid() __sanitizer_syscall_pre_impl_getuid()
-#define __sanitizer_syscall_post_getuid(res) \
+#define __sanitizer_syscall_post_getuid(res) \
__sanitizer_syscall_post_impl_getuid(res)
#define __sanitizer_syscall_pre_geteuid() __sanitizer_syscall_pre_impl_geteuid()
-#define __sanitizer_syscall_post_geteuid(res) \
+#define __sanitizer_syscall_post_geteuid(res) \
__sanitizer_syscall_post_impl_geteuid(res)
#define __sanitizer_syscall_pre_getgid() __sanitizer_syscall_pre_impl_getgid()
-#define __sanitizer_syscall_post_getgid(res) \
+#define __sanitizer_syscall_post_getgid(res) \
__sanitizer_syscall_post_impl_getgid(res)
#define __sanitizer_syscall_pre_getegid() __sanitizer_syscall_pre_impl_getegid()
-#define __sanitizer_syscall_post_getegid(res) \
+#define __sanitizer_syscall_post_getegid(res) \
__sanitizer_syscall_post_impl_getegid(res)
-#define __sanitizer_syscall_pre_getresuid(ruid, euid, suid) \
- __sanitizer_syscall_pre_impl_getresuid((long)(ruid), (long)(euid), \
+#define __sanitizer_syscall_pre_getresuid(ruid, euid, suid) \
+ __sanitizer_syscall_pre_impl_getresuid((long)(ruid), (long)(euid), \
(long)(suid))
-#define __sanitizer_syscall_post_getresuid(res, ruid, euid, suid) \
- __sanitizer_syscall_post_impl_getresuid(res, (long)(ruid), (long)(euid), \
+#define __sanitizer_syscall_post_getresuid(res, ruid, euid, suid) \
+ __sanitizer_syscall_post_impl_getresuid(res, (long)(ruid), (long)(euid), \
(long)(suid))
-#define __sanitizer_syscall_pre_getresgid(rgid, egid, sgid) \
- __sanitizer_syscall_pre_impl_getresgid((long)(rgid), (long)(egid), \
+#define __sanitizer_syscall_pre_getresgid(rgid, egid, sgid) \
+ __sanitizer_syscall_pre_impl_getresgid((long)(rgid), (long)(egid), \
(long)(sgid))
-#define __sanitizer_syscall_post_getresgid(res, rgid, egid, sgid) \
- __sanitizer_syscall_post_impl_getresgid(res, (long)(rgid), (long)(egid), \
+#define __sanitizer_syscall_post_getresgid(res, rgid, egid, sgid) \
+ __sanitizer_syscall_post_impl_getresgid(res, (long)(rgid), (long)(egid), \
(long)(sgid))
-#define __sanitizer_syscall_pre_getpgid(pid) \
+#define __sanitizer_syscall_pre_getpgid(pid) \
__sanitizer_syscall_pre_impl_getpgid((long)(pid))
-#define __sanitizer_syscall_post_getpgid(res, pid) \
+#define __sanitizer_syscall_post_getpgid(res, pid) \
__sanitizer_syscall_post_impl_getpgid(res, (long)(pid))
#define __sanitizer_syscall_pre_getpgrp() __sanitizer_syscall_pre_impl_getpgrp()
-#define __sanitizer_syscall_post_getpgrp(res) \
+#define __sanitizer_syscall_post_getpgrp(res) \
__sanitizer_syscall_post_impl_getpgrp(res)
-#define __sanitizer_syscall_pre_getsid(pid) \
+#define __sanitizer_syscall_pre_getsid(pid) \
__sanitizer_syscall_pre_impl_getsid((long)(pid))
-#define __sanitizer_syscall_post_getsid(res, pid) \
+#define __sanitizer_syscall_post_getsid(res, pid) \
__sanitizer_syscall_post_impl_getsid(res, (long)(pid))
-#define __sanitizer_syscall_pre_getgroups(gidsetsize, grouplist) \
+#define __sanitizer_syscall_pre_getgroups(gidsetsize, grouplist) \
__sanitizer_syscall_pre_impl_getgroups((long)(gidsetsize), (long)(grouplist))
-#define __sanitizer_syscall_post_getgroups(res, gidsetsize, grouplist) \
- __sanitizer_syscall_post_impl_getgroups(res, (long)(gidsetsize), \
+#define __sanitizer_syscall_post_getgroups(res, gidsetsize, grouplist) \
+ __sanitizer_syscall_post_impl_getgroups(res, (long)(gidsetsize), \
(long)(grouplist))
-#define __sanitizer_syscall_pre_setregid(rgid, egid) \
+#define __sanitizer_syscall_pre_setregid(rgid, egid) \
__sanitizer_syscall_pre_impl_setregid((long)(rgid), (long)(egid))
-#define __sanitizer_syscall_post_setregid(res, rgid, egid) \
+#define __sanitizer_syscall_post_setregid(res, rgid, egid) \
__sanitizer_syscall_post_impl_setregid(res, (long)(rgid), (long)(egid))
-#define __sanitizer_syscall_pre_setgid(gid) \
+#define __sanitizer_syscall_pre_setgid(gid) \
__sanitizer_syscall_pre_impl_setgid((long)(gid))
-#define __sanitizer_syscall_post_setgid(res, gid) \
+#define __sanitizer_syscall_post_setgid(res, gid) \
__sanitizer_syscall_post_impl_setgid(res, (long)(gid))
-#define __sanitizer_syscall_pre_setreuid(ruid, euid) \
+#define __sanitizer_syscall_pre_setreuid(ruid, euid) \
__sanitizer_syscall_pre_impl_setreuid((long)(ruid), (long)(euid))
-#define __sanitizer_syscall_post_setreuid(res, ruid, euid) \
+#define __sanitizer_syscall_post_setreuid(res, ruid, euid) \
__sanitizer_syscall_post_impl_setreuid(res, (long)(ruid), (long)(euid))
-#define __sanitizer_syscall_pre_setuid(uid) \
+#define __sanitizer_syscall_pre_setuid(uid) \
__sanitizer_syscall_pre_impl_setuid((long)(uid))
-#define __sanitizer_syscall_post_setuid(res, uid) \
+#define __sanitizer_syscall_post_setuid(res, uid) \
__sanitizer_syscall_post_impl_setuid(res, (long)(uid))
-#define __sanitizer_syscall_pre_setresuid(ruid, euid, suid) \
- __sanitizer_syscall_pre_impl_setresuid((long)(ruid), (long)(euid), \
+#define __sanitizer_syscall_pre_setresuid(ruid, euid, suid) \
+ __sanitizer_syscall_pre_impl_setresuid((long)(ruid), (long)(euid), \
(long)(suid))
-#define __sanitizer_syscall_post_setresuid(res, ruid, euid, suid) \
- __sanitizer_syscall_post_impl_setresuid(res, (long)(ruid), (long)(euid), \
+#define __sanitizer_syscall_post_setresuid(res, ruid, euid, suid) \
+ __sanitizer_syscall_post_impl_setresuid(res, (long)(ruid), (long)(euid), \
(long)(suid))
-#define __sanitizer_syscall_pre_setresgid(rgid, egid, sgid) \
- __sanitizer_syscall_pre_impl_setresgid((long)(rgid), (long)(egid), \
+#define __sanitizer_syscall_pre_setresgid(rgid, egid, sgid) \
+ __sanitizer_syscall_pre_impl_setresgid((long)(rgid), (long)(egid), \
(long)(sgid))
-#define __sanitizer_syscall_post_setresgid(res, rgid, egid, sgid) \
- __sanitizer_syscall_post_impl_setresgid(res, (long)(rgid), (long)(egid), \
+#define __sanitizer_syscall_post_setresgid(res, rgid, egid, sgid) \
+ __sanitizer_syscall_post_impl_setresgid(res, (long)(rgid), (long)(egid), \
(long)(sgid))
-#define __sanitizer_syscall_pre_setfsuid(uid) \
+#define __sanitizer_syscall_pre_setfsuid(uid) \
__sanitizer_syscall_pre_impl_setfsuid((long)(uid))
-#define __sanitizer_syscall_post_setfsuid(res, uid) \
+#define __sanitizer_syscall_post_setfsuid(res, uid) \
__sanitizer_syscall_post_impl_setfsuid(res, (long)(uid))
-#define __sanitizer_syscall_pre_setfsgid(gid) \
+#define __sanitizer_syscall_pre_setfsgid(gid) \
__sanitizer_syscall_pre_impl_setfsgid((long)(gid))
-#define __sanitizer_syscall_post_setfsgid(res, gid) \
+#define __sanitizer_syscall_post_setfsgid(res, gid) \
__sanitizer_syscall_post_impl_setfsgid(res, (long)(gid))
-#define __sanitizer_syscall_pre_setpgid(pid, pgid) \
+#define __sanitizer_syscall_pre_setpgid(pid, pgid) \
__sanitizer_syscall_pre_impl_setpgid((long)(pid), (long)(pgid))
-#define __sanitizer_syscall_post_setpgid(res, pid, pgid) \
+#define __sanitizer_syscall_post_setpgid(res, pid, pgid) \
__sanitizer_syscall_post_impl_setpgid(res, (long)(pid), (long)(pgid))
#define __sanitizer_syscall_pre_setsid() __sanitizer_syscall_pre_impl_setsid()
-#define __sanitizer_syscall_post_setsid(res) \
+#define __sanitizer_syscall_post_setsid(res) \
__sanitizer_syscall_post_impl_setsid(res)
-#define __sanitizer_syscall_pre_setgroups(gidsetsize, grouplist) \
+#define __sanitizer_syscall_pre_setgroups(gidsetsize, grouplist) \
__sanitizer_syscall_pre_impl_setgroups((long)(gidsetsize), (long)(grouplist))
-#define __sanitizer_syscall_post_setgroups(res, gidsetsize, grouplist) \
- __sanitizer_syscall_post_impl_setgroups(res, (long)(gidsetsize), \
+#define __sanitizer_syscall_post_setgroups(res, gidsetsize, grouplist) \
+ __sanitizer_syscall_post_impl_setgroups(res, (long)(gidsetsize), \
(long)(grouplist))
-#define __sanitizer_syscall_pre_acct(name) \
+#define __sanitizer_syscall_pre_acct(name) \
__sanitizer_syscall_pre_impl_acct((long)(name))
-#define __sanitizer_syscall_post_acct(res, name) \
+#define __sanitizer_syscall_post_acct(res, name) \
__sanitizer_syscall_post_impl_acct(res, (long)(name))
-#define __sanitizer_syscall_pre_capget(header, dataptr) \
+#define __sanitizer_syscall_pre_capget(header, dataptr) \
__sanitizer_syscall_pre_impl_capget((long)(header), (long)(dataptr))
-#define __sanitizer_syscall_post_capget(res, header, dataptr) \
+#define __sanitizer_syscall_post_capget(res, header, dataptr) \
__sanitizer_syscall_post_impl_capget(res, (long)(header), (long)(dataptr))
-#define __sanitizer_syscall_pre_capset(header, data) \
+#define __sanitizer_syscall_pre_capset(header, data) \
__sanitizer_syscall_pre_impl_capset((long)(header), (long)(data))
-#define __sanitizer_syscall_post_capset(res, header, data) \
+#define __sanitizer_syscall_post_capset(res, header, data) \
__sanitizer_syscall_post_impl_capset(res, (long)(header), (long)(data))
-#define __sanitizer_syscall_pre_personality(personality) \
+#define __sanitizer_syscall_pre_personality(personality) \
__sanitizer_syscall_pre_impl_personality((long)(personality))
-#define __sanitizer_syscall_post_personality(res, personality) \
+#define __sanitizer_syscall_post_personality(res, personality) \
__sanitizer_syscall_post_impl_personality(res, (long)(personality))
-#define __sanitizer_syscall_pre_sigpending(set) \
+#define __sanitizer_syscall_pre_sigpending(set) \
__sanitizer_syscall_pre_impl_sigpending((long)(set))
-#define __sanitizer_syscall_post_sigpending(res, set) \
+#define __sanitizer_syscall_post_sigpending(res, set) \
__sanitizer_syscall_post_impl_sigpending(res, (long)(set))
-#define __sanitizer_syscall_pre_sigprocmask(how, set, oset) \
- __sanitizer_syscall_pre_impl_sigprocmask((long)(how), (long)(set), \
+#define __sanitizer_syscall_pre_sigprocmask(how, set, oset) \
+ __sanitizer_syscall_pre_impl_sigprocmask((long)(how), (long)(set), \
(long)(oset))
-#define __sanitizer_syscall_post_sigprocmask(res, how, set, oset) \
- __sanitizer_syscall_post_impl_sigprocmask(res, (long)(how), (long)(set), \
+#define __sanitizer_syscall_post_sigprocmask(res, how, set, oset) \
+ __sanitizer_syscall_post_impl_sigprocmask(res, (long)(how), (long)(set), \
(long)(oset))
-#define __sanitizer_syscall_pre_getitimer(which, value) \
+#define __sanitizer_syscall_pre_getitimer(which, value) \
__sanitizer_syscall_pre_impl_getitimer((long)(which), (long)(value))
-#define __sanitizer_syscall_post_getitimer(res, which, value) \
+#define __sanitizer_syscall_post_getitimer(res, which, value) \
__sanitizer_syscall_post_impl_getitimer(res, (long)(which), (long)(value))
-#define __sanitizer_syscall_pre_setitimer(which, value, ovalue) \
- __sanitizer_syscall_pre_impl_setitimer((long)(which), (long)(value), \
+#define __sanitizer_syscall_pre_setitimer(which, value, ovalue) \
+ __sanitizer_syscall_pre_impl_setitimer((long)(which), (long)(value), \
(long)(ovalue))
-#define __sanitizer_syscall_post_setitimer(res, which, value, ovalue) \
- __sanitizer_syscall_post_impl_setitimer(res, (long)(which), (long)(value), \
+#define __sanitizer_syscall_post_setitimer(res, which, value, ovalue) \
+ __sanitizer_syscall_post_impl_setitimer(res, (long)(which), (long)(value), \
(long)(ovalue))
-#define __sanitizer_syscall_pre_timer_create(which_clock, timer_event_spec, \
- created_timer_id) \
- __sanitizer_syscall_pre_impl_timer_create( \
+#define __sanitizer_syscall_pre_timer_create(which_clock, timer_event_spec, \
+ created_timer_id) \
+ __sanitizer_syscall_pre_impl_timer_create( \
(long)(which_clock), (long)(timer_event_spec), (long)(created_timer_id))
-#define __sanitizer_syscall_post_timer_create( \
- res, which_clock, timer_event_spec, created_timer_id) \
- __sanitizer_syscall_post_impl_timer_create(res, (long)(which_clock), \
- (long)(timer_event_spec), \
+#define __sanitizer_syscall_post_timer_create( \
+ res, which_clock, timer_event_spec, created_timer_id) \
+ __sanitizer_syscall_post_impl_timer_create(res, (long)(which_clock), \
+ (long)(timer_event_spec), \
(long)(created_timer_id))
-#define __sanitizer_syscall_pre_timer_gettime(timer_id, setting) \
+#define __sanitizer_syscall_pre_timer_gettime(timer_id, setting) \
__sanitizer_syscall_pre_impl_timer_gettime((long)(timer_id), (long)(setting))
-#define __sanitizer_syscall_post_timer_gettime(res, timer_id, setting) \
- __sanitizer_syscall_post_impl_timer_gettime(res, (long)(timer_id), \
+#define __sanitizer_syscall_post_timer_gettime(res, timer_id, setting) \
+ __sanitizer_syscall_post_impl_timer_gettime(res, (long)(timer_id), \
(long)(setting))
-#define __sanitizer_syscall_pre_timer_getoverrun(timer_id) \
+#define __sanitizer_syscall_pre_timer_getoverrun(timer_id) \
__sanitizer_syscall_pre_impl_timer_getoverrun((long)(timer_id))
-#define __sanitizer_syscall_post_timer_getoverrun(res, timer_id) \
+#define __sanitizer_syscall_post_timer_getoverrun(res, timer_id) \
__sanitizer_syscall_post_impl_timer_getoverrun(res, (long)(timer_id))
-#define __sanitizer_syscall_pre_timer_settime(timer_id, flags, new_setting, \
- old_setting) \
- __sanitizer_syscall_pre_impl_timer_settime((long)(timer_id), (long)(flags), \
- (long)(new_setting), \
+#define __sanitizer_syscall_pre_timer_settime(timer_id, flags, new_setting, \
+ old_setting) \
+ __sanitizer_syscall_pre_impl_timer_settime((long)(timer_id), (long)(flags), \
+ (long)(new_setting), \
(long)(old_setting))
-#define __sanitizer_syscall_post_timer_settime(res, timer_id, flags, \
- new_setting, old_setting) \
- __sanitizer_syscall_post_impl_timer_settime( \
- res, (long)(timer_id), (long)(flags), (long)(new_setting), \
+#define __sanitizer_syscall_post_timer_settime(res, timer_id, flags, \
+ new_setting, old_setting) \
+ __sanitizer_syscall_post_impl_timer_settime( \
+ res, (long)(timer_id), (long)(flags), (long)(new_setting), \
(long)(old_setting))
-#define __sanitizer_syscall_pre_timer_delete(timer_id) \
+#define __sanitizer_syscall_pre_timer_delete(timer_id) \
__sanitizer_syscall_pre_impl_timer_delete((long)(timer_id))
-#define __sanitizer_syscall_post_timer_delete(res, timer_id) \
+#define __sanitizer_syscall_post_timer_delete(res, timer_id) \
__sanitizer_syscall_post_impl_timer_delete(res, (long)(timer_id))
-#define __sanitizer_syscall_pre_clock_settime(which_clock, tp) \
+#define __sanitizer_syscall_pre_clock_settime(which_clock, tp) \
__sanitizer_syscall_pre_impl_clock_settime((long)(which_clock), (long)(tp))
-#define __sanitizer_syscall_post_clock_settime(res, which_clock, tp) \
- __sanitizer_syscall_post_impl_clock_settime(res, (long)(which_clock), \
+#define __sanitizer_syscall_post_clock_settime(res, which_clock, tp) \
+ __sanitizer_syscall_post_impl_clock_settime(res, (long)(which_clock), \
(long)(tp))
-#define __sanitizer_syscall_pre_clock_gettime(which_clock, tp) \
+#define __sanitizer_syscall_pre_clock_gettime(which_clock, tp) \
__sanitizer_syscall_pre_impl_clock_gettime((long)(which_clock), (long)(tp))
-#define __sanitizer_syscall_post_clock_gettime(res, which_clock, tp) \
- __sanitizer_syscall_post_impl_clock_gettime(res, (long)(which_clock), \
+#define __sanitizer_syscall_post_clock_gettime(res, which_clock, tp) \
+ __sanitizer_syscall_post_impl_clock_gettime(res, (long)(which_clock), \
(long)(tp))
-#define __sanitizer_syscall_pre_clock_adjtime(which_clock, tx) \
+#define __sanitizer_syscall_pre_clock_adjtime(which_clock, tx) \
__sanitizer_syscall_pre_impl_clock_adjtime((long)(which_clock), (long)(tx))
-#define __sanitizer_syscall_post_clock_adjtime(res, which_clock, tx) \
- __sanitizer_syscall_post_impl_clock_adjtime(res, (long)(which_clock), \
+#define __sanitizer_syscall_post_clock_adjtime(res, which_clock, tx) \
+ __sanitizer_syscall_post_impl_clock_adjtime(res, (long)(which_clock), \
(long)(tx))
-#define __sanitizer_syscall_pre_clock_getres(which_clock, tp) \
+#define __sanitizer_syscall_pre_clock_getres(which_clock, tp) \
__sanitizer_syscall_pre_impl_clock_getres((long)(which_clock), (long)(tp))
-#define __sanitizer_syscall_post_clock_getres(res, which_clock, tp) \
- __sanitizer_syscall_post_impl_clock_getres(res, (long)(which_clock), \
+#define __sanitizer_syscall_post_clock_getres(res, which_clock, tp) \
+ __sanitizer_syscall_post_impl_clock_getres(res, (long)(which_clock), \
(long)(tp))
-#define __sanitizer_syscall_pre_clock_nanosleep(which_clock, flags, rqtp, \
- rmtp) \
- __sanitizer_syscall_pre_impl_clock_nanosleep( \
+#define __sanitizer_syscall_pre_clock_nanosleep(which_clock, flags, rqtp, \
+ rmtp) \
+ __sanitizer_syscall_pre_impl_clock_nanosleep( \
(long)(which_clock), (long)(flags), (long)(rqtp), (long)(rmtp))
-#define __sanitizer_syscall_post_clock_nanosleep(res, which_clock, flags, \
- rqtp, rmtp) \
- __sanitizer_syscall_post_impl_clock_nanosleep( \
+#define __sanitizer_syscall_post_clock_nanosleep(res, which_clock, flags, \
+ rqtp, rmtp) \
+ __sanitizer_syscall_post_impl_clock_nanosleep( \
res, (long)(which_clock), (long)(flags), (long)(rqtp), (long)(rmtp))
-#define __sanitizer_syscall_pre_nice(increment) \
+#define __sanitizer_syscall_pre_nice(increment) \
__sanitizer_syscall_pre_impl_nice((long)(increment))
-#define __sanitizer_syscall_post_nice(res, increment) \
+#define __sanitizer_syscall_post_nice(res, increment) \
__sanitizer_syscall_post_impl_nice(res, (long)(increment))
#define __sanitizer_syscall_pre_sched_setscheduler(pid, policy, param) \
__sanitizer_syscall_pre_impl_sched_setscheduler((long)(pid), (long)(policy), \
(long)(param))
-#define __sanitizer_syscall_post_sched_setscheduler(res, pid, policy, param) \
- __sanitizer_syscall_post_impl_sched_setscheduler( \
+#define __sanitizer_syscall_post_sched_setscheduler(res, pid, policy, param) \
+ __sanitizer_syscall_post_impl_sched_setscheduler( \
res, (long)(pid), (long)(policy), (long)(param))
-#define __sanitizer_syscall_pre_sched_setparam(pid, param) \
+#define __sanitizer_syscall_pre_sched_setparam(pid, param) \
__sanitizer_syscall_pre_impl_sched_setparam((long)(pid), (long)(param))
-#define __sanitizer_syscall_post_sched_setparam(res, pid, param) \
+#define __sanitizer_syscall_post_sched_setparam(res, pid, param) \
__sanitizer_syscall_post_impl_sched_setparam(res, (long)(pid), (long)(param))
-#define __sanitizer_syscall_pre_sched_getscheduler(pid) \
+#define __sanitizer_syscall_pre_sched_getscheduler(pid) \
__sanitizer_syscall_pre_impl_sched_getscheduler((long)(pid))
-#define __sanitizer_syscall_post_sched_getscheduler(res, pid) \
+#define __sanitizer_syscall_post_sched_getscheduler(res, pid) \
__sanitizer_syscall_post_impl_sched_getscheduler(res, (long)(pid))
-#define __sanitizer_syscall_pre_sched_getparam(pid, param) \
+#define __sanitizer_syscall_pre_sched_getparam(pid, param) \
__sanitizer_syscall_pre_impl_sched_getparam((long)(pid), (long)(param))
-#define __sanitizer_syscall_post_sched_getparam(res, pid, param) \
+#define __sanitizer_syscall_post_sched_getparam(res, pid, param) \
__sanitizer_syscall_post_impl_sched_getparam(res, (long)(pid), (long)(param))
-#define __sanitizer_syscall_pre_sched_setaffinity(pid, len, user_mask_ptr) \
- __sanitizer_syscall_pre_impl_sched_setaffinity((long)(pid), (long)(len), \
+#define __sanitizer_syscall_pre_sched_setaffinity(pid, len, user_mask_ptr) \
+ __sanitizer_syscall_pre_impl_sched_setaffinity((long)(pid), (long)(len), \
(long)(user_mask_ptr))
-#define __sanitizer_syscall_post_sched_setaffinity(res, pid, len, \
- user_mask_ptr) \
- __sanitizer_syscall_post_impl_sched_setaffinity( \
+#define __sanitizer_syscall_post_sched_setaffinity(res, pid, len, \
+ user_mask_ptr) \
+ __sanitizer_syscall_post_impl_sched_setaffinity( \
res, (long)(pid), (long)(len), (long)(user_mask_ptr))
-#define __sanitizer_syscall_pre_sched_getaffinity(pid, len, user_mask_ptr) \
- __sanitizer_syscall_pre_impl_sched_getaffinity((long)(pid), (long)(len), \
+#define __sanitizer_syscall_pre_sched_getaffinity(pid, len, user_mask_ptr) \
+ __sanitizer_syscall_pre_impl_sched_getaffinity((long)(pid), (long)(len), \
(long)(user_mask_ptr))
-#define __sanitizer_syscall_post_sched_getaffinity(res, pid, len, \
- user_mask_ptr) \
- __sanitizer_syscall_post_impl_sched_getaffinity( \
+#define __sanitizer_syscall_post_sched_getaffinity(res, pid, len, \
+ user_mask_ptr) \
+ __sanitizer_syscall_post_impl_sched_getaffinity( \
res, (long)(pid), (long)(len), (long)(user_mask_ptr))
-#define __sanitizer_syscall_pre_sched_yield() \
+#define __sanitizer_syscall_pre_sched_yield() \
__sanitizer_syscall_pre_impl_sched_yield()
-#define __sanitizer_syscall_post_sched_yield(res) \
+#define __sanitizer_syscall_post_sched_yield(res) \
__sanitizer_syscall_post_impl_sched_yield(res)
-#define __sanitizer_syscall_pre_sched_get_priority_max(policy) \
+#define __sanitizer_syscall_pre_sched_get_priority_max(policy) \
__sanitizer_syscall_pre_impl_sched_get_priority_max((long)(policy))
-#define __sanitizer_syscall_post_sched_get_priority_max(res, policy) \
+#define __sanitizer_syscall_post_sched_get_priority_max(res, policy) \
__sanitizer_syscall_post_impl_sched_get_priority_max(res, (long)(policy))
-#define __sanitizer_syscall_pre_sched_get_priority_min(policy) \
+#define __sanitizer_syscall_pre_sched_get_priority_min(policy) \
__sanitizer_syscall_pre_impl_sched_get_priority_min((long)(policy))
-#define __sanitizer_syscall_post_sched_get_priority_min(res, policy) \
+#define __sanitizer_syscall_post_sched_get_priority_min(res, policy) \
__sanitizer_syscall_post_impl_sched_get_priority_min(res, (long)(policy))
-#define __sanitizer_syscall_pre_sched_rr_get_interval(pid, interval) \
- __sanitizer_syscall_pre_impl_sched_rr_get_interval((long)(pid), \
+#define __sanitizer_syscall_pre_sched_rr_get_interval(pid, interval) \
+ __sanitizer_syscall_pre_impl_sched_rr_get_interval((long)(pid), \
(long)(interval))
-#define __sanitizer_syscall_post_sched_rr_get_interval(res, pid, interval) \
- __sanitizer_syscall_post_impl_sched_rr_get_interval(res, (long)(pid), \
+#define __sanitizer_syscall_post_sched_rr_get_interval(res, pid, interval) \
+ __sanitizer_syscall_post_impl_sched_rr_get_interval(res, (long)(pid), \
(long)(interval))
-#define __sanitizer_syscall_pre_setpriority(which, who, niceval) \
- __sanitizer_syscall_pre_impl_setpriority((long)(which), (long)(who), \
+#define __sanitizer_syscall_pre_setpriority(which, who, niceval) \
+ __sanitizer_syscall_pre_impl_setpriority((long)(which), (long)(who), \
(long)(niceval))
-#define __sanitizer_syscall_post_setpriority(res, which, who, niceval) \
- __sanitizer_syscall_post_impl_setpriority(res, (long)(which), (long)(who), \
+#define __sanitizer_syscall_post_setpriority(res, which, who, niceval) \
+ __sanitizer_syscall_post_impl_setpriority(res, (long)(which), (long)(who), \
(long)(niceval))
-#define __sanitizer_syscall_pre_getpriority(which, who) \
+#define __sanitizer_syscall_pre_getpriority(which, who) \
__sanitizer_syscall_pre_impl_getpriority((long)(which), (long)(who))
-#define __sanitizer_syscall_post_getpriority(res, which, who) \
+#define __sanitizer_syscall_post_getpriority(res, which, who) \
__sanitizer_syscall_post_impl_getpriority(res, (long)(which), (long)(who))
-#define __sanitizer_syscall_pre_shutdown(arg0, arg1) \
+#define __sanitizer_syscall_pre_shutdown(arg0, arg1) \
__sanitizer_syscall_pre_impl_shutdown((long)(arg0), (long)(arg1))
-#define __sanitizer_syscall_post_shutdown(res, arg0, arg1) \
+#define __sanitizer_syscall_post_shutdown(res, arg0, arg1) \
__sanitizer_syscall_post_impl_shutdown(res, (long)(arg0), (long)(arg1))
-#define __sanitizer_syscall_pre_reboot(magic1, magic2, cmd, arg) \
- __sanitizer_syscall_pre_impl_reboot((long)(magic1), (long)(magic2), \
+#define __sanitizer_syscall_pre_reboot(magic1, magic2, cmd, arg) \
+ __sanitizer_syscall_pre_impl_reboot((long)(magic1), (long)(magic2), \
(long)(cmd), (long)(arg))
-#define __sanitizer_syscall_post_reboot(res, magic1, magic2, cmd, arg) \
- __sanitizer_syscall_post_impl_reboot(res, (long)(magic1), (long)(magic2), \
+#define __sanitizer_syscall_post_reboot(res, magic1, magic2, cmd, arg) \
+ __sanitizer_syscall_post_impl_reboot(res, (long)(magic1), (long)(magic2), \
(long)(cmd), (long)(arg))
-#define __sanitizer_syscall_pre_restart_syscall() \
+#define __sanitizer_syscall_pre_restart_syscall() \
__sanitizer_syscall_pre_impl_restart_syscall()
-#define __sanitizer_syscall_post_restart_syscall(res) \
+#define __sanitizer_syscall_post_restart_syscall(res) \
__sanitizer_syscall_post_impl_restart_syscall(res)
-#define __sanitizer_syscall_pre_kexec_load(entry, nr_segments, segments, \
- flags) \
- __sanitizer_syscall_pre_impl_kexec_load((long)(entry), (long)(nr_segments), \
+#define __sanitizer_syscall_pre_kexec_load(entry, nr_segments, segments, \
+ flags) \
+ __sanitizer_syscall_pre_impl_kexec_load((long)(entry), (long)(nr_segments), \
(long)(segments), (long)(flags))
#define __sanitizer_syscall_post_kexec_load(res, entry, nr_segments, segments, \
flags) \
__sanitizer_syscall_post_impl_kexec_load(res, (long)(entry), \
(long)(nr_segments), \
(long)(segments), (long)(flags))
-#define __sanitizer_syscall_pre_exit(error_code) \
+#define __sanitizer_syscall_pre_exit(error_code) \
__sanitizer_syscall_pre_impl_exit((long)(error_code))
-#define __sanitizer_syscall_post_exit(res, error_code) \
+#define __sanitizer_syscall_post_exit(res, error_code) \
__sanitizer_syscall_post_impl_exit(res, (long)(error_code))
-#define __sanitizer_syscall_pre_exit_group(error_code) \
+#define __sanitizer_syscall_pre_exit_group(error_code) \
__sanitizer_syscall_pre_impl_exit_group((long)(error_code))
-#define __sanitizer_syscall_post_exit_group(res, error_code) \
+#define __sanitizer_syscall_post_exit_group(res, error_code) \
__sanitizer_syscall_post_impl_exit_group(res, (long)(error_code))
-#define __sanitizer_syscall_pre_wait4(pid, stat_addr, options, ru) \
- __sanitizer_syscall_pre_impl_wait4((long)(pid), (long)(stat_addr), \
+#define __sanitizer_syscall_pre_wait4(pid, stat_addr, options, ru) \
+ __sanitizer_syscall_pre_impl_wait4((long)(pid), (long)(stat_addr), \
(long)(options), (long)(ru))
-#define __sanitizer_syscall_post_wait4(res, pid, stat_addr, options, ru) \
- __sanitizer_syscall_post_impl_wait4(res, (long)(pid), (long)(stat_addr), \
+#define __sanitizer_syscall_post_wait4(res, pid, stat_addr, options, ru) \
+ __sanitizer_syscall_post_impl_wait4(res, (long)(pid), (long)(stat_addr), \
(long)(options), (long)(ru))
-#define __sanitizer_syscall_pre_waitid(which, pid, infop, options, ru) \
- __sanitizer_syscall_pre_impl_waitid( \
+#define __sanitizer_syscall_pre_waitid(which, pid, infop, options, ru) \
+ __sanitizer_syscall_pre_impl_waitid( \
(long)(which), (long)(pid), (long)(infop), (long)(options), (long)(ru))
-#define __sanitizer_syscall_post_waitid(res, which, pid, infop, options, ru) \
- __sanitizer_syscall_post_impl_waitid(res, (long)(which), (long)(pid), \
- (long)(infop), (long)(options), \
+#define __sanitizer_syscall_post_waitid(res, which, pid, infop, options, ru) \
+ __sanitizer_syscall_post_impl_waitid(res, (long)(which), (long)(pid), \
+ (long)(infop), (long)(options), \
(long)(ru))
-#define __sanitizer_syscall_pre_waitpid(pid, stat_addr, options) \
- __sanitizer_syscall_pre_impl_waitpid((long)(pid), (long)(stat_addr), \
+#define __sanitizer_syscall_pre_waitpid(pid, stat_addr, options) \
+ __sanitizer_syscall_pre_impl_waitpid((long)(pid), (long)(stat_addr), \
(long)(options))
-#define __sanitizer_syscall_post_waitpid(res, pid, stat_addr, options) \
- __sanitizer_syscall_post_impl_waitpid(res, (long)(pid), (long)(stat_addr), \
+#define __sanitizer_syscall_post_waitpid(res, pid, stat_addr, options) \
+ __sanitizer_syscall_post_impl_waitpid(res, (long)(pid), (long)(stat_addr), \
(long)(options))
-#define __sanitizer_syscall_pre_set_tid_address(tidptr) \
+#define __sanitizer_syscall_pre_set_tid_address(tidptr) \
__sanitizer_syscall_pre_impl_set_tid_address((long)(tidptr))
-#define __sanitizer_syscall_post_set_tid_address(res, tidptr) \
+#define __sanitizer_syscall_post_set_tid_address(res, tidptr) \
__sanitizer_syscall_post_impl_set_tid_address(res, (long)(tidptr))
-#define __sanitizer_syscall_pre_init_module(umod, len, uargs) \
- __sanitizer_syscall_pre_impl_init_module((long)(umod), (long)(len), \
+#define __sanitizer_syscall_pre_init_module(umod, len, uargs) \
+ __sanitizer_syscall_pre_impl_init_module((long)(umod), (long)(len), \
(long)(uargs))
-#define __sanitizer_syscall_post_init_module(res, umod, len, uargs) \
- __sanitizer_syscall_post_impl_init_module(res, (long)(umod), (long)(len), \
+#define __sanitizer_syscall_post_init_module(res, umod, len, uargs) \
+ __sanitizer_syscall_post_impl_init_module(res, (long)(umod), (long)(len), \
(long)(uargs))
-#define __sanitizer_syscall_pre_delete_module(name_user, flags) \
+#define __sanitizer_syscall_pre_delete_module(name_user, flags) \
__sanitizer_syscall_pre_impl_delete_module((long)(name_user), (long)(flags))
-#define __sanitizer_syscall_post_delete_module(res, name_user, flags) \
- __sanitizer_syscall_post_impl_delete_module(res, (long)(name_user), \
+#define __sanitizer_syscall_post_delete_module(res, name_user, flags) \
+ __sanitizer_syscall_post_impl_delete_module(res, (long)(name_user), \
(long)(flags))
-#define __sanitizer_syscall_pre_rt_sigprocmask(how, set, oset, sigsetsize) \
- __sanitizer_syscall_pre_impl_rt_sigprocmask( \
+#define __sanitizer_syscall_pre_rt_sigprocmask(how, set, oset, sigsetsize) \
+ __sanitizer_syscall_pre_impl_rt_sigprocmask( \
(long)(how), (long)(set), (long)(oset), (long)(sigsetsize))
-#define __sanitizer_syscall_post_rt_sigprocmask(res, how, set, oset, \
- sigsetsize) \
- __sanitizer_syscall_post_impl_rt_sigprocmask( \
+#define __sanitizer_syscall_post_rt_sigprocmask(res, how, set, oset, \
+ sigsetsize) \
+ __sanitizer_syscall_post_impl_rt_sigprocmask( \
res, (long)(how), (long)(set), (long)(oset), (long)(sigsetsize))
-#define __sanitizer_syscall_pre_rt_sigpending(set, sigsetsize) \
+#define __sanitizer_syscall_pre_rt_sigpending(set, sigsetsize) \
__sanitizer_syscall_pre_impl_rt_sigpending((long)(set), (long)(sigsetsize))
-#define __sanitizer_syscall_post_rt_sigpending(res, set, sigsetsize) \
- __sanitizer_syscall_post_impl_rt_sigpending(res, (long)(set), \
+#define __sanitizer_syscall_post_rt_sigpending(res, set, sigsetsize) \
+ __sanitizer_syscall_post_impl_rt_sigpending(res, (long)(set), \
(long)(sigsetsize))
-#define __sanitizer_syscall_pre_rt_sigtimedwait(uthese, uinfo, uts, \
- sigsetsize) \
- __sanitizer_syscall_pre_impl_rt_sigtimedwait( \
+#define __sanitizer_syscall_pre_rt_sigtimedwait(uthese, uinfo, uts, \
+ sigsetsize) \
+ __sanitizer_syscall_pre_impl_rt_sigtimedwait( \
(long)(uthese), (long)(uinfo), (long)(uts), (long)(sigsetsize))
-#define __sanitizer_syscall_post_rt_sigtimedwait(res, uthese, uinfo, uts, \
- sigsetsize) \
- __sanitizer_syscall_post_impl_rt_sigtimedwait( \
+#define __sanitizer_syscall_post_rt_sigtimedwait(res, uthese, uinfo, uts, \
+ sigsetsize) \
+ __sanitizer_syscall_post_impl_rt_sigtimedwait( \
res, (long)(uthese), (long)(uinfo), (long)(uts), (long)(sigsetsize))
-#define __sanitizer_syscall_pre_rt_tgsigqueueinfo(tgid, pid, sig, uinfo) \
- __sanitizer_syscall_pre_impl_rt_tgsigqueueinfo((long)(tgid), (long)(pid), \
+#define __sanitizer_syscall_pre_rt_tgsigqueueinfo(tgid, pid, sig, uinfo) \
+ __sanitizer_syscall_pre_impl_rt_tgsigqueueinfo((long)(tgid), (long)(pid), \
(long)(sig), (long)(uinfo))
#define __sanitizer_syscall_post_rt_tgsigqueueinfo(res, tgid, pid, sig, uinfo) \
__sanitizer_syscall_post_impl_rt_tgsigqueueinfo( \
res, (long)(tgid), (long)(pid), (long)(sig), (long)(uinfo))
-#define __sanitizer_syscall_pre_kill(pid, sig) \
+#define __sanitizer_syscall_pre_kill(pid, sig) \
__sanitizer_syscall_pre_impl_kill((long)(pid), (long)(sig))
-#define __sanitizer_syscall_post_kill(res, pid, sig) \
+#define __sanitizer_syscall_post_kill(res, pid, sig) \
__sanitizer_syscall_post_impl_kill(res, (long)(pid), (long)(sig))
-#define __sanitizer_syscall_pre_tgkill(tgid, pid, sig) \
+#define __sanitizer_syscall_pre_tgkill(tgid, pid, sig) \
__sanitizer_syscall_pre_impl_tgkill((long)(tgid), (long)(pid), (long)(sig))
-#define __sanitizer_syscall_post_tgkill(res, tgid, pid, sig) \
- __sanitizer_syscall_post_impl_tgkill(res, (long)(tgid), (long)(pid), \
+#define __sanitizer_syscall_post_tgkill(res, tgid, pid, sig) \
+ __sanitizer_syscall_post_impl_tgkill(res, (long)(tgid), (long)(pid), \
(long)(sig))
-#define __sanitizer_syscall_pre_tkill(pid, sig) \
+#define __sanitizer_syscall_pre_tkill(pid, sig) \
__sanitizer_syscall_pre_impl_tkill((long)(pid), (long)(sig))
-#define __sanitizer_syscall_post_tkill(res, pid, sig) \
+#define __sanitizer_syscall_post_tkill(res, pid, sig) \
__sanitizer_syscall_post_impl_tkill(res, (long)(pid), (long)(sig))
-#define __sanitizer_syscall_pre_rt_sigqueueinfo(pid, sig, uinfo) \
- __sanitizer_syscall_pre_impl_rt_sigqueueinfo((long)(pid), (long)(sig), \
+#define __sanitizer_syscall_pre_rt_sigqueueinfo(pid, sig, uinfo) \
+ __sanitizer_syscall_pre_impl_rt_sigqueueinfo((long)(pid), (long)(sig), \
(long)(uinfo))
#define __sanitizer_syscall_post_rt_sigqueueinfo(res, pid, sig, uinfo) \
__sanitizer_syscall_post_impl_rt_sigqueueinfo(res, (long)(pid), (long)(sig), \
(long)(uinfo))
-#define __sanitizer_syscall_pre_sgetmask() \
+#define __sanitizer_syscall_pre_sgetmask() \
__sanitizer_syscall_pre_impl_sgetmask()
-#define __sanitizer_syscall_post_sgetmask(res) \
+#define __sanitizer_syscall_post_sgetmask(res) \
__sanitizer_syscall_post_impl_sgetmask(res)
-#define __sanitizer_syscall_pre_ssetmask(newmask) \
+#define __sanitizer_syscall_pre_ssetmask(newmask) \
__sanitizer_syscall_pre_impl_ssetmask((long)(newmask))
-#define __sanitizer_syscall_post_ssetmask(res, newmask) \
+#define __sanitizer_syscall_post_ssetmask(res, newmask) \
__sanitizer_syscall_post_impl_ssetmask(res, (long)(newmask))
-#define __sanitizer_syscall_pre_signal(sig, handler) \
+#define __sanitizer_syscall_pre_signal(sig, handler) \
__sanitizer_syscall_pre_impl_signal((long)(sig), (long)(handler))
-#define __sanitizer_syscall_post_signal(res, sig, handler) \
+#define __sanitizer_syscall_post_signal(res, sig, handler) \
__sanitizer_syscall_post_impl_signal(res, (long)(sig), (long)(handler))
#define __sanitizer_syscall_pre_pause() __sanitizer_syscall_pre_impl_pause()
-#define __sanitizer_syscall_post_pause(res) \
+#define __sanitizer_syscall_post_pause(res) \
__sanitizer_syscall_post_impl_pause(res)
#define __sanitizer_syscall_pre_sync() __sanitizer_syscall_pre_impl_sync()
-#define __sanitizer_syscall_post_sync(res) \
+#define __sanitizer_syscall_post_sync(res) \
__sanitizer_syscall_post_impl_sync(res)
-#define __sanitizer_syscall_pre_fsync(fd) \
+#define __sanitizer_syscall_pre_fsync(fd) \
__sanitizer_syscall_pre_impl_fsync((long)(fd))
-#define __sanitizer_syscall_post_fsync(res, fd) \
+#define __sanitizer_syscall_post_fsync(res, fd) \
__sanitizer_syscall_post_impl_fsync(res, (long)(fd))
-#define __sanitizer_syscall_pre_fdatasync(fd) \
+#define __sanitizer_syscall_pre_fdatasync(fd) \
__sanitizer_syscall_pre_impl_fdatasync((long)(fd))
-#define __sanitizer_syscall_post_fdatasync(res, fd) \
+#define __sanitizer_syscall_post_fdatasync(res, fd) \
__sanitizer_syscall_post_impl_fdatasync(res, (long)(fd))
-#define __sanitizer_syscall_pre_bdflush(func, data) \
+#define __sanitizer_syscall_pre_bdflush(func, data) \
__sanitizer_syscall_pre_impl_bdflush((long)(func), (long)(data))
-#define __sanitizer_syscall_post_bdflush(res, func, data) \
+#define __sanitizer_syscall_post_bdflush(res, func, data) \
__sanitizer_syscall_post_impl_bdflush(res, (long)(func), (long)(data))
-#define __sanitizer_syscall_pre_mount(dev_name, dir_name, type, flags, data) \
- __sanitizer_syscall_pre_impl_mount((long)(dev_name), (long)(dir_name), \
- (long)(type), (long)(flags), \
+#define __sanitizer_syscall_pre_mount(dev_name, dir_name, type, flags, data) \
+ __sanitizer_syscall_pre_impl_mount((long)(dev_name), (long)(dir_name), \
+ (long)(type), (long)(flags), \
(long)(data))
#define __sanitizer_syscall_post_mount(res, dev_name, dir_name, type, flags, \
data) \
__sanitizer_syscall_post_impl_mount(res, (long)(dev_name), (long)(dir_name), \
(long)(type), (long)(flags), \
(long)(data))
-#define __sanitizer_syscall_pre_umount(name, flags) \
+#define __sanitizer_syscall_pre_umount(name, flags) \
__sanitizer_syscall_pre_impl_umount((long)(name), (long)(flags))
-#define __sanitizer_syscall_post_umount(res, name, flags) \
+#define __sanitizer_syscall_post_umount(res, name, flags) \
__sanitizer_syscall_post_impl_umount(res, (long)(name), (long)(flags))
-#define __sanitizer_syscall_pre_oldumount(name) \
+#define __sanitizer_syscall_pre_oldumount(name) \
__sanitizer_syscall_pre_impl_oldumount((long)(name))
-#define __sanitizer_syscall_post_oldumount(res, name) \
+#define __sanitizer_syscall_post_oldumount(res, name) \
__sanitizer_syscall_post_impl_oldumount(res, (long)(name))
-#define __sanitizer_syscall_pre_truncate(path, length) \
+#define __sanitizer_syscall_pre_truncate(path, length) \
__sanitizer_syscall_pre_impl_truncate((long)(path), (long)(length))
-#define __sanitizer_syscall_post_truncate(res, path, length) \
+#define __sanitizer_syscall_post_truncate(res, path, length) \
__sanitizer_syscall_post_impl_truncate(res, (long)(path), (long)(length))
-#define __sanitizer_syscall_pre_ftruncate(fd, length) \
+#define __sanitizer_syscall_pre_ftruncate(fd, length) \
__sanitizer_syscall_pre_impl_ftruncate((long)(fd), (long)(length))
-#define __sanitizer_syscall_post_ftruncate(res, fd, length) \
+#define __sanitizer_syscall_post_ftruncate(res, fd, length) \
__sanitizer_syscall_post_impl_ftruncate(res, (long)(fd), (long)(length))
-#define __sanitizer_syscall_pre_stat(filename, statbuf) \
+#define __sanitizer_syscall_pre_stat(filename, statbuf) \
__sanitizer_syscall_pre_impl_stat((long)(filename), (long)(statbuf))
-#define __sanitizer_syscall_post_stat(res, filename, statbuf) \
+#define __sanitizer_syscall_post_stat(res, filename, statbuf) \
__sanitizer_syscall_post_impl_stat(res, (long)(filename), (long)(statbuf))
-#define __sanitizer_syscall_pre_statfs(path, buf) \
+#define __sanitizer_syscall_pre_statfs(path, buf) \
__sanitizer_syscall_pre_impl_statfs((long)(path), (long)(buf))
-#define __sanitizer_syscall_post_statfs(res, path, buf) \
+#define __sanitizer_syscall_post_statfs(res, path, buf) \
__sanitizer_syscall_post_impl_statfs(res, (long)(path), (long)(buf))
-#define __sanitizer_syscall_pre_statfs64(path, sz, buf) \
+#define __sanitizer_syscall_pre_statfs64(path, sz, buf) \
__sanitizer_syscall_pre_impl_statfs64((long)(path), (long)(sz), (long)(buf))
-#define __sanitizer_syscall_post_statfs64(res, path, sz, buf) \
- __sanitizer_syscall_post_impl_statfs64(res, (long)(path), (long)(sz), \
+#define __sanitizer_syscall_post_statfs64(res, path, sz, buf) \
+ __sanitizer_syscall_post_impl_statfs64(res, (long)(path), (long)(sz), \
(long)(buf))
-#define __sanitizer_syscall_pre_fstatfs(fd, buf) \
+#define __sanitizer_syscall_pre_fstatfs(fd, buf) \
__sanitizer_syscall_pre_impl_fstatfs((long)(fd), (long)(buf))
-#define __sanitizer_syscall_post_fstatfs(res, fd, buf) \
+#define __sanitizer_syscall_post_fstatfs(res, fd, buf) \
__sanitizer_syscall_post_impl_fstatfs(res, (long)(fd), (long)(buf))
-#define __sanitizer_syscall_pre_fstatfs64(fd, sz, buf) \
+#define __sanitizer_syscall_pre_fstatfs64(fd, sz, buf) \
__sanitizer_syscall_pre_impl_fstatfs64((long)(fd), (long)(sz), (long)(buf))
-#define __sanitizer_syscall_post_fstatfs64(res, fd, sz, buf) \
- __sanitizer_syscall_post_impl_fstatfs64(res, (long)(fd), (long)(sz), \
+#define __sanitizer_syscall_post_fstatfs64(res, fd, sz, buf) \
+ __sanitizer_syscall_post_impl_fstatfs64(res, (long)(fd), (long)(sz), \
(long)(buf))
-#define __sanitizer_syscall_pre_lstat(filename, statbuf) \
+#define __sanitizer_syscall_pre_lstat(filename, statbuf) \
__sanitizer_syscall_pre_impl_lstat((long)(filename), (long)(statbuf))
-#define __sanitizer_syscall_post_lstat(res, filename, statbuf) \
+#define __sanitizer_syscall_post_lstat(res, filename, statbuf) \
__sanitizer_syscall_post_impl_lstat(res, (long)(filename), (long)(statbuf))
-#define __sanitizer_syscall_pre_fstat(fd, statbuf) \
+#define __sanitizer_syscall_pre_fstat(fd, statbuf) \
__sanitizer_syscall_pre_impl_fstat((long)(fd), (long)(statbuf))
-#define __sanitizer_syscall_post_fstat(res, fd, statbuf) \
+#define __sanitizer_syscall_post_fstat(res, fd, statbuf) \
__sanitizer_syscall_post_impl_fstat(res, (long)(fd), (long)(statbuf))
-#define __sanitizer_syscall_pre_newstat(filename, statbuf) \
+#define __sanitizer_syscall_pre_newstat(filename, statbuf) \
__sanitizer_syscall_pre_impl_newstat((long)(filename), (long)(statbuf))
-#define __sanitizer_syscall_post_newstat(res, filename, statbuf) \
+#define __sanitizer_syscall_post_newstat(res, filename, statbuf) \
__sanitizer_syscall_post_impl_newstat(res, (long)(filename), (long)(statbuf))
-#define __sanitizer_syscall_pre_newlstat(filename, statbuf) \
+#define __sanitizer_syscall_pre_newlstat(filename, statbuf) \
__sanitizer_syscall_pre_impl_newlstat((long)(filename), (long)(statbuf))
-#define __sanitizer_syscall_post_newlstat(res, filename, statbuf) \
+#define __sanitizer_syscall_post_newlstat(res, filename, statbuf) \
__sanitizer_syscall_post_impl_newlstat(res, (long)(filename), (long)(statbuf))
-#define __sanitizer_syscall_pre_newfstat(fd, statbuf) \
+#define __sanitizer_syscall_pre_newfstat(fd, statbuf) \
__sanitizer_syscall_pre_impl_newfstat((long)(fd), (long)(statbuf))
-#define __sanitizer_syscall_post_newfstat(res, fd, statbuf) \
+#define __sanitizer_syscall_post_newfstat(res, fd, statbuf) \
__sanitizer_syscall_post_impl_newfstat(res, (long)(fd), (long)(statbuf))
-#define __sanitizer_syscall_pre_ustat(dev, ubuf) \
+#define __sanitizer_syscall_pre_ustat(dev, ubuf) \
__sanitizer_syscall_pre_impl_ustat((long)(dev), (long)(ubuf))
-#define __sanitizer_syscall_post_ustat(res, dev, ubuf) \
+#define __sanitizer_syscall_post_ustat(res, dev, ubuf) \
__sanitizer_syscall_post_impl_ustat(res, (long)(dev), (long)(ubuf))
-#define __sanitizer_syscall_pre_stat64(filename, statbuf) \
+#define __sanitizer_syscall_pre_stat64(filename, statbuf) \
__sanitizer_syscall_pre_impl_stat64((long)(filename), (long)(statbuf))
-#define __sanitizer_syscall_post_stat64(res, filename, statbuf) \
+#define __sanitizer_syscall_post_stat64(res, filename, statbuf) \
__sanitizer_syscall_post_impl_stat64(res, (long)(filename), (long)(statbuf))
-#define __sanitizer_syscall_pre_fstat64(fd, statbuf) \
+#define __sanitizer_syscall_pre_fstat64(fd, statbuf) \
__sanitizer_syscall_pre_impl_fstat64((long)(fd), (long)(statbuf))
-#define __sanitizer_syscall_post_fstat64(res, fd, statbuf) \
+#define __sanitizer_syscall_post_fstat64(res, fd, statbuf) \
__sanitizer_syscall_post_impl_fstat64(res, (long)(fd), (long)(statbuf))
-#define __sanitizer_syscall_pre_lstat64(filename, statbuf) \
+#define __sanitizer_syscall_pre_lstat64(filename, statbuf) \
__sanitizer_syscall_pre_impl_lstat64((long)(filename), (long)(statbuf))
-#define __sanitizer_syscall_post_lstat64(res, filename, statbuf) \
+#define __sanitizer_syscall_post_lstat64(res, filename, statbuf) \
__sanitizer_syscall_post_impl_lstat64(res, (long)(filename), (long)(statbuf))
-#define __sanitizer_syscall_pre_setxattr(path, name, value, size, flags) \
- __sanitizer_syscall_pre_impl_setxattr( \
+#define __sanitizer_syscall_pre_setxattr(path, name, value, size, flags) \
+ __sanitizer_syscall_pre_impl_setxattr( \
(long)(path), (long)(name), (long)(value), (long)(size), (long)(flags))
#define __sanitizer_syscall_post_setxattr(res, path, name, value, size, flags) \
__sanitizer_syscall_post_impl_setxattr(res, (long)(path), (long)(name), \
(long)(value), (long)(size), \
(long)(flags))
-#define __sanitizer_syscall_pre_lsetxattr(path, name, value, size, flags) \
- __sanitizer_syscall_pre_impl_lsetxattr( \
+#define __sanitizer_syscall_pre_lsetxattr(path, name, value, size, flags) \
+ __sanitizer_syscall_pre_impl_lsetxattr( \
(long)(path), (long)(name), (long)(value), (long)(size), (long)(flags))
-#define __sanitizer_syscall_post_lsetxattr(res, path, name, value, size, \
- flags) \
- __sanitizer_syscall_post_impl_lsetxattr(res, (long)(path), (long)(name), \
- (long)(value), (long)(size), \
+#define __sanitizer_syscall_post_lsetxattr(res, path, name, value, size, \
+ flags) \
+ __sanitizer_syscall_post_impl_lsetxattr(res, (long)(path), (long)(name), \
+ (long)(value), (long)(size), \
(long)(flags))
-#define __sanitizer_syscall_pre_fsetxattr(fd, name, value, size, flags) \
- __sanitizer_syscall_pre_impl_fsetxattr( \
+#define __sanitizer_syscall_pre_fsetxattr(fd, name, value, size, flags) \
+ __sanitizer_syscall_pre_impl_fsetxattr( \
(long)(fd), (long)(name), (long)(value), (long)(size), (long)(flags))
-#define __sanitizer_syscall_post_fsetxattr(res, fd, name, value, size, flags) \
- __sanitizer_syscall_post_impl_fsetxattr(res, (long)(fd), (long)(name), \
- (long)(value), (long)(size), \
+#define __sanitizer_syscall_post_fsetxattr(res, fd, name, value, size, flags) \
+ __sanitizer_syscall_post_impl_fsetxattr(res, (long)(fd), (long)(name), \
+ (long)(value), (long)(size), \
(long)(flags))
-#define __sanitizer_syscall_pre_getxattr(path, name, value, size) \
- __sanitizer_syscall_pre_impl_getxattr((long)(path), (long)(name), \
+#define __sanitizer_syscall_pre_getxattr(path, name, value, size) \
+ __sanitizer_syscall_pre_impl_getxattr((long)(path), (long)(name), \
(long)(value), (long)(size))
-#define __sanitizer_syscall_post_getxattr(res, path, name, value, size) \
- __sanitizer_syscall_post_impl_getxattr(res, (long)(path), (long)(name), \
+#define __sanitizer_syscall_post_getxattr(res, path, name, value, size) \
+ __sanitizer_syscall_post_impl_getxattr(res, (long)(path), (long)(name), \
(long)(value), (long)(size))
-#define __sanitizer_syscall_pre_lgetxattr(path, name, value, size) \
- __sanitizer_syscall_pre_impl_lgetxattr((long)(path), (long)(name), \
+#define __sanitizer_syscall_pre_lgetxattr(path, name, value, size) \
+ __sanitizer_syscall_pre_impl_lgetxattr((long)(path), (long)(name), \
(long)(value), (long)(size))
-#define __sanitizer_syscall_post_lgetxattr(res, path, name, value, size) \
- __sanitizer_syscall_post_impl_lgetxattr(res, (long)(path), (long)(name), \
+#define __sanitizer_syscall_post_lgetxattr(res, path, name, value, size) \
+ __sanitizer_syscall_post_impl_lgetxattr(res, (long)(path), (long)(name), \
(long)(value), (long)(size))
-#define __sanitizer_syscall_pre_fgetxattr(fd, name, value, size) \
- __sanitizer_syscall_pre_impl_fgetxattr((long)(fd), (long)(name), \
+#define __sanitizer_syscall_pre_fgetxattr(fd, name, value, size) \
+ __sanitizer_syscall_pre_impl_fgetxattr((long)(fd), (long)(name), \
(long)(value), (long)(size))
-#define __sanitizer_syscall_post_fgetxattr(res, fd, name, value, size) \
- __sanitizer_syscall_post_impl_fgetxattr(res, (long)(fd), (long)(name), \
+#define __sanitizer_syscall_post_fgetxattr(res, fd, name, value, size) \
+ __sanitizer_syscall_post_impl_fgetxattr(res, (long)(fd), (long)(name), \
(long)(value), (long)(size))
-#define __sanitizer_syscall_pre_listxattr(path, list, size) \
- __sanitizer_syscall_pre_impl_listxattr((long)(path), (long)(list), \
+#define __sanitizer_syscall_pre_listxattr(path, list, size) \
+ __sanitizer_syscall_pre_impl_listxattr((long)(path), (long)(list), \
(long)(size))
-#define __sanitizer_syscall_post_listxattr(res, path, list, size) \
- __sanitizer_syscall_post_impl_listxattr(res, (long)(path), (long)(list), \
+#define __sanitizer_syscall_post_listxattr(res, path, list, size) \
+ __sanitizer_syscall_post_impl_listxattr(res, (long)(path), (long)(list), \
(long)(size))
-#define __sanitizer_syscall_pre_llistxattr(path, list, size) \
- __sanitizer_syscall_pre_impl_llistxattr((long)(path), (long)(list), \
+#define __sanitizer_syscall_pre_llistxattr(path, list, size) \
+ __sanitizer_syscall_pre_impl_llistxattr((long)(path), (long)(list), \
(long)(size))
-#define __sanitizer_syscall_post_llistxattr(res, path, list, size) \
- __sanitizer_syscall_post_impl_llistxattr(res, (long)(path), (long)(list), \
+#define __sanitizer_syscall_post_llistxattr(res, path, list, size) \
+ __sanitizer_syscall_post_impl_llistxattr(res, (long)(path), (long)(list), \
(long)(size))
-#define __sanitizer_syscall_pre_flistxattr(fd, list, size) \
- __sanitizer_syscall_pre_impl_flistxattr((long)(fd), (long)(list), \
+#define __sanitizer_syscall_pre_flistxattr(fd, list, size) \
+ __sanitizer_syscall_pre_impl_flistxattr((long)(fd), (long)(list), \
(long)(size))
-#define __sanitizer_syscall_post_flistxattr(res, fd, list, size) \
- __sanitizer_syscall_post_impl_flistxattr(res, (long)(fd), (long)(list), \
+#define __sanitizer_syscall_post_flistxattr(res, fd, list, size) \
+ __sanitizer_syscall_post_impl_flistxattr(res, (long)(fd), (long)(list), \
(long)(size))
-#define __sanitizer_syscall_pre_removexattr(path, name) \
+#define __sanitizer_syscall_pre_removexattr(path, name) \
__sanitizer_syscall_pre_impl_removexattr((long)(path), (long)(name))
-#define __sanitizer_syscall_post_removexattr(res, path, name) \
+#define __sanitizer_syscall_post_removexattr(res, path, name) \
__sanitizer_syscall_post_impl_removexattr(res, (long)(path), (long)(name))
-#define __sanitizer_syscall_pre_lremovexattr(path, name) \
+#define __sanitizer_syscall_pre_lremovexattr(path, name) \
__sanitizer_syscall_pre_impl_lremovexattr((long)(path), (long)(name))
-#define __sanitizer_syscall_post_lremovexattr(res, path, name) \
+#define __sanitizer_syscall_post_lremovexattr(res, path, name) \
__sanitizer_syscall_post_impl_lremovexattr(res, (long)(path), (long)(name))
-#define __sanitizer_syscall_pre_fremovexattr(fd, name) \
+#define __sanitizer_syscall_pre_fremovexattr(fd, name) \
__sanitizer_syscall_pre_impl_fremovexattr((long)(fd), (long)(name))
-#define __sanitizer_syscall_post_fremovexattr(res, fd, name) \
+#define __sanitizer_syscall_post_fremovexattr(res, fd, name) \
__sanitizer_syscall_post_impl_fremovexattr(res, (long)(fd), (long)(name))
-#define __sanitizer_syscall_pre_brk(brk) \
+#define __sanitizer_syscall_pre_brk(brk) \
__sanitizer_syscall_pre_impl_brk((long)(brk))
-#define __sanitizer_syscall_post_brk(res, brk) \
+#define __sanitizer_syscall_post_brk(res, brk) \
__sanitizer_syscall_post_impl_brk(res, (long)(brk))
-#define __sanitizer_syscall_pre_mprotect(start, len, prot) \
- __sanitizer_syscall_pre_impl_mprotect((long)(start), (long)(len), \
+#define __sanitizer_syscall_pre_mprotect(start, len, prot) \
+ __sanitizer_syscall_pre_impl_mprotect((long)(start), (long)(len), \
(long)(prot))
-#define __sanitizer_syscall_post_mprotect(res, start, len, prot) \
- __sanitizer_syscall_post_impl_mprotect(res, (long)(start), (long)(len), \
+#define __sanitizer_syscall_post_mprotect(res, start, len, prot) \
+ __sanitizer_syscall_post_impl_mprotect(res, (long)(start), (long)(len), \
(long)(prot))
-#define __sanitizer_syscall_pre_mremap(addr, old_len, new_len, flags, \
- new_addr) \
- __sanitizer_syscall_pre_impl_mremap((long)(addr), (long)(old_len), \
- (long)(new_len), (long)(flags), \
+#define __sanitizer_syscall_pre_mremap(addr, old_len, new_len, flags, \
+ new_addr) \
+ __sanitizer_syscall_pre_impl_mremap((long)(addr), (long)(old_len), \
+ (long)(new_len), (long)(flags), \
(long)(new_addr))
-#define __sanitizer_syscall_post_mremap(res, addr, old_len, new_len, flags, \
- new_addr) \
- __sanitizer_syscall_post_impl_mremap(res, (long)(addr), (long)(old_len), \
- (long)(new_len), (long)(flags), \
+#define __sanitizer_syscall_post_mremap(res, addr, old_len, new_len, flags, \
+ new_addr) \
+ __sanitizer_syscall_post_impl_mremap(res, (long)(addr), (long)(old_len), \
+ (long)(new_len), (long)(flags), \
(long)(new_addr))
-#define __sanitizer_syscall_pre_remap_file_pages(start, size, prot, pgoff, \
- flags) \
- __sanitizer_syscall_pre_impl_remap_file_pages( \
+#define __sanitizer_syscall_pre_remap_file_pages(start, size, prot, pgoff, \
+ flags) \
+ __sanitizer_syscall_pre_impl_remap_file_pages( \
(long)(start), (long)(size), (long)(prot), (long)(pgoff), (long)(flags))
-#define __sanitizer_syscall_post_remap_file_pages(res, start, size, prot, \
- pgoff, flags) \
- __sanitizer_syscall_post_impl_remap_file_pages(res, (long)(start), \
- (long)(size), (long)(prot), \
+#define __sanitizer_syscall_post_remap_file_pages(res, start, size, prot, \
+ pgoff, flags) \
+ __sanitizer_syscall_post_impl_remap_file_pages(res, (long)(start), \
+ (long)(size), (long)(prot), \
(long)(pgoff), (long)(flags))
-#define __sanitizer_syscall_pre_msync(start, len, flags) \
+#define __sanitizer_syscall_pre_msync(start, len, flags) \
__sanitizer_syscall_pre_impl_msync((long)(start), (long)(len), (long)(flags))
-#define __sanitizer_syscall_post_msync(res, start, len, flags) \
- __sanitizer_syscall_post_impl_msync(res, (long)(start), (long)(len), \
+#define __sanitizer_syscall_post_msync(res, start, len, flags) \
+ __sanitizer_syscall_post_impl_msync(res, (long)(start), (long)(len), \
(long)(flags))
-#define __sanitizer_syscall_pre_munmap(addr, len) \
+#define __sanitizer_syscall_pre_munmap(addr, len) \
__sanitizer_syscall_pre_impl_munmap((long)(addr), (long)(len))
-#define __sanitizer_syscall_post_munmap(res, addr, len) \
+#define __sanitizer_syscall_post_munmap(res, addr, len) \
__sanitizer_syscall_post_impl_munmap(res, (long)(addr), (long)(len))
-#define __sanitizer_syscall_pre_mlock(start, len) \
+#define __sanitizer_syscall_pre_mlock(start, len) \
__sanitizer_syscall_pre_impl_mlock((long)(start), (long)(len))
-#define __sanitizer_syscall_post_mlock(res, start, len) \
+#define __sanitizer_syscall_post_mlock(res, start, len) \
__sanitizer_syscall_post_impl_mlock(res, (long)(start), (long)(len))
-#define __sanitizer_syscall_pre_munlock(start, len) \
+#define __sanitizer_syscall_pre_munlock(start, len) \
__sanitizer_syscall_pre_impl_munlock((long)(start), (long)(len))
-#define __sanitizer_syscall_post_munlock(res, start, len) \
+#define __sanitizer_syscall_post_munlock(res, start, len) \
__sanitizer_syscall_post_impl_munlock(res, (long)(start), (long)(len))
-#define __sanitizer_syscall_pre_mlockall(flags) \
+#define __sanitizer_syscall_pre_mlockall(flags) \
__sanitizer_syscall_pre_impl_mlockall((long)(flags))
-#define __sanitizer_syscall_post_mlockall(res, flags) \
+#define __sanitizer_syscall_post_mlockall(res, flags) \
__sanitizer_syscall_post_impl_mlockall(res, (long)(flags))
-#define __sanitizer_syscall_pre_munlockall() \
+#define __sanitizer_syscall_pre_munlockall() \
__sanitizer_syscall_pre_impl_munlockall()
-#define __sanitizer_syscall_post_munlockall(res) \
+#define __sanitizer_syscall_post_munlockall(res) \
__sanitizer_syscall_post_impl_munlockall(res)
-#define __sanitizer_syscall_pre_madvise(start, len, behavior) \
- __sanitizer_syscall_pre_impl_madvise((long)(start), (long)(len), \
+#define __sanitizer_syscall_pre_madvise(start, len, behavior) \
+ __sanitizer_syscall_pre_impl_madvise((long)(start), (long)(len), \
(long)(behavior))
-#define __sanitizer_syscall_post_madvise(res, start, len, behavior) \
- __sanitizer_syscall_post_impl_madvise(res, (long)(start), (long)(len), \
+#define __sanitizer_syscall_post_madvise(res, start, len, behavior) \
+ __sanitizer_syscall_post_impl_madvise(res, (long)(start), (long)(len), \
(long)(behavior))
-#define __sanitizer_syscall_pre_mincore(start, len, vec) \
+#define __sanitizer_syscall_pre_mincore(start, len, vec) \
__sanitizer_syscall_pre_impl_mincore((long)(start), (long)(len), (long)(vec))
-#define __sanitizer_syscall_post_mincore(res, start, len, vec) \
- __sanitizer_syscall_post_impl_mincore(res, (long)(start), (long)(len), \
+#define __sanitizer_syscall_post_mincore(res, start, len, vec) \
+ __sanitizer_syscall_post_impl_mincore(res, (long)(start), (long)(len), \
(long)(vec))
-#define __sanitizer_syscall_pre_pivot_root(new_root, put_old) \
+#define __sanitizer_syscall_pre_pivot_root(new_root, put_old) \
__sanitizer_syscall_pre_impl_pivot_root((long)(new_root), (long)(put_old))
-#define __sanitizer_syscall_post_pivot_root(res, new_root, put_old) \
- __sanitizer_syscall_post_impl_pivot_root(res, (long)(new_root), \
+#define __sanitizer_syscall_post_pivot_root(res, new_root, put_old) \
+ __sanitizer_syscall_post_impl_pivot_root(res, (long)(new_root), \
(long)(put_old))
-#define __sanitizer_syscall_pre_chroot(filename) \
+#define __sanitizer_syscall_pre_chroot(filename) \
__sanitizer_syscall_pre_impl_chroot((long)(filename))
-#define __sanitizer_syscall_post_chroot(res, filename) \
+#define __sanitizer_syscall_post_chroot(res, filename) \
__sanitizer_syscall_post_impl_chroot(res, (long)(filename))
-#define __sanitizer_syscall_pre_mknod(filename, mode, dev) \
- __sanitizer_syscall_pre_impl_mknod((long)(filename), (long)(mode), \
+#define __sanitizer_syscall_pre_mknod(filename, mode, dev) \
+ __sanitizer_syscall_pre_impl_mknod((long)(filename), (long)(mode), \
(long)(dev))
-#define __sanitizer_syscall_post_mknod(res, filename, mode, dev) \
- __sanitizer_syscall_post_impl_mknod(res, (long)(filename), (long)(mode), \
+#define __sanitizer_syscall_post_mknod(res, filename, mode, dev) \
+ __sanitizer_syscall_post_impl_mknod(res, (long)(filename), (long)(mode), \
(long)(dev))
-#define __sanitizer_syscall_pre_link(oldname, newname) \
+#define __sanitizer_syscall_pre_link(oldname, newname) \
__sanitizer_syscall_pre_impl_link((long)(oldname), (long)(newname))
-#define __sanitizer_syscall_post_link(res, oldname, newname) \
+#define __sanitizer_syscall_post_link(res, oldname, newname) \
__sanitizer_syscall_post_impl_link(res, (long)(oldname), (long)(newname))
-#define __sanitizer_syscall_pre_symlink(old, new_) \
+#define __sanitizer_syscall_pre_symlink(old, new_) \
__sanitizer_syscall_pre_impl_symlink((long)(old), (long)(new_))
-#define __sanitizer_syscall_post_symlink(res, old, new_) \
+#define __sanitizer_syscall_post_symlink(res, old, new_) \
__sanitizer_syscall_post_impl_symlink(res, (long)(old), (long)(new_))
-#define __sanitizer_syscall_pre_unlink(pathname) \
+#define __sanitizer_syscall_pre_unlink(pathname) \
__sanitizer_syscall_pre_impl_unlink((long)(pathname))
-#define __sanitizer_syscall_post_unlink(res, pathname) \
+#define __sanitizer_syscall_post_unlink(res, pathname) \
__sanitizer_syscall_post_impl_unlink(res, (long)(pathname))
-#define __sanitizer_syscall_pre_rename(oldname, newname) \
+#define __sanitizer_syscall_pre_rename(oldname, newname) \
__sanitizer_syscall_pre_impl_rename((long)(oldname), (long)(newname))
-#define __sanitizer_syscall_post_rename(res, oldname, newname) \
+#define __sanitizer_syscall_post_rename(res, oldname, newname) \
__sanitizer_syscall_post_impl_rename(res, (long)(oldname), (long)(newname))
-#define __sanitizer_syscall_pre_chmod(filename, mode) \
+#define __sanitizer_syscall_pre_chmod(filename, mode) \
__sanitizer_syscall_pre_impl_chmod((long)(filename), (long)(mode))
-#define __sanitizer_syscall_post_chmod(res, filename, mode) \
+#define __sanitizer_syscall_post_chmod(res, filename, mode) \
__sanitizer_syscall_post_impl_chmod(res, (long)(filename), (long)(mode))
-#define __sanitizer_syscall_pre_fchmod(fd, mode) \
+#define __sanitizer_syscall_pre_fchmod(fd, mode) \
__sanitizer_syscall_pre_impl_fchmod((long)(fd), (long)(mode))
-#define __sanitizer_syscall_post_fchmod(res, fd, mode) \
+#define __sanitizer_syscall_post_fchmod(res, fd, mode) \
__sanitizer_syscall_post_impl_fchmod(res, (long)(fd), (long)(mode))
-#define __sanitizer_syscall_pre_fcntl(fd, cmd, arg) \
+#define __sanitizer_syscall_pre_fcntl(fd, cmd, arg) \
__sanitizer_syscall_pre_impl_fcntl((long)(fd), (long)(cmd), (long)(arg))
-#define __sanitizer_syscall_post_fcntl(res, fd, cmd, arg) \
+#define __sanitizer_syscall_post_fcntl(res, fd, cmd, arg) \
__sanitizer_syscall_post_impl_fcntl(res, (long)(fd), (long)(cmd), (long)(arg))
-#define __sanitizer_syscall_pre_fcntl64(fd, cmd, arg) \
+#define __sanitizer_syscall_pre_fcntl64(fd, cmd, arg) \
__sanitizer_syscall_pre_impl_fcntl64((long)(fd), (long)(cmd), (long)(arg))
-#define __sanitizer_syscall_post_fcntl64(res, fd, cmd, arg) \
- __sanitizer_syscall_post_impl_fcntl64(res, (long)(fd), (long)(cmd), \
+#define __sanitizer_syscall_post_fcntl64(res, fd, cmd, arg) \
+ __sanitizer_syscall_post_impl_fcntl64(res, (long)(fd), (long)(cmd), \
(long)(arg))
-#define __sanitizer_syscall_pre_pipe(fildes) \
+#define __sanitizer_syscall_pre_pipe(fildes) \
__sanitizer_syscall_pre_impl_pipe((long)(fildes))
-#define __sanitizer_syscall_post_pipe(res, fildes) \
+#define __sanitizer_syscall_post_pipe(res, fildes) \
__sanitizer_syscall_post_impl_pipe(res, (long)(fildes))
-#define __sanitizer_syscall_pre_pipe2(fildes, flags) \
+#define __sanitizer_syscall_pre_pipe2(fildes, flags) \
__sanitizer_syscall_pre_impl_pipe2((long)(fildes), (long)(flags))
-#define __sanitizer_syscall_post_pipe2(res, fildes, flags) \
+#define __sanitizer_syscall_post_pipe2(res, fildes, flags) \
__sanitizer_syscall_post_impl_pipe2(res, (long)(fildes), (long)(flags))
-#define __sanitizer_syscall_pre_dup(fildes) \
+#define __sanitizer_syscall_pre_dup(fildes) \
__sanitizer_syscall_pre_impl_dup((long)(fildes))
-#define __sanitizer_syscall_post_dup(res, fildes) \
+#define __sanitizer_syscall_post_dup(res, fildes) \
__sanitizer_syscall_post_impl_dup(res, (long)(fildes))
-#define __sanitizer_syscall_pre_dup2(oldfd, newfd) \
+#define __sanitizer_syscall_pre_dup2(oldfd, newfd) \
__sanitizer_syscall_pre_impl_dup2((long)(oldfd), (long)(newfd))
-#define __sanitizer_syscall_post_dup2(res, oldfd, newfd) \
+#define __sanitizer_syscall_post_dup2(res, oldfd, newfd) \
__sanitizer_syscall_post_impl_dup2(res, (long)(oldfd), (long)(newfd))
-#define __sanitizer_syscall_pre_dup3(oldfd, newfd, flags) \
+#define __sanitizer_syscall_pre_dup3(oldfd, newfd, flags) \
__sanitizer_syscall_pre_impl_dup3((long)(oldfd), (long)(newfd), (long)(flags))
-#define __sanitizer_syscall_post_dup3(res, oldfd, newfd, flags) \
- __sanitizer_syscall_post_impl_dup3(res, (long)(oldfd), (long)(newfd), \
+#define __sanitizer_syscall_post_dup3(res, oldfd, newfd, flags) \
+ __sanitizer_syscall_post_impl_dup3(res, (long)(oldfd), (long)(newfd), \
(long)(flags))
-#define __sanitizer_syscall_pre_ioperm(from, num, on) \
+#define __sanitizer_syscall_pre_ioperm(from, num, on) \
__sanitizer_syscall_pre_impl_ioperm((long)(from), (long)(num), (long)(on))
-#define __sanitizer_syscall_post_ioperm(res, from, num, on) \
- __sanitizer_syscall_post_impl_ioperm(res, (long)(from), (long)(num), \
+#define __sanitizer_syscall_post_ioperm(res, from, num, on) \
+ __sanitizer_syscall_post_impl_ioperm(res, (long)(from), (long)(num), \
(long)(on))
-#define __sanitizer_syscall_pre_ioctl(fd, cmd, arg) \
+#define __sanitizer_syscall_pre_ioctl(fd, cmd, arg) \
__sanitizer_syscall_pre_impl_ioctl((long)(fd), (long)(cmd), (long)(arg))
-#define __sanitizer_syscall_post_ioctl(res, fd, cmd, arg) \
+#define __sanitizer_syscall_post_ioctl(res, fd, cmd, arg) \
__sanitizer_syscall_post_impl_ioctl(res, (long)(fd), (long)(cmd), (long)(arg))
-#define __sanitizer_syscall_pre_flock(fd, cmd) \
+#define __sanitizer_syscall_pre_flock(fd, cmd) \
__sanitizer_syscall_pre_impl_flock((long)(fd), (long)(cmd))
-#define __sanitizer_syscall_post_flock(res, fd, cmd) \
+#define __sanitizer_syscall_post_flock(res, fd, cmd) \
__sanitizer_syscall_post_impl_flock(res, (long)(fd), (long)(cmd))
-#define __sanitizer_syscall_pre_io_setup(nr_reqs, ctx) \
+#define __sanitizer_syscall_pre_io_setup(nr_reqs, ctx) \
__sanitizer_syscall_pre_impl_io_setup((long)(nr_reqs), (long)(ctx))
-#define __sanitizer_syscall_post_io_setup(res, nr_reqs, ctx) \
+#define __sanitizer_syscall_post_io_setup(res, nr_reqs, ctx) \
__sanitizer_syscall_post_impl_io_setup(res, (long)(nr_reqs), (long)(ctx))
-#define __sanitizer_syscall_pre_io_destroy(ctx) \
+#define __sanitizer_syscall_pre_io_destroy(ctx) \
__sanitizer_syscall_pre_impl_io_destroy((long)(ctx))
-#define __sanitizer_syscall_post_io_destroy(res, ctx) \
+#define __sanitizer_syscall_post_io_destroy(res, ctx) \
__sanitizer_syscall_post_impl_io_destroy(res, (long)(ctx))
-#define __sanitizer_syscall_pre_io_getevents(ctx_id, min_nr, nr, events, \
- timeout) \
- __sanitizer_syscall_pre_impl_io_getevents((long)(ctx_id), (long)(min_nr), \
- (long)(nr), (long)(events), \
+#define __sanitizer_syscall_pre_io_getevents(ctx_id, min_nr, nr, events, \
+ timeout) \
+ __sanitizer_syscall_pre_impl_io_getevents((long)(ctx_id), (long)(min_nr), \
+ (long)(nr), (long)(events), \
(long)(timeout))
#define __sanitizer_syscall_post_io_getevents(res, ctx_id, min_nr, nr, events, \
timeout) \
__sanitizer_syscall_post_impl_io_getevents(res, (long)(ctx_id), \
(long)(min_nr), (long)(nr), \
(long)(events), (long)(timeout))
-#define __sanitizer_syscall_pre_io_submit(ctx_id, arg1, arg2) \
- __sanitizer_syscall_pre_impl_io_submit((long)(ctx_id), (long)(arg1), \
+#define __sanitizer_syscall_pre_io_submit(ctx_id, arg1, arg2) \
+ __sanitizer_syscall_pre_impl_io_submit((long)(ctx_id), (long)(arg1), \
(long)(arg2))
-#define __sanitizer_syscall_post_io_submit(res, ctx_id, arg1, arg2) \
- __sanitizer_syscall_post_impl_io_submit(res, (long)(ctx_id), (long)(arg1), \
+#define __sanitizer_syscall_post_io_submit(res, ctx_id, arg1, arg2) \
+ __sanitizer_syscall_post_impl_io_submit(res, (long)(ctx_id), (long)(arg1), \
(long)(arg2))
-#define __sanitizer_syscall_pre_io_cancel(ctx_id, iocb, result) \
- __sanitizer_syscall_pre_impl_io_cancel((long)(ctx_id), (long)(iocb), \
+#define __sanitizer_syscall_pre_io_cancel(ctx_id, iocb, result) \
+ __sanitizer_syscall_pre_impl_io_cancel((long)(ctx_id), (long)(iocb), \
(long)(result))
-#define __sanitizer_syscall_post_io_cancel(res, ctx_id, iocb, result) \
- __sanitizer_syscall_post_impl_io_cancel(res, (long)(ctx_id), (long)(iocb), \
+#define __sanitizer_syscall_post_io_cancel(res, ctx_id, iocb, result) \
+ __sanitizer_syscall_post_impl_io_cancel(res, (long)(ctx_id), (long)(iocb), \
(long)(result))
-#define __sanitizer_syscall_pre_sendfile(out_fd, in_fd, offset, count) \
- __sanitizer_syscall_pre_impl_sendfile((long)(out_fd), (long)(in_fd), \
+#define __sanitizer_syscall_pre_sendfile(out_fd, in_fd, offset, count) \
+ __sanitizer_syscall_pre_impl_sendfile((long)(out_fd), (long)(in_fd), \
(long)(offset), (long)(count))
-#define __sanitizer_syscall_post_sendfile(res, out_fd, in_fd, offset, count) \
- __sanitizer_syscall_post_impl_sendfile(res, (long)(out_fd), (long)(in_fd), \
+#define __sanitizer_syscall_post_sendfile(res, out_fd, in_fd, offset, count) \
+ __sanitizer_syscall_post_impl_sendfile(res, (long)(out_fd), (long)(in_fd), \
(long)(offset), (long)(count))
-#define __sanitizer_syscall_pre_sendfile64(out_fd, in_fd, offset, count) \
- __sanitizer_syscall_pre_impl_sendfile64((long)(out_fd), (long)(in_fd), \
+#define __sanitizer_syscall_pre_sendfile64(out_fd, in_fd, offset, count) \
+ __sanitizer_syscall_pre_impl_sendfile64((long)(out_fd), (long)(in_fd), \
(long)(offset), (long)(count))
#define __sanitizer_syscall_post_sendfile64(res, out_fd, in_fd, offset, count) \
__sanitizer_syscall_post_impl_sendfile64(res, (long)(out_fd), (long)(in_fd), \
(long)(offset), (long)(count))
-#define __sanitizer_syscall_pre_readlink(path, buf, bufsiz) \
- __sanitizer_syscall_pre_impl_readlink((long)(path), (long)(buf), \
+#define __sanitizer_syscall_pre_readlink(path, buf, bufsiz) \
+ __sanitizer_syscall_pre_impl_readlink((long)(path), (long)(buf), \
(long)(bufsiz))
-#define __sanitizer_syscall_post_readlink(res, path, buf, bufsiz) \
- __sanitizer_syscall_post_impl_readlink(res, (long)(path), (long)(buf), \
+#define __sanitizer_syscall_post_readlink(res, path, buf, bufsiz) \
+ __sanitizer_syscall_post_impl_readlink(res, (long)(path), (long)(buf), \
(long)(bufsiz))
-#define __sanitizer_syscall_pre_creat(pathname, mode) \
+#define __sanitizer_syscall_pre_creat(pathname, mode) \
__sanitizer_syscall_pre_impl_creat((long)(pathname), (long)(mode))
-#define __sanitizer_syscall_post_creat(res, pathname, mode) \
+#define __sanitizer_syscall_post_creat(res, pathname, mode) \
__sanitizer_syscall_post_impl_creat(res, (long)(pathname), (long)(mode))
-#define __sanitizer_syscall_pre_open(filename, flags, mode) \
- __sanitizer_syscall_pre_impl_open((long)(filename), (long)(flags), \
+#define __sanitizer_syscall_pre_open(filename, flags, mode) \
+ __sanitizer_syscall_pre_impl_open((long)(filename), (long)(flags), \
(long)(mode))
-#define __sanitizer_syscall_post_open(res, filename, flags, mode) \
- __sanitizer_syscall_post_impl_open(res, (long)(filename), (long)(flags), \
+#define __sanitizer_syscall_post_open(res, filename, flags, mode) \
+ __sanitizer_syscall_post_impl_open(res, (long)(filename), (long)(flags), \
(long)(mode))
-#define __sanitizer_syscall_pre_close(fd) \
+#define __sanitizer_syscall_pre_close(fd) \
__sanitizer_syscall_pre_impl_close((long)(fd))
-#define __sanitizer_syscall_post_close(res, fd) \
+#define __sanitizer_syscall_post_close(res, fd) \
__sanitizer_syscall_post_impl_close(res, (long)(fd))
-#define __sanitizer_syscall_pre_access(filename, mode) \
+#define __sanitizer_syscall_pre_access(filename, mode) \
__sanitizer_syscall_pre_impl_access((long)(filename), (long)(mode))
-#define __sanitizer_syscall_post_access(res, filename, mode) \
+#define __sanitizer_syscall_post_access(res, filename, mode) \
__sanitizer_syscall_post_impl_access(res, (long)(filename), (long)(mode))
#define __sanitizer_syscall_pre_vhangup() __sanitizer_syscall_pre_impl_vhangup()
-#define __sanitizer_syscall_post_vhangup(res) \
+#define __sanitizer_syscall_post_vhangup(res) \
__sanitizer_syscall_post_impl_vhangup(res)
-#define __sanitizer_syscall_pre_chown(filename, user, group) \
- __sanitizer_syscall_pre_impl_chown((long)(filename), (long)(user), \
+#define __sanitizer_syscall_pre_chown(filename, user, group) \
+ __sanitizer_syscall_pre_impl_chown((long)(filename), (long)(user), \
(long)(group))
-#define __sanitizer_syscall_post_chown(res, filename, user, group) \
- __sanitizer_syscall_post_impl_chown(res, (long)(filename), (long)(user), \
+#define __sanitizer_syscall_post_chown(res, filename, user, group) \
+ __sanitizer_syscall_post_impl_chown(res, (long)(filename), (long)(user), \
(long)(group))
-#define __sanitizer_syscall_pre_lchown(filename, user, group) \
- __sanitizer_syscall_pre_impl_lchown((long)(filename), (long)(user), \
+#define __sanitizer_syscall_pre_lchown(filename, user, group) \
+ __sanitizer_syscall_pre_impl_lchown((long)(filename), (long)(user), \
(long)(group))
-#define __sanitizer_syscall_post_lchown(res, filename, user, group) \
- __sanitizer_syscall_post_impl_lchown(res, (long)(filename), (long)(user), \
+#define __sanitizer_syscall_post_lchown(res, filename, user, group) \
+ __sanitizer_syscall_post_impl_lchown(res, (long)(filename), (long)(user), \
(long)(group))
-#define __sanitizer_syscall_pre_fchown(fd, user, group) \
+#define __sanitizer_syscall_pre_fchown(fd, user, group) \
__sanitizer_syscall_pre_impl_fchown((long)(fd), (long)(user), (long)(group))
-#define __sanitizer_syscall_post_fchown(res, fd, user, group) \
- __sanitizer_syscall_post_impl_fchown(res, (long)(fd), (long)(user), \
+#define __sanitizer_syscall_post_fchown(res, fd, user, group) \
+ __sanitizer_syscall_post_impl_fchown(res, (long)(fd), (long)(user), \
(long)(group))
-#define __sanitizer_syscall_pre_chown16(filename, user, group) \
- __sanitizer_syscall_pre_impl_chown16((long)(filename), (long)user, \
+#define __sanitizer_syscall_pre_chown16(filename, user, group) \
+ __sanitizer_syscall_pre_impl_chown16((long)(filename), (long)user, \
(long)group)
-#define __sanitizer_syscall_post_chown16(res, filename, user, group) \
- __sanitizer_syscall_post_impl_chown16(res, (long)(filename), (long)user, \
+#define __sanitizer_syscall_post_chown16(res, filename, user, group) \
+ __sanitizer_syscall_post_impl_chown16(res, (long)(filename), (long)user, \
(long)group)
-#define __sanitizer_syscall_pre_lchown16(filename, user, group) \
- __sanitizer_syscall_pre_impl_lchown16((long)(filename), (long)user, \
+#define __sanitizer_syscall_pre_lchown16(filename, user, group) \
+ __sanitizer_syscall_pre_impl_lchown16((long)(filename), (long)user, \
(long)group)
-#define __sanitizer_syscall_post_lchown16(res, filename, user, group) \
- __sanitizer_syscall_post_impl_lchown16(res, (long)(filename), (long)user, \
+#define __sanitizer_syscall_post_lchown16(res, filename, user, group) \
+ __sanitizer_syscall_post_impl_lchown16(res, (long)(filename), (long)user, \
(long)group)
-#define __sanitizer_syscall_pre_fchown16(fd, user, group) \
+#define __sanitizer_syscall_pre_fchown16(fd, user, group) \
__sanitizer_syscall_pre_impl_fchown16((long)(fd), (long)user, (long)group)
-#define __sanitizer_syscall_post_fchown16(res, fd, user, group) \
- __sanitizer_syscall_post_impl_fchown16(res, (long)(fd), (long)user, \
+#define __sanitizer_syscall_post_fchown16(res, fd, user, group) \
+ __sanitizer_syscall_post_impl_fchown16(res, (long)(fd), (long)user, \
(long)group)
-#define __sanitizer_syscall_pre_setregid16(rgid, egid) \
+#define __sanitizer_syscall_pre_setregid16(rgid, egid) \
__sanitizer_syscall_pre_impl_setregid16((long)rgid, (long)egid)
-#define __sanitizer_syscall_post_setregid16(res, rgid, egid) \
+#define __sanitizer_syscall_post_setregid16(res, rgid, egid) \
__sanitizer_syscall_post_impl_setregid16(res, (long)rgid, (long)egid)
-#define __sanitizer_syscall_pre_setgid16(gid) \
+#define __sanitizer_syscall_pre_setgid16(gid) \
__sanitizer_syscall_pre_impl_setgid16((long)gid)
-#define __sanitizer_syscall_post_setgid16(res, gid) \
+#define __sanitizer_syscall_post_setgid16(res, gid) \
__sanitizer_syscall_post_impl_setgid16(res, (long)gid)
-#define __sanitizer_syscall_pre_setreuid16(ruid, euid) \
+#define __sanitizer_syscall_pre_setreuid16(ruid, euid) \
__sanitizer_syscall_pre_impl_setreuid16((long)ruid, (long)euid)
-#define __sanitizer_syscall_post_setreuid16(res, ruid, euid) \
+#define __sanitizer_syscall_post_setreuid16(res, ruid, euid) \
__sanitizer_syscall_post_impl_setreuid16(res, (long)ruid, (long)euid)
-#define __sanitizer_syscall_pre_setuid16(uid) \
+#define __sanitizer_syscall_pre_setuid16(uid) \
__sanitizer_syscall_pre_impl_setuid16((long)uid)
-#define __sanitizer_syscall_post_setuid16(res, uid) \
+#define __sanitizer_syscall_post_setuid16(res, uid) \
__sanitizer_syscall_post_impl_setuid16(res, (long)uid)
-#define __sanitizer_syscall_pre_setresuid16(ruid, euid, suid) \
+#define __sanitizer_syscall_pre_setresuid16(ruid, euid, suid) \
__sanitizer_syscall_pre_impl_setresuid16((long)ruid, (long)euid, (long)suid)
-#define __sanitizer_syscall_post_setresuid16(res, ruid, euid, suid) \
- __sanitizer_syscall_post_impl_setresuid16(res, (long)ruid, (long)euid, \
+#define __sanitizer_syscall_post_setresuid16(res, ruid, euid, suid) \
+ __sanitizer_syscall_post_impl_setresuid16(res, (long)ruid, (long)euid, \
(long)suid)
-#define __sanitizer_syscall_pre_getresuid16(ruid, euid, suid) \
- __sanitizer_syscall_pre_impl_getresuid16((long)(ruid), (long)(euid), \
+#define __sanitizer_syscall_pre_getresuid16(ruid, euid, suid) \
+ __sanitizer_syscall_pre_impl_getresuid16((long)(ruid), (long)(euid), \
(long)(suid))
-#define __sanitizer_syscall_post_getresuid16(res, ruid, euid, suid) \
- __sanitizer_syscall_post_impl_getresuid16(res, (long)(ruid), (long)(euid), \
+#define __sanitizer_syscall_post_getresuid16(res, ruid, euid, suid) \
+ __sanitizer_syscall_post_impl_getresuid16(res, (long)(ruid), (long)(euid), \
(long)(suid))
-#define __sanitizer_syscall_pre_setresgid16(rgid, egid, sgid) \
+#define __sanitizer_syscall_pre_setresgid16(rgid, egid, sgid) \
__sanitizer_syscall_pre_impl_setresgid16((long)rgid, (long)egid, (long)sgid)
-#define __sanitizer_syscall_post_setresgid16(res, rgid, egid, sgid) \
- __sanitizer_syscall_post_impl_setresgid16(res, (long)rgid, (long)egid, \
+#define __sanitizer_syscall_post_setresgid16(res, rgid, egid, sgid) \
+ __sanitizer_syscall_post_impl_setresgid16(res, (long)rgid, (long)egid, \
(long)sgid)
-#define __sanitizer_syscall_pre_getresgid16(rgid, egid, sgid) \
- __sanitizer_syscall_pre_impl_getresgid16((long)(rgid), (long)(egid), \
+#define __sanitizer_syscall_pre_getresgid16(rgid, egid, sgid) \
+ __sanitizer_syscall_pre_impl_getresgid16((long)(rgid), (long)(egid), \
(long)(sgid))
-#define __sanitizer_syscall_post_getresgid16(res, rgid, egid, sgid) \
- __sanitizer_syscall_post_impl_getresgid16(res, (long)(rgid), (long)(egid), \
+#define __sanitizer_syscall_post_getresgid16(res, rgid, egid, sgid) \
+ __sanitizer_syscall_post_impl_getresgid16(res, (long)(rgid), (long)(egid), \
(long)(sgid))
-#define __sanitizer_syscall_pre_setfsuid16(uid) \
+#define __sanitizer_syscall_pre_setfsuid16(uid) \
__sanitizer_syscall_pre_impl_setfsuid16((long)uid)
-#define __sanitizer_syscall_post_setfsuid16(res, uid) \
+#define __sanitizer_syscall_post_setfsuid16(res, uid) \
__sanitizer_syscall_post_impl_setfsuid16(res, (long)uid)
-#define __sanitizer_syscall_pre_setfsgid16(gid) \
+#define __sanitizer_syscall_pre_setfsgid16(gid) \
__sanitizer_syscall_pre_impl_setfsgid16((long)gid)
-#define __sanitizer_syscall_post_setfsgid16(res, gid) \
+#define __sanitizer_syscall_post_setfsgid16(res, gid) \
__sanitizer_syscall_post_impl_setfsgid16(res, (long)gid)
-#define __sanitizer_syscall_pre_getgroups16(gidsetsize, grouplist) \
- __sanitizer_syscall_pre_impl_getgroups16((long)(gidsetsize), \
+#define __sanitizer_syscall_pre_getgroups16(gidsetsize, grouplist) \
+ __sanitizer_syscall_pre_impl_getgroups16((long)(gidsetsize), \
(long)(grouplist))
-#define __sanitizer_syscall_post_getgroups16(res, gidsetsize, grouplist) \
- __sanitizer_syscall_post_impl_getgroups16(res, (long)(gidsetsize), \
+#define __sanitizer_syscall_post_getgroups16(res, gidsetsize, grouplist) \
+ __sanitizer_syscall_post_impl_getgroups16(res, (long)(gidsetsize), \
(long)(grouplist))
-#define __sanitizer_syscall_pre_setgroups16(gidsetsize, grouplist) \
- __sanitizer_syscall_pre_impl_setgroups16((long)(gidsetsize), \
+#define __sanitizer_syscall_pre_setgroups16(gidsetsize, grouplist) \
+ __sanitizer_syscall_pre_impl_setgroups16((long)(gidsetsize), \
(long)(grouplist))
-#define __sanitizer_syscall_post_setgroups16(res, gidsetsize, grouplist) \
- __sanitizer_syscall_post_impl_setgroups16(res, (long)(gidsetsize), \
+#define __sanitizer_syscall_post_setgroups16(res, gidsetsize, grouplist) \
+ __sanitizer_syscall_post_impl_setgroups16(res, (long)(gidsetsize), \
(long)(grouplist))
-#define __sanitizer_syscall_pre_getuid16() \
+#define __sanitizer_syscall_pre_getuid16() \
__sanitizer_syscall_pre_impl_getuid16()
-#define __sanitizer_syscall_post_getuid16(res) \
+#define __sanitizer_syscall_post_getuid16(res) \
__sanitizer_syscall_post_impl_getuid16(res)
-#define __sanitizer_syscall_pre_geteuid16() \
+#define __sanitizer_syscall_pre_geteuid16() \
__sanitizer_syscall_pre_impl_geteuid16()
-#define __sanitizer_syscall_post_geteuid16(res) \
+#define __sanitizer_syscall_post_geteuid16(res) \
__sanitizer_syscall_post_impl_geteuid16(res)
-#define __sanitizer_syscall_pre_getgid16() \
+#define __sanitizer_syscall_pre_getgid16() \
__sanitizer_syscall_pre_impl_getgid16()
-#define __sanitizer_syscall_post_getgid16(res) \
+#define __sanitizer_syscall_post_getgid16(res) \
__sanitizer_syscall_post_impl_getgid16(res)
-#define __sanitizer_syscall_pre_getegid16() \
+#define __sanitizer_syscall_pre_getegid16() \
__sanitizer_syscall_pre_impl_getegid16()
-#define __sanitizer_syscall_post_getegid16(res) \
+#define __sanitizer_syscall_post_getegid16(res) \
__sanitizer_syscall_post_impl_getegid16(res)
-#define __sanitizer_syscall_pre_utime(filename, times) \
+#define __sanitizer_syscall_pre_utime(filename, times) \
__sanitizer_syscall_pre_impl_utime((long)(filename), (long)(times))
-#define __sanitizer_syscall_post_utime(res, filename, times) \
+#define __sanitizer_syscall_post_utime(res, filename, times) \
__sanitizer_syscall_post_impl_utime(res, (long)(filename), (long)(times))
-#define __sanitizer_syscall_pre_utimes(filename, utimes) \
+#define __sanitizer_syscall_pre_utimes(filename, utimes) \
__sanitizer_syscall_pre_impl_utimes((long)(filename), (long)(utimes))
-#define __sanitizer_syscall_post_utimes(res, filename, utimes) \
+#define __sanitizer_syscall_post_utimes(res, filename, utimes) \
__sanitizer_syscall_post_impl_utimes(res, (long)(filename), (long)(utimes))
-#define __sanitizer_syscall_pre_lseek(fd, offset, origin) \
+#define __sanitizer_syscall_pre_lseek(fd, offset, origin) \
__sanitizer_syscall_pre_impl_lseek((long)(fd), (long)(offset), (long)(origin))
-#define __sanitizer_syscall_post_lseek(res, fd, offset, origin) \
- __sanitizer_syscall_post_impl_lseek(res, (long)(fd), (long)(offset), \
+#define __sanitizer_syscall_post_lseek(res, fd, offset, origin) \
+ __sanitizer_syscall_post_impl_lseek(res, (long)(fd), (long)(offset), \
(long)(origin))
-#define __sanitizer_syscall_pre_llseek(fd, offset_high, offset_low, result, \
- origin) \
- __sanitizer_syscall_pre_impl_llseek((long)(fd), (long)(offset_high), \
- (long)(offset_low), (long)(result), \
+#define __sanitizer_syscall_pre_llseek(fd, offset_high, offset_low, result, \
+ origin) \
+ __sanitizer_syscall_pre_impl_llseek((long)(fd), (long)(offset_high), \
+ (long)(offset_low), (long)(result), \
(long)(origin))
-#define __sanitizer_syscall_post_llseek(res, fd, offset_high, offset_low, \
- result, origin) \
- __sanitizer_syscall_post_impl_llseek(res, (long)(fd), (long)(offset_high), \
- (long)(offset_low), (long)(result), \
+#define __sanitizer_syscall_post_llseek(res, fd, offset_high, offset_low, \
+ result, origin) \
+ __sanitizer_syscall_post_impl_llseek(res, (long)(fd), (long)(offset_high), \
+ (long)(offset_low), (long)(result), \
(long)(origin))
-#define __sanitizer_syscall_pre_read(fd, buf, count) \
+#define __sanitizer_syscall_pre_read(fd, buf, count) \
__sanitizer_syscall_pre_impl_read((long)(fd), (long)(buf), (long)(count))
-#define __sanitizer_syscall_post_read(res, fd, buf, count) \
- __sanitizer_syscall_post_impl_read(res, (long)(fd), (long)(buf), \
+#define __sanitizer_syscall_post_read(res, fd, buf, count) \
+ __sanitizer_syscall_post_impl_read(res, (long)(fd), (long)(buf), \
(long)(count))
-#define __sanitizer_syscall_pre_readv(fd, vec, vlen) \
+#define __sanitizer_syscall_pre_readv(fd, vec, vlen) \
__sanitizer_syscall_pre_impl_readv((long)(fd), (long)(vec), (long)(vlen))
-#define __sanitizer_syscall_post_readv(res, fd, vec, vlen) \
- __sanitizer_syscall_post_impl_readv(res, (long)(fd), (long)(vec), \
+#define __sanitizer_syscall_post_readv(res, fd, vec, vlen) \
+ __sanitizer_syscall_post_impl_readv(res, (long)(fd), (long)(vec), \
(long)(vlen))
-#define __sanitizer_syscall_pre_write(fd, buf, count) \
+#define __sanitizer_syscall_pre_write(fd, buf, count) \
__sanitizer_syscall_pre_impl_write((long)(fd), (long)(buf), (long)(count))
-#define __sanitizer_syscall_post_write(res, fd, buf, count) \
- __sanitizer_syscall_post_impl_write(res, (long)(fd), (long)(buf), \
+#define __sanitizer_syscall_post_write(res, fd, buf, count) \
+ __sanitizer_syscall_post_impl_write(res, (long)(fd), (long)(buf), \
(long)(count))
-#define __sanitizer_syscall_pre_writev(fd, vec, vlen) \
+#define __sanitizer_syscall_pre_writev(fd, vec, vlen) \
__sanitizer_syscall_pre_impl_writev((long)(fd), (long)(vec), (long)(vlen))
-#define __sanitizer_syscall_post_writev(res, fd, vec, vlen) \
- __sanitizer_syscall_post_impl_writev(res, (long)(fd), (long)(vec), \
+#define __sanitizer_syscall_post_writev(res, fd, vec, vlen) \
+ __sanitizer_syscall_post_impl_writev(res, (long)(fd), (long)(vec), \
(long)(vlen))
#ifdef _LP64
#define __sanitizer_syscall_pre_pread64(fd, buf, count, pos) \
__sanitizer_syscall_pre_impl_pread64((long)(fd), (long)(buf), (long)(count), \
(long)(pos))
-#define __sanitizer_syscall_post_pread64(res, fd, buf, count, pos) \
- __sanitizer_syscall_post_impl_pread64(res, (long)(fd), (long)(buf), \
+#define __sanitizer_syscall_post_pread64(res, fd, buf, count, pos) \
+ __sanitizer_syscall_post_impl_pread64(res, (long)(fd), (long)(buf), \
(long)(count), (long)(pos))
-#define __sanitizer_syscall_pre_pwrite64(fd, buf, count, pos) \
- __sanitizer_syscall_pre_impl_pwrite64((long)(fd), (long)(buf), \
+#define __sanitizer_syscall_pre_pwrite64(fd, buf, count, pos) \
+ __sanitizer_syscall_pre_impl_pwrite64((long)(fd), (long)(buf), \
(long)(count), (long)(pos))
-#define __sanitizer_syscall_post_pwrite64(res, fd, buf, count, pos) \
- __sanitizer_syscall_post_impl_pwrite64(res, (long)(fd), (long)(buf), \
+#define __sanitizer_syscall_post_pwrite64(res, fd, buf, count, pos) \
+ __sanitizer_syscall_post_impl_pwrite64(res, (long)(fd), (long)(buf), \
(long)(count), (long)(pos))
#else
#define __sanitizer_syscall_pre_pread64(fd, buf, count, pos0, pos1) \
__sanitizer_syscall_pre_impl_pread64((long)(fd), (long)(buf), (long)(count), \
(long)(pos0), (long)(pos1))
-#define __sanitizer_syscall_post_pread64(res, fd, buf, count, pos0, pos1) \
- __sanitizer_syscall_post_impl_pread64(res, (long)(fd), (long)(buf), \
- (long)(count), (long)(pos0), \
- (long)(pos1))
-#define __sanitizer_syscall_pre_pwrite64(fd, buf, count, pos0, pos1) \
- __sanitizer_syscall_pre_impl_pwrite64( \
+#define __sanitizer_syscall_post_pread64(res, fd, buf, count, pos0, pos1) \
+ __sanitizer_syscall_post_impl_pread64( \
+ res, (long)(fd), (long)(buf), (long)(count), (long)(pos0), (long)(pos1))
+#define __sanitizer_syscall_pre_pwrite64(fd, buf, count, pos0, pos1) \
+ __sanitizer_syscall_pre_impl_pwrite64( \
(long)(fd), (long)(buf), (long)(count), (long)(pos0), (long)(pos1))
-#define __sanitizer_syscall_post_pwrite64(res, fd, buf, count, pos0, pos1) \
- __sanitizer_syscall_post_impl_pwrite64( \
+#define __sanitizer_syscall_post_pwrite64(res, fd, buf, count, pos0, pos1) \
+ __sanitizer_syscall_post_impl_pwrite64( \
res, (long)(fd), (long)(buf), (long)(count), (long)(pos0), (long)(pos1))
#endif
-#define __sanitizer_syscall_pre_preadv(fd, vec, vlen, pos_l, pos_h) \
- __sanitizer_syscall_pre_impl_preadv((long)(fd), (long)(vec), (long)(vlen), \
+#define __sanitizer_syscall_pre_preadv(fd, vec, vlen, pos_l, pos_h) \
+ __sanitizer_syscall_pre_impl_preadv((long)(fd), (long)(vec), (long)(vlen), \
(long)(pos_l), (long)(pos_h))
-#define __sanitizer_syscall_post_preadv(res, fd, vec, vlen, pos_l, pos_h) \
- __sanitizer_syscall_post_impl_preadv(res, (long)(fd), (long)(vec), \
- (long)(vlen), (long)(pos_l), \
+#define __sanitizer_syscall_post_preadv(res, fd, vec, vlen, pos_l, pos_h) \
+ __sanitizer_syscall_post_impl_preadv(res, (long)(fd), (long)(vec), \
+ (long)(vlen), (long)(pos_l), \
(long)(pos_h))
-#define __sanitizer_syscall_pre_pwritev(fd, vec, vlen, pos_l, pos_h) \
- __sanitizer_syscall_pre_impl_pwritev((long)(fd), (long)(vec), (long)(vlen), \
+#define __sanitizer_syscall_pre_pwritev(fd, vec, vlen, pos_l, pos_h) \
+ __sanitizer_syscall_pre_impl_pwritev((long)(fd), (long)(vec), (long)(vlen), \
(long)(pos_l), (long)(pos_h))
-#define __sanitizer_syscall_post_pwritev(res, fd, vec, vlen, pos_l, pos_h) \
- __sanitizer_syscall_post_impl_pwritev(res, (long)(fd), (long)(vec), \
- (long)(vlen), (long)(pos_l), \
+#define __sanitizer_syscall_post_pwritev(res, fd, vec, vlen, pos_l, pos_h) \
+ __sanitizer_syscall_post_impl_pwritev(res, (long)(fd), (long)(vec), \
+ (long)(vlen), (long)(pos_l), \
(long)(pos_h))
-#define __sanitizer_syscall_pre_getcwd(buf, size) \
+#define __sanitizer_syscall_pre_getcwd(buf, size) \
__sanitizer_syscall_pre_impl_getcwd((long)(buf), (long)(size))
-#define __sanitizer_syscall_post_getcwd(res, buf, size) \
+#define __sanitizer_syscall_post_getcwd(res, buf, size) \
__sanitizer_syscall_post_impl_getcwd(res, (long)(buf), (long)(size))
-#define __sanitizer_syscall_pre_mkdir(pathname, mode) \
+#define __sanitizer_syscall_pre_mkdir(pathname, mode) \
__sanitizer_syscall_pre_impl_mkdir((long)(pathname), (long)(mode))
-#define __sanitizer_syscall_post_mkdir(res, pathname, mode) \
+#define __sanitizer_syscall_post_mkdir(res, pathname, mode) \
__sanitizer_syscall_post_impl_mkdir(res, (long)(pathname), (long)(mode))
-#define __sanitizer_syscall_pre_chdir(filename) \
+#define __sanitizer_syscall_pre_chdir(filename) \
__sanitizer_syscall_pre_impl_chdir((long)(filename))
-#define __sanitizer_syscall_post_chdir(res, filename) \
+#define __sanitizer_syscall_post_chdir(res, filename) \
__sanitizer_syscall_post_impl_chdir(res, (long)(filename))
-#define __sanitizer_syscall_pre_fchdir(fd) \
+#define __sanitizer_syscall_pre_fchdir(fd) \
__sanitizer_syscall_pre_impl_fchdir((long)(fd))
-#define __sanitizer_syscall_post_fchdir(res, fd) \
+#define __sanitizer_syscall_post_fchdir(res, fd) \
__sanitizer_syscall_post_impl_fchdir(res, (long)(fd))
-#define __sanitizer_syscall_pre_rmdir(pathname) \
+#define __sanitizer_syscall_pre_rmdir(pathname) \
__sanitizer_syscall_pre_impl_rmdir((long)(pathname))
-#define __sanitizer_syscall_post_rmdir(res, pathname) \
+#define __sanitizer_syscall_post_rmdir(res, pathname) \
__sanitizer_syscall_post_impl_rmdir(res, (long)(pathname))
-#define __sanitizer_syscall_pre_lookup_dcookie(cookie64, buf, len) \
- __sanitizer_syscall_pre_impl_lookup_dcookie((long)(cookie64), (long)(buf), \
+#define __sanitizer_syscall_pre_lookup_dcookie(cookie64, buf, len) \
+ __sanitizer_syscall_pre_impl_lookup_dcookie((long)(cookie64), (long)(buf), \
(long)(len))
-#define __sanitizer_syscall_post_lookup_dcookie(res, cookie64, buf, len) \
- __sanitizer_syscall_post_impl_lookup_dcookie(res, (long)(cookie64), \
+#define __sanitizer_syscall_post_lookup_dcookie(res, cookie64, buf, len) \
+ __sanitizer_syscall_post_impl_lookup_dcookie(res, (long)(cookie64), \
(long)(buf), (long)(len))
-#define __sanitizer_syscall_pre_quotactl(cmd, special, id, addr) \
- __sanitizer_syscall_pre_impl_quotactl((long)(cmd), (long)(special), \
+#define __sanitizer_syscall_pre_quotactl(cmd, special, id, addr) \
+ __sanitizer_syscall_pre_impl_quotactl((long)(cmd), (long)(special), \
(long)(id), (long)(addr))
-#define __sanitizer_syscall_post_quotactl(res, cmd, special, id, addr) \
- __sanitizer_syscall_post_impl_quotactl(res, (long)(cmd), (long)(special), \
+#define __sanitizer_syscall_post_quotactl(res, cmd, special, id, addr) \
+ __sanitizer_syscall_post_impl_quotactl(res, (long)(cmd), (long)(special), \
(long)(id), (long)(addr))
-#define __sanitizer_syscall_pre_getdents(fd, dirent, count) \
- __sanitizer_syscall_pre_impl_getdents((long)(fd), (long)(dirent), \
+#define __sanitizer_syscall_pre_getdents(fd, dirent, count) \
+ __sanitizer_syscall_pre_impl_getdents((long)(fd), (long)(dirent), \
(long)(count))
-#define __sanitizer_syscall_post_getdents(res, fd, dirent, count) \
- __sanitizer_syscall_post_impl_getdents(res, (long)(fd), (long)(dirent), \
+#define __sanitizer_syscall_post_getdents(res, fd, dirent, count) \
+ __sanitizer_syscall_post_impl_getdents(res, (long)(fd), (long)(dirent), \
(long)(count))
-#define __sanitizer_syscall_pre_getdents64(fd, dirent, count) \
- __sanitizer_syscall_pre_impl_getdents64((long)(fd), (long)(dirent), \
+#define __sanitizer_syscall_pre_getdents64(fd, dirent, count) \
+ __sanitizer_syscall_pre_impl_getdents64((long)(fd), (long)(dirent), \
(long)(count))
-#define __sanitizer_syscall_post_getdents64(res, fd, dirent, count) \
- __sanitizer_syscall_post_impl_getdents64(res, (long)(fd), (long)(dirent), \
+#define __sanitizer_syscall_post_getdents64(res, fd, dirent, count) \
+ __sanitizer_syscall_post_impl_getdents64(res, (long)(fd), (long)(dirent), \
(long)(count))
#define __sanitizer_syscall_pre_setsockopt(fd, level, optname, optval, optlen) \
__sanitizer_syscall_pre_impl_setsockopt((long)(fd), (long)(level), \
(long)(optname), (long)(optval), \
(long)(optlen))
-#define __sanitizer_syscall_post_setsockopt(res, fd, level, optname, optval, \
- optlen) \
- __sanitizer_syscall_post_impl_setsockopt(res, (long)(fd), (long)(level), \
- (long)(optname), (long)(optval), \
+#define __sanitizer_syscall_post_setsockopt(res, fd, level, optname, optval, \
+ optlen) \
+ __sanitizer_syscall_post_impl_setsockopt(res, (long)(fd), (long)(level), \
+ (long)(optname), (long)(optval), \
(long)(optlen))
#define __sanitizer_syscall_pre_getsockopt(fd, level, optname, optval, optlen) \
__sanitizer_syscall_pre_impl_getsockopt((long)(fd), (long)(level), \
(long)(optname), (long)(optval), \
(long)(optlen))
-#define __sanitizer_syscall_post_getsockopt(res, fd, level, optname, optval, \
- optlen) \
- __sanitizer_syscall_post_impl_getsockopt(res, (long)(fd), (long)(level), \
- (long)(optname), (long)(optval), \
+#define __sanitizer_syscall_post_getsockopt(res, fd, level, optname, optval, \
+ optlen) \
+ __sanitizer_syscall_post_impl_getsockopt(res, (long)(fd), (long)(level), \
+ (long)(optname), (long)(optval), \
(long)(optlen))
-#define __sanitizer_syscall_pre_bind(arg0, arg1, arg2) \
+#define __sanitizer_syscall_pre_bind(arg0, arg1, arg2) \
__sanitizer_syscall_pre_impl_bind((long)(arg0), (long)(arg1), (long)(arg2))
-#define __sanitizer_syscall_post_bind(res, arg0, arg1, arg2) \
- __sanitizer_syscall_post_impl_bind(res, (long)(arg0), (long)(arg1), \
+#define __sanitizer_syscall_post_bind(res, arg0, arg1, arg2) \
+ __sanitizer_syscall_post_impl_bind(res, (long)(arg0), (long)(arg1), \
(long)(arg2))
-#define __sanitizer_syscall_pre_connect(arg0, arg1, arg2) \
+#define __sanitizer_syscall_pre_connect(arg0, arg1, arg2) \
__sanitizer_syscall_pre_impl_connect((long)(arg0), (long)(arg1), (long)(arg2))
-#define __sanitizer_syscall_post_connect(res, arg0, arg1, arg2) \
- __sanitizer_syscall_post_impl_connect(res, (long)(arg0), (long)(arg1), \
+#define __sanitizer_syscall_post_connect(res, arg0, arg1, arg2) \
+ __sanitizer_syscall_post_impl_connect(res, (long)(arg0), (long)(arg1), \
(long)(arg2))
-#define __sanitizer_syscall_pre_accept(arg0, arg1, arg2) \
+#define __sanitizer_syscall_pre_accept(arg0, arg1, arg2) \
__sanitizer_syscall_pre_impl_accept((long)(arg0), (long)(arg1), (long)(arg2))
-#define __sanitizer_syscall_post_accept(res, arg0, arg1, arg2) \
- __sanitizer_syscall_post_impl_accept(res, (long)(arg0), (long)(arg1), \
+#define __sanitizer_syscall_post_accept(res, arg0, arg1, arg2) \
+ __sanitizer_syscall_post_impl_accept(res, (long)(arg0), (long)(arg1), \
(long)(arg2))
-#define __sanitizer_syscall_pre_accept4(arg0, arg1, arg2, arg3) \
- __sanitizer_syscall_pre_impl_accept4((long)(arg0), (long)(arg1), \
+#define __sanitizer_syscall_pre_accept4(arg0, arg1, arg2, arg3) \
+ __sanitizer_syscall_pre_impl_accept4((long)(arg0), (long)(arg1), \
(long)(arg2), (long)(arg3))
-#define __sanitizer_syscall_post_accept4(res, arg0, arg1, arg2, arg3) \
- __sanitizer_syscall_post_impl_accept4(res, (long)(arg0), (long)(arg1), \
+#define __sanitizer_syscall_post_accept4(res, arg0, arg1, arg2, arg3) \
+ __sanitizer_syscall_post_impl_accept4(res, (long)(arg0), (long)(arg1), \
(long)(arg2), (long)(arg3))
-#define __sanitizer_syscall_pre_getsockname(arg0, arg1, arg2) \
- __sanitizer_syscall_pre_impl_getsockname((long)(arg0), (long)(arg1), \
+#define __sanitizer_syscall_pre_getsockname(arg0, arg1, arg2) \
+ __sanitizer_syscall_pre_impl_getsockname((long)(arg0), (long)(arg1), \
(long)(arg2))
-#define __sanitizer_syscall_post_getsockname(res, arg0, arg1, arg2) \
- __sanitizer_syscall_post_impl_getsockname(res, (long)(arg0), (long)(arg1), \
+#define __sanitizer_syscall_post_getsockname(res, arg0, arg1, arg2) \
+ __sanitizer_syscall_post_impl_getsockname(res, (long)(arg0), (long)(arg1), \
(long)(arg2))
-#define __sanitizer_syscall_pre_getpeername(arg0, arg1, arg2) \
- __sanitizer_syscall_pre_impl_getpeername((long)(arg0), (long)(arg1), \
+#define __sanitizer_syscall_pre_getpeername(arg0, arg1, arg2) \
+ __sanitizer_syscall_pre_impl_getpeername((long)(arg0), (long)(arg1), \
(long)(arg2))
-#define __sanitizer_syscall_post_getpeername(res, arg0, arg1, arg2) \
- __sanitizer_syscall_post_impl_getpeername(res, (long)(arg0), (long)(arg1), \
+#define __sanitizer_syscall_post_getpeername(res, arg0, arg1, arg2) \
+ __sanitizer_syscall_post_impl_getpeername(res, (long)(arg0), (long)(arg1), \
(long)(arg2))
-#define __sanitizer_syscall_pre_send(arg0, arg1, arg2, arg3) \
- __sanitizer_syscall_pre_impl_send((long)(arg0), (long)(arg1), (long)(arg2), \
+#define __sanitizer_syscall_pre_send(arg0, arg1, arg2, arg3) \
+ __sanitizer_syscall_pre_impl_send((long)(arg0), (long)(arg1), (long)(arg2), \
(long)(arg3))
-#define __sanitizer_syscall_post_send(res, arg0, arg1, arg2, arg3) \
- __sanitizer_syscall_post_impl_send(res, (long)(arg0), (long)(arg1), \
+#define __sanitizer_syscall_post_send(res, arg0, arg1, arg2, arg3) \
+ __sanitizer_syscall_post_impl_send(res, (long)(arg0), (long)(arg1), \
(long)(arg2), (long)(arg3))
-#define __sanitizer_syscall_pre_sendto(arg0, arg1, arg2, arg3, arg4, arg5) \
- __sanitizer_syscall_pre_impl_sendto((long)(arg0), (long)(arg1), \
- (long)(arg2), (long)(arg3), \
+#define __sanitizer_syscall_pre_sendto(arg0, arg1, arg2, arg3, arg4, arg5) \
+ __sanitizer_syscall_pre_impl_sendto((long)(arg0), (long)(arg1), \
+ (long)(arg2), (long)(arg3), \
(long)(arg4), (long)(arg5))
-#define __sanitizer_syscall_post_sendto(res, arg0, arg1, arg2, arg3, arg4, \
- arg5) \
- __sanitizer_syscall_post_impl_sendto(res, (long)(arg0), (long)(arg1), \
- (long)(arg2), (long)(arg3), \
+#define __sanitizer_syscall_post_sendto(res, arg0, arg1, arg2, arg3, arg4, \
+ arg5) \
+ __sanitizer_syscall_post_impl_sendto(res, (long)(arg0), (long)(arg1), \
+ (long)(arg2), (long)(arg3), \
(long)(arg4), (long)(arg5))
-#define __sanitizer_syscall_pre_sendmsg(fd, msg, flags) \
+#define __sanitizer_syscall_pre_sendmsg(fd, msg, flags) \
__sanitizer_syscall_pre_impl_sendmsg((long)(fd), (long)(msg), (long)(flags))
-#define __sanitizer_syscall_post_sendmsg(res, fd, msg, flags) \
- __sanitizer_syscall_post_impl_sendmsg(res, (long)(fd), (long)(msg), \
+#define __sanitizer_syscall_post_sendmsg(res, fd, msg, flags) \
+ __sanitizer_syscall_post_impl_sendmsg(res, (long)(fd), (long)(msg), \
(long)(flags))
#define __sanitizer_syscall_pre_sendmmsg(fd, msg, vlen, flags) \
__sanitizer_syscall_pre_impl_sendmmsg((long)(fd), (long)(msg), (long)(vlen), \
(long)(flags))
-#define __sanitizer_syscall_post_sendmmsg(res, fd, msg, vlen, flags) \
- __sanitizer_syscall_post_impl_sendmmsg(res, (long)(fd), (long)(msg), \
+#define __sanitizer_syscall_post_sendmmsg(res, fd, msg, vlen, flags) \
+ __sanitizer_syscall_post_impl_sendmmsg(res, (long)(fd), (long)(msg), \
(long)(vlen), (long)(flags))
-#define __sanitizer_syscall_pre_recv(arg0, arg1, arg2, arg3) \
- __sanitizer_syscall_pre_impl_recv((long)(arg0), (long)(arg1), (long)(arg2), \
+#define __sanitizer_syscall_pre_recv(arg0, arg1, arg2, arg3) \
+ __sanitizer_syscall_pre_impl_recv((long)(arg0), (long)(arg1), (long)(arg2), \
(long)(arg3))
-#define __sanitizer_syscall_post_recv(res, arg0, arg1, arg2, arg3) \
- __sanitizer_syscall_post_impl_recv(res, (long)(arg0), (long)(arg1), \
+#define __sanitizer_syscall_post_recv(res, arg0, arg1, arg2, arg3) \
+ __sanitizer_syscall_post_impl_recv(res, (long)(arg0), (long)(arg1), \
(long)(arg2), (long)(arg3))
-#define __sanitizer_syscall_pre_recvfrom(arg0, arg1, arg2, arg3, arg4, arg5) \
- __sanitizer_syscall_pre_impl_recvfrom((long)(arg0), (long)(arg1), \
- (long)(arg2), (long)(arg3), \
+#define __sanitizer_syscall_pre_recvfrom(arg0, arg1, arg2, arg3, arg4, arg5) \
+ __sanitizer_syscall_pre_impl_recvfrom((long)(arg0), (long)(arg1), \
+ (long)(arg2), (long)(arg3), \
(long)(arg4), (long)(arg5))
-#define __sanitizer_syscall_post_recvfrom(res, arg0, arg1, arg2, arg3, arg4, \
- arg5) \
- __sanitizer_syscall_post_impl_recvfrom(res, (long)(arg0), (long)(arg1), \
- (long)(arg2), (long)(arg3), \
+#define __sanitizer_syscall_post_recvfrom(res, arg0, arg1, arg2, arg3, arg4, \
+ arg5) \
+ __sanitizer_syscall_post_impl_recvfrom(res, (long)(arg0), (long)(arg1), \
+ (long)(arg2), (long)(arg3), \
(long)(arg4), (long)(arg5))
-#define __sanitizer_syscall_pre_recvmsg(fd, msg, flags) \
+#define __sanitizer_syscall_pre_recvmsg(fd, msg, flags) \
__sanitizer_syscall_pre_impl_recvmsg((long)(fd), (long)(msg), (long)(flags))
-#define __sanitizer_syscall_post_recvmsg(res, fd, msg, flags) \
- __sanitizer_syscall_post_impl_recvmsg(res, (long)(fd), (long)(msg), \
+#define __sanitizer_syscall_post_recvmsg(res, fd, msg, flags) \
+ __sanitizer_syscall_post_impl_recvmsg(res, (long)(fd), (long)(msg), \
(long)(flags))
#define __sanitizer_syscall_pre_recvmmsg(fd, msg, vlen, flags, timeout) \
__sanitizer_syscall_pre_impl_recvmmsg((long)(fd), (long)(msg), (long)(vlen), \
(long)(flags), (long)(timeout))
-#define __sanitizer_syscall_post_recvmmsg(res, fd, msg, vlen, flags, timeout) \
- __sanitizer_syscall_post_impl_recvmmsg(res, (long)(fd), (long)(msg), \
- (long)(vlen), (long)(flags), \
+#define __sanitizer_syscall_post_recvmmsg(res, fd, msg, vlen, flags, timeout) \
+ __sanitizer_syscall_post_impl_recvmmsg(res, (long)(fd), (long)(msg), \
+ (long)(vlen), (long)(flags), \
(long)(timeout))
-#define __sanitizer_syscall_pre_socket(arg0, arg1, arg2) \
+#define __sanitizer_syscall_pre_socket(arg0, arg1, arg2) \
__sanitizer_syscall_pre_impl_socket((long)(arg0), (long)(arg1), (long)(arg2))
-#define __sanitizer_syscall_post_socket(res, arg0, arg1, arg2) \
- __sanitizer_syscall_post_impl_socket(res, (long)(arg0), (long)(arg1), \
+#define __sanitizer_syscall_post_socket(res, arg0, arg1, arg2) \
+ __sanitizer_syscall_post_impl_socket(res, (long)(arg0), (long)(arg1), \
(long)(arg2))
-#define __sanitizer_syscall_pre_socketpair(arg0, arg1, arg2, arg3) \
- __sanitizer_syscall_pre_impl_socketpair((long)(arg0), (long)(arg1), \
+#define __sanitizer_syscall_pre_socketpair(arg0, arg1, arg2, arg3) \
+ __sanitizer_syscall_pre_impl_socketpair((long)(arg0), (long)(arg1), \
(long)(arg2), (long)(arg3))
-#define __sanitizer_syscall_post_socketpair(res, arg0, arg1, arg2, arg3) \
- __sanitizer_syscall_post_impl_socketpair(res, (long)(arg0), (long)(arg1), \
+#define __sanitizer_syscall_post_socketpair(res, arg0, arg1, arg2, arg3) \
+ __sanitizer_syscall_post_impl_socketpair(res, (long)(arg0), (long)(arg1), \
(long)(arg2), (long)(arg3))
-#define __sanitizer_syscall_pre_socketcall(call, args) \
+#define __sanitizer_syscall_pre_socketcall(call, args) \
__sanitizer_syscall_pre_impl_socketcall((long)(call), (long)(args))
-#define __sanitizer_syscall_post_socketcall(res, call, args) \
+#define __sanitizer_syscall_post_socketcall(res, call, args) \
__sanitizer_syscall_post_impl_socketcall(res, (long)(call), (long)(args))
-#define __sanitizer_syscall_pre_listen(arg0, arg1) \
+#define __sanitizer_syscall_pre_listen(arg0, arg1) \
__sanitizer_syscall_pre_impl_listen((long)(arg0), (long)(arg1))
-#define __sanitizer_syscall_post_listen(res, arg0, arg1) \
+#define __sanitizer_syscall_post_listen(res, arg0, arg1) \
__sanitizer_syscall_post_impl_listen(res, (long)(arg0), (long)(arg1))
-#define __sanitizer_syscall_pre_poll(ufds, nfds, timeout) \
+#define __sanitizer_syscall_pre_poll(ufds, nfds, timeout) \
__sanitizer_syscall_pre_impl_poll((long)(ufds), (long)(nfds), (long)(timeout))
-#define __sanitizer_syscall_post_poll(res, ufds, nfds, timeout) \
- __sanitizer_syscall_post_impl_poll(res, (long)(ufds), (long)(nfds), \
+#define __sanitizer_syscall_post_poll(res, ufds, nfds, timeout) \
+ __sanitizer_syscall_post_impl_poll(res, (long)(ufds), (long)(nfds), \
(long)(timeout))
-#define __sanitizer_syscall_pre_select(n, inp, outp, exp, tvp) \
- __sanitizer_syscall_pre_impl_select((long)(n), (long)(inp), (long)(outp), \
+#define __sanitizer_syscall_pre_select(n, inp, outp, exp, tvp) \
+ __sanitizer_syscall_pre_impl_select((long)(n), (long)(inp), (long)(outp), \
(long)(exp), (long)(tvp))
-#define __sanitizer_syscall_post_select(res, n, inp, outp, exp, tvp) \
- __sanitizer_syscall_post_impl_select(res, (long)(n), (long)(inp), \
+#define __sanitizer_syscall_post_select(res, n, inp, outp, exp, tvp) \
+ __sanitizer_syscall_post_impl_select(res, (long)(n), (long)(inp), \
(long)(outp), (long)(exp), (long)(tvp))
-#define __sanitizer_syscall_pre_old_select(arg) \
+#define __sanitizer_syscall_pre_old_select(arg) \
__sanitizer_syscall_pre_impl_old_select((long)(arg))
-#define __sanitizer_syscall_post_old_select(res, arg) \
+#define __sanitizer_syscall_post_old_select(res, arg) \
__sanitizer_syscall_post_impl_old_select(res, (long)(arg))
-#define __sanitizer_syscall_pre_epoll_create(size) \
+#define __sanitizer_syscall_pre_epoll_create(size) \
__sanitizer_syscall_pre_impl_epoll_create((long)(size))
-#define __sanitizer_syscall_post_epoll_create(res, size) \
+#define __sanitizer_syscall_post_epoll_create(res, size) \
__sanitizer_syscall_post_impl_epoll_create(res, (long)(size))
-#define __sanitizer_syscall_pre_epoll_create1(flags) \
+#define __sanitizer_syscall_pre_epoll_create1(flags) \
__sanitizer_syscall_pre_impl_epoll_create1((long)(flags))
-#define __sanitizer_syscall_post_epoll_create1(res, flags) \
+#define __sanitizer_syscall_post_epoll_create1(res, flags) \
__sanitizer_syscall_post_impl_epoll_create1(res, (long)(flags))
#define __sanitizer_syscall_pre_epoll_ctl(epfd, op, fd, event) \
__sanitizer_syscall_pre_impl_epoll_ctl((long)(epfd), (long)(op), (long)(fd), \
(long)(event))
-#define __sanitizer_syscall_post_epoll_ctl(res, epfd, op, fd, event) \
- __sanitizer_syscall_post_impl_epoll_ctl(res, (long)(epfd), (long)(op), \
+#define __sanitizer_syscall_post_epoll_ctl(res, epfd, op, fd, event) \
+ __sanitizer_syscall_post_impl_epoll_ctl(res, (long)(epfd), (long)(op), \
(long)(fd), (long)(event))
-#define __sanitizer_syscall_pre_epoll_wait(epfd, events, maxevents, timeout) \
- __sanitizer_syscall_pre_impl_epoll_wait((long)(epfd), (long)(events), \
+#define __sanitizer_syscall_pre_epoll_wait(epfd, events, maxevents, timeout) \
+ __sanitizer_syscall_pre_impl_epoll_wait((long)(epfd), (long)(events), \
(long)(maxevents), (long)(timeout))
-#define __sanitizer_syscall_post_epoll_wait(res, epfd, events, maxevents, \
- timeout) \
- __sanitizer_syscall_post_impl_epoll_wait(res, (long)(epfd), (long)(events), \
+#define __sanitizer_syscall_post_epoll_wait(res, epfd, events, maxevents, \
+ timeout) \
+ __sanitizer_syscall_post_impl_epoll_wait(res, (long)(epfd), (long)(events), \
(long)(maxevents), (long)(timeout))
-#define __sanitizer_syscall_pre_epoll_pwait(epfd, events, maxevents, timeout, \
- sigmask, sigsetsize) \
- __sanitizer_syscall_pre_impl_epoll_pwait( \
- (long)(epfd), (long)(events), (long)(maxevents), (long)(timeout), \
+#define __sanitizer_syscall_pre_epoll_pwait(epfd, events, maxevents, timeout, \
+ sigmask, sigsetsize) \
+ __sanitizer_syscall_pre_impl_epoll_pwait( \
+ (long)(epfd), (long)(events), (long)(maxevents), (long)(timeout), \
+ (long)(sigmask), (long)(sigsetsize))
+#define __sanitizer_syscall_post_epoll_pwait(res, epfd, events, maxevents, \
+ timeout, sigmask, sigsetsize) \
+ __sanitizer_syscall_post_impl_epoll_pwait( \
+ res, (long)(epfd), (long)(events), (long)(maxevents), (long)(timeout), \
(long)(sigmask), (long)(sigsetsize))
-#define __sanitizer_syscall_post_epoll_pwait(res, epfd, events, maxevents, \
- timeout, sigmask, sigsetsize) \
- __sanitizer_syscall_post_impl_epoll_pwait( \
- res, (long)(epfd), (long)(events), (long)(maxevents), (long)(timeout), \
+#define __sanitizer_syscall_pre_epoll_pwait2(epfd, events, maxevents, timeout, \
+ sigmask, sigsetsize) \
+ __sanitizer_syscall_pre_impl_epoll_pwait2( \
+ (long)(epfd), (long)(events), (long)(maxevents), (long)(timeout), \
(long)(sigmask), (long)(sigsetsize))
-#define __sanitizer_syscall_pre_gethostname(name, len) \
+#define __sanitizer_syscall_post_epoll_pwait2(res, epfd, events, maxevents, \
+ timeout, sigmask, sigsetsize) \
+ __sanitizer_syscall_post_impl_epoll_pwait2( \
+ res, (long)(epfd), (long)(events), (long)(maxevents), (long)(timeout), \
+ (long)(sigmask), (long)(sigsetsize))
+#define __sanitizer_syscall_pre_gethostname(name, len) \
__sanitizer_syscall_pre_impl_gethostname((long)(name), (long)(len))
-#define __sanitizer_syscall_post_gethostname(res, name, len) \
+#define __sanitizer_syscall_post_gethostname(res, name, len) \
__sanitizer_syscall_post_impl_gethostname(res, (long)(name), (long)(len))
-#define __sanitizer_syscall_pre_sethostname(name, len) \
+#define __sanitizer_syscall_pre_sethostname(name, len) \
__sanitizer_syscall_pre_impl_sethostname((long)(name), (long)(len))
-#define __sanitizer_syscall_post_sethostname(res, name, len) \
+#define __sanitizer_syscall_post_sethostname(res, name, len) \
__sanitizer_syscall_post_impl_sethostname(res, (long)(name), (long)(len))
-#define __sanitizer_syscall_pre_setdomainname(name, len) \
+#define __sanitizer_syscall_pre_setdomainname(name, len) \
__sanitizer_syscall_pre_impl_setdomainname((long)(name), (long)(len))
-#define __sanitizer_syscall_post_setdomainname(res, name, len) \
+#define __sanitizer_syscall_post_setdomainname(res, name, len) \
__sanitizer_syscall_post_impl_setdomainname(res, (long)(name), (long)(len))
-#define __sanitizer_syscall_pre_newuname(name) \
+#define __sanitizer_syscall_pre_newuname(name) \
__sanitizer_syscall_pre_impl_newuname((long)(name))
-#define __sanitizer_syscall_post_newuname(res, name) \
+#define __sanitizer_syscall_post_newuname(res, name) \
__sanitizer_syscall_post_impl_newuname(res, (long)(name))
-#define __sanitizer_syscall_pre_uname(arg0) \
+#define __sanitizer_syscall_pre_uname(arg0) \
__sanitizer_syscall_pre_impl_uname((long)(arg0))
-#define __sanitizer_syscall_post_uname(res, arg0) \
+#define __sanitizer_syscall_post_uname(res, arg0) \
__sanitizer_syscall_post_impl_uname(res, (long)(arg0))
-#define __sanitizer_syscall_pre_olduname(arg0) \
+#define __sanitizer_syscall_pre_olduname(arg0) \
__sanitizer_syscall_pre_impl_olduname((long)(arg0))
-#define __sanitizer_syscall_post_olduname(res, arg0) \
+#define __sanitizer_syscall_post_olduname(res, arg0) \
__sanitizer_syscall_post_impl_olduname(res, (long)(arg0))
-#define __sanitizer_syscall_pre_getrlimit(resource, rlim) \
+#define __sanitizer_syscall_pre_getrlimit(resource, rlim) \
__sanitizer_syscall_pre_impl_getrlimit((long)(resource), (long)(rlim))
-#define __sanitizer_syscall_post_getrlimit(res, resource, rlim) \
+#define __sanitizer_syscall_post_getrlimit(res, resource, rlim) \
__sanitizer_syscall_post_impl_getrlimit(res, (long)(resource), (long)(rlim))
-#define __sanitizer_syscall_pre_old_getrlimit(resource, rlim) \
+#define __sanitizer_syscall_pre_old_getrlimit(resource, rlim) \
__sanitizer_syscall_pre_impl_old_getrlimit((long)(resource), (long)(rlim))
-#define __sanitizer_syscall_post_old_getrlimit(res, resource, rlim) \
- __sanitizer_syscall_post_impl_old_getrlimit(res, (long)(resource), \
+#define __sanitizer_syscall_post_old_getrlimit(res, resource, rlim) \
+ __sanitizer_syscall_post_impl_old_getrlimit(res, (long)(resource), \
(long)(rlim))
-#define __sanitizer_syscall_pre_setrlimit(resource, rlim) \
+#define __sanitizer_syscall_pre_setrlimit(resource, rlim) \
__sanitizer_syscall_pre_impl_setrlimit((long)(resource), (long)(rlim))
-#define __sanitizer_syscall_post_setrlimit(res, resource, rlim) \
+#define __sanitizer_syscall_post_setrlimit(res, resource, rlim) \
__sanitizer_syscall_post_impl_setrlimit(res, (long)(resource), (long)(rlim))
-#define __sanitizer_syscall_pre_prlimit64(pid, resource, new_rlim, old_rlim) \
- __sanitizer_syscall_pre_impl_prlimit64((long)(pid), (long)(resource), \
+#define __sanitizer_syscall_pre_prlimit64(pid, resource, new_rlim, old_rlim) \
+ __sanitizer_syscall_pre_impl_prlimit64((long)(pid), (long)(resource), \
(long)(new_rlim), (long)(old_rlim))
-#define __sanitizer_syscall_post_prlimit64(res, pid, resource, new_rlim, \
- old_rlim) \
- __sanitizer_syscall_post_impl_prlimit64(res, (long)(pid), (long)(resource), \
+#define __sanitizer_syscall_post_prlimit64(res, pid, resource, new_rlim, \
+ old_rlim) \
+ __sanitizer_syscall_post_impl_prlimit64(res, (long)(pid), (long)(resource), \
(long)(new_rlim), (long)(old_rlim))
-#define __sanitizer_syscall_pre_getrusage(who, ru) \
+#define __sanitizer_syscall_pre_getrusage(who, ru) \
__sanitizer_syscall_pre_impl_getrusage((long)(who), (long)(ru))
-#define __sanitizer_syscall_post_getrusage(res, who, ru) \
+#define __sanitizer_syscall_post_getrusage(res, who, ru) \
__sanitizer_syscall_post_impl_getrusage(res, (long)(who), (long)(ru))
-#define __sanitizer_syscall_pre_umask(mask) \
+#define __sanitizer_syscall_pre_umask(mask) \
__sanitizer_syscall_pre_impl_umask((long)(mask))
-#define __sanitizer_syscall_post_umask(res, mask) \
+#define __sanitizer_syscall_post_umask(res, mask) \
__sanitizer_syscall_post_impl_umask(res, (long)(mask))
-#define __sanitizer_syscall_pre_msgget(key, msgflg) \
+#define __sanitizer_syscall_pre_msgget(key, msgflg) \
__sanitizer_syscall_pre_impl_msgget((long)(key), (long)(msgflg))
-#define __sanitizer_syscall_post_msgget(res, key, msgflg) \
+#define __sanitizer_syscall_post_msgget(res, key, msgflg) \
__sanitizer_syscall_post_impl_msgget(res, (long)(key), (long)(msgflg))
-#define __sanitizer_syscall_pre_msgsnd(msqid, msgp, msgsz, msgflg) \
- __sanitizer_syscall_pre_impl_msgsnd((long)(msqid), (long)(msgp), \
+#define __sanitizer_syscall_pre_msgsnd(msqid, msgp, msgsz, msgflg) \
+ __sanitizer_syscall_pre_impl_msgsnd((long)(msqid), (long)(msgp), \
(long)(msgsz), (long)(msgflg))
-#define __sanitizer_syscall_post_msgsnd(res, msqid, msgp, msgsz, msgflg) \
- __sanitizer_syscall_post_impl_msgsnd(res, (long)(msqid), (long)(msgp), \
+#define __sanitizer_syscall_post_msgsnd(res, msqid, msgp, msgsz, msgflg) \
+ __sanitizer_syscall_post_impl_msgsnd(res, (long)(msqid), (long)(msgp), \
(long)(msgsz), (long)(msgflg))
-#define __sanitizer_syscall_pre_msgrcv(msqid, msgp, msgsz, msgtyp, msgflg) \
- __sanitizer_syscall_pre_impl_msgrcv((long)(msqid), (long)(msgp), \
- (long)(msgsz), (long)(msgtyp), \
+#define __sanitizer_syscall_pre_msgrcv(msqid, msgp, msgsz, msgtyp, msgflg) \
+ __sanitizer_syscall_pre_impl_msgrcv((long)(msqid), (long)(msgp), \
+ (long)(msgsz), (long)(msgtyp), \
(long)(msgflg))
-#define __sanitizer_syscall_post_msgrcv(res, msqid, msgp, msgsz, msgtyp, \
- msgflg) \
- __sanitizer_syscall_post_impl_msgrcv(res, (long)(msqid), (long)(msgp), \
- (long)(msgsz), (long)(msgtyp), \
+#define __sanitizer_syscall_post_msgrcv(res, msqid, msgp, msgsz, msgtyp, \
+ msgflg) \
+ __sanitizer_syscall_post_impl_msgrcv(res, (long)(msqid), (long)(msgp), \
+ (long)(msgsz), (long)(msgtyp), \
(long)(msgflg))
-#define __sanitizer_syscall_pre_msgctl(msqid, cmd, buf) \
+#define __sanitizer_syscall_pre_msgctl(msqid, cmd, buf) \
__sanitizer_syscall_pre_impl_msgctl((long)(msqid), (long)(cmd), (long)(buf))
-#define __sanitizer_syscall_post_msgctl(res, msqid, cmd, buf) \
- __sanitizer_syscall_post_impl_msgctl(res, (long)(msqid), (long)(cmd), \
+#define __sanitizer_syscall_post_msgctl(res, msqid, cmd, buf) \
+ __sanitizer_syscall_post_impl_msgctl(res, (long)(msqid), (long)(cmd), \
(long)(buf))
-#define __sanitizer_syscall_pre_semget(key, nsems, semflg) \
- __sanitizer_syscall_pre_impl_semget((long)(key), (long)(nsems), \
+#define __sanitizer_syscall_pre_semget(key, nsems, semflg) \
+ __sanitizer_syscall_pre_impl_semget((long)(key), (long)(nsems), \
(long)(semflg))
-#define __sanitizer_syscall_post_semget(res, key, nsems, semflg) \
- __sanitizer_syscall_post_impl_semget(res, (long)(key), (long)(nsems), \
+#define __sanitizer_syscall_post_semget(res, key, nsems, semflg) \
+ __sanitizer_syscall_post_impl_semget(res, (long)(key), (long)(nsems), \
(long)(semflg))
-#define __sanitizer_syscall_pre_semop(semid, sops, nsops) \
+#define __sanitizer_syscall_pre_semop(semid, sops, nsops) \
__sanitizer_syscall_pre_impl_semop((long)(semid), (long)(sops), (long)(nsops))
-#define __sanitizer_syscall_post_semop(res, semid, sops, nsops) \
- __sanitizer_syscall_post_impl_semop(res, (long)(semid), (long)(sops), \
+#define __sanitizer_syscall_post_semop(res, semid, sops, nsops) \
+ __sanitizer_syscall_post_impl_semop(res, (long)(semid), (long)(sops), \
(long)(nsops))
-#define __sanitizer_syscall_pre_semctl(semid, semnum, cmd, arg) \
- __sanitizer_syscall_pre_impl_semctl((long)(semid), (long)(semnum), \
+#define __sanitizer_syscall_pre_semctl(semid, semnum, cmd, arg) \
+ __sanitizer_syscall_pre_impl_semctl((long)(semid), (long)(semnum), \
(long)(cmd), (long)(arg))
-#define __sanitizer_syscall_post_semctl(res, semid, semnum, cmd, arg) \
- __sanitizer_syscall_post_impl_semctl(res, (long)(semid), (long)(semnum), \
+#define __sanitizer_syscall_post_semctl(res, semid, semnum, cmd, arg) \
+ __sanitizer_syscall_post_impl_semctl(res, (long)(semid), (long)(semnum), \
(long)(cmd), (long)(arg))
-#define __sanitizer_syscall_pre_semtimedop(semid, sops, nsops, timeout) \
- __sanitizer_syscall_pre_impl_semtimedop((long)(semid), (long)(sops), \
+#define __sanitizer_syscall_pre_semtimedop(semid, sops, nsops, timeout) \
+ __sanitizer_syscall_pre_impl_semtimedop((long)(semid), (long)(sops), \
(long)(nsops), (long)(timeout))
-#define __sanitizer_syscall_post_semtimedop(res, semid, sops, nsops, timeout) \
- __sanitizer_syscall_post_impl_semtimedop(res, (long)(semid), (long)(sops), \
+#define __sanitizer_syscall_post_semtimedop(res, semid, sops, nsops, timeout) \
+ __sanitizer_syscall_post_impl_semtimedop(res, (long)(semid), (long)(sops), \
(long)(nsops), (long)(timeout))
-#define __sanitizer_syscall_pre_shmat(shmid, shmaddr, shmflg) \
- __sanitizer_syscall_pre_impl_shmat((long)(shmid), (long)(shmaddr), \
+#define __sanitizer_syscall_pre_shmat(shmid, shmaddr, shmflg) \
+ __sanitizer_syscall_pre_impl_shmat((long)(shmid), (long)(shmaddr), \
(long)(shmflg))
-#define __sanitizer_syscall_post_shmat(res, shmid, shmaddr, shmflg) \
- __sanitizer_syscall_post_impl_shmat(res, (long)(shmid), (long)(shmaddr), \
+#define __sanitizer_syscall_post_shmat(res, shmid, shmaddr, shmflg) \
+ __sanitizer_syscall_post_impl_shmat(res, (long)(shmid), (long)(shmaddr), \
(long)(shmflg))
-#define __sanitizer_syscall_pre_shmget(key, size, flag) \
+#define __sanitizer_syscall_pre_shmget(key, size, flag) \
__sanitizer_syscall_pre_impl_shmget((long)(key), (long)(size), (long)(flag))
-#define __sanitizer_syscall_post_shmget(res, key, size, flag) \
- __sanitizer_syscall_post_impl_shmget(res, (long)(key), (long)(size), \
+#define __sanitizer_syscall_post_shmget(res, key, size, flag) \
+ __sanitizer_syscall_post_impl_shmget(res, (long)(key), (long)(size), \
(long)(flag))
-#define __sanitizer_syscall_pre_shmdt(shmaddr) \
+#define __sanitizer_syscall_pre_shmdt(shmaddr) \
__sanitizer_syscall_pre_impl_shmdt((long)(shmaddr))
-#define __sanitizer_syscall_post_shmdt(res, shmaddr) \
+#define __sanitizer_syscall_post_shmdt(res, shmaddr) \
__sanitizer_syscall_post_impl_shmdt(res, (long)(shmaddr))
-#define __sanitizer_syscall_pre_shmctl(shmid, cmd, buf) \
+#define __sanitizer_syscall_pre_shmctl(shmid, cmd, buf) \
__sanitizer_syscall_pre_impl_shmctl((long)(shmid), (long)(cmd), (long)(buf))
-#define __sanitizer_syscall_post_shmctl(res, shmid, cmd, buf) \
- __sanitizer_syscall_post_impl_shmctl(res, (long)(shmid), (long)(cmd), \
+#define __sanitizer_syscall_post_shmctl(res, shmid, cmd, buf) \
+ __sanitizer_syscall_post_impl_shmctl(res, (long)(shmid), (long)(cmd), \
(long)(buf))
#define __sanitizer_syscall_pre_ipc(call, first, second, third, ptr, fifth) \
__sanitizer_syscall_pre_impl_ipc((long)(call), (long)(first), \
(long)(second), (long)(third), (long)(ptr), \
(long)(fifth))
-#define __sanitizer_syscall_post_ipc(res, call, first, second, third, ptr, \
- fifth) \
- __sanitizer_syscall_post_impl_ipc(res, (long)(call), (long)(first), \
- (long)(second), (long)(third), \
+#define __sanitizer_syscall_post_ipc(res, call, first, second, third, ptr, \
+ fifth) \
+ __sanitizer_syscall_post_impl_ipc(res, (long)(call), (long)(first), \
+ (long)(second), (long)(third), \
(long)(ptr), (long)(fifth))
-#define __sanitizer_syscall_pre_mq_open(name, oflag, mode, attr) \
- __sanitizer_syscall_pre_impl_mq_open((long)(name), (long)(oflag), \
+#define __sanitizer_syscall_pre_mq_open(name, oflag, mode, attr) \
+ __sanitizer_syscall_pre_impl_mq_open((long)(name), (long)(oflag), \
(long)(mode), (long)(attr))
-#define __sanitizer_syscall_post_mq_open(res, name, oflag, mode, attr) \
- __sanitizer_syscall_post_impl_mq_open(res, (long)(name), (long)(oflag), \
+#define __sanitizer_syscall_post_mq_open(res, name, oflag, mode, attr) \
+ __sanitizer_syscall_post_impl_mq_open(res, (long)(name), (long)(oflag), \
(long)(mode), (long)(attr))
-#define __sanitizer_syscall_pre_mq_unlink(name) \
+#define __sanitizer_syscall_pre_mq_unlink(name) \
__sanitizer_syscall_pre_impl_mq_unlink((long)(name))
-#define __sanitizer_syscall_post_mq_unlink(res, name) \
+#define __sanitizer_syscall_post_mq_unlink(res, name) \
__sanitizer_syscall_post_impl_mq_unlink(res, (long)(name))
#define __sanitizer_syscall_pre_mq_timedsend(mqdes, msg_ptr, msg_len, \
msg_prio, abs_timeout) \
__sanitizer_syscall_pre_impl_mq_timedsend((long)(mqdes), (long)(msg_ptr), \
(long)(msg_len), (long)(msg_prio), \
(long)(abs_timeout))
-#define __sanitizer_syscall_post_mq_timedsend(res, mqdes, msg_ptr, msg_len, \
- msg_prio, abs_timeout) \
- __sanitizer_syscall_post_impl_mq_timedsend( \
- res, (long)(mqdes), (long)(msg_ptr), (long)(msg_len), (long)(msg_prio), \
+#define __sanitizer_syscall_post_mq_timedsend(res, mqdes, msg_ptr, msg_len, \
+ msg_prio, abs_timeout) \
+ __sanitizer_syscall_post_impl_mq_timedsend( \
+ res, (long)(mqdes), (long)(msg_ptr), (long)(msg_len), (long)(msg_prio), \
(long)(abs_timeout))
-#define __sanitizer_syscall_pre_mq_timedreceive(mqdes, msg_ptr, msg_len, \
- msg_prio, abs_timeout) \
- __sanitizer_syscall_pre_impl_mq_timedreceive( \
- (long)(mqdes), (long)(msg_ptr), (long)(msg_len), (long)(msg_prio), \
+#define __sanitizer_syscall_pre_mq_timedreceive(mqdes, msg_ptr, msg_len, \
+ msg_prio, abs_timeout) \
+ __sanitizer_syscall_pre_impl_mq_timedreceive( \
+ (long)(mqdes), (long)(msg_ptr), (long)(msg_len), (long)(msg_prio), \
(long)(abs_timeout))
#define __sanitizer_syscall_post_mq_timedreceive(res, mqdes, msg_ptr, msg_len, \
msg_prio, abs_timeout) \
__sanitizer_syscall_post_impl_mq_timedreceive( \
res, (long)(mqdes), (long)(msg_ptr), (long)(msg_len), (long)(msg_prio), \
(long)(abs_timeout))
-#define __sanitizer_syscall_pre_mq_notify(mqdes, notification) \
+#define __sanitizer_syscall_pre_mq_notify(mqdes, notification) \
__sanitizer_syscall_pre_impl_mq_notify((long)(mqdes), (long)(notification))
-#define __sanitizer_syscall_post_mq_notify(res, mqdes, notification) \
- __sanitizer_syscall_post_impl_mq_notify(res, (long)(mqdes), \
+#define __sanitizer_syscall_post_mq_notify(res, mqdes, notification) \
+ __sanitizer_syscall_post_impl_mq_notify(res, (long)(mqdes), \
(long)(notification))
-#define __sanitizer_syscall_pre_mq_getsetattr(mqdes, mqstat, omqstat) \
- __sanitizer_syscall_pre_impl_mq_getsetattr((long)(mqdes), (long)(mqstat), \
+#define __sanitizer_syscall_pre_mq_getsetattr(mqdes, mqstat, omqstat) \
+ __sanitizer_syscall_pre_impl_mq_getsetattr((long)(mqdes), (long)(mqstat), \
(long)(omqstat))
-#define __sanitizer_syscall_post_mq_getsetattr(res, mqdes, mqstat, omqstat) \
- __sanitizer_syscall_post_impl_mq_getsetattr(res, (long)(mqdes), \
+#define __sanitizer_syscall_post_mq_getsetattr(res, mqdes, mqstat, omqstat) \
+ __sanitizer_syscall_post_impl_mq_getsetattr(res, (long)(mqdes), \
(long)(mqstat), (long)(omqstat))
-#define __sanitizer_syscall_pre_pciconfig_iobase(which, bus, devfn) \
- __sanitizer_syscall_pre_impl_pciconfig_iobase((long)(which), (long)(bus), \
+#define __sanitizer_syscall_pre_pciconfig_iobase(which, bus, devfn) \
+ __sanitizer_syscall_pre_impl_pciconfig_iobase((long)(which), (long)(bus), \
(long)(devfn))
-#define __sanitizer_syscall_post_pciconfig_iobase(res, which, bus, devfn) \
- __sanitizer_syscall_post_impl_pciconfig_iobase(res, (long)(which), \
+#define __sanitizer_syscall_post_pciconfig_iobase(res, which, bus, devfn) \
+ __sanitizer_syscall_post_impl_pciconfig_iobase(res, (long)(which), \
(long)(bus), (long)(devfn))
-#define __sanitizer_syscall_pre_pciconfig_read(bus, dfn, off, len, buf) \
- __sanitizer_syscall_pre_impl_pciconfig_read( \
+#define __sanitizer_syscall_pre_pciconfig_read(bus, dfn, off, len, buf) \
+ __sanitizer_syscall_pre_impl_pciconfig_read( \
(long)(bus), (long)(dfn), (long)(off), (long)(len), (long)(buf))
-#define __sanitizer_syscall_post_pciconfig_read(res, bus, dfn, off, len, buf) \
- __sanitizer_syscall_post_impl_pciconfig_read( \
+#define __sanitizer_syscall_post_pciconfig_read(res, bus, dfn, off, len, buf) \
+ __sanitizer_syscall_post_impl_pciconfig_read( \
res, (long)(bus), (long)(dfn), (long)(off), (long)(len), (long)(buf))
-#define __sanitizer_syscall_pre_pciconfig_write(bus, dfn, off, len, buf) \
- __sanitizer_syscall_pre_impl_pciconfig_write( \
+#define __sanitizer_syscall_pre_pciconfig_write(bus, dfn, off, len, buf) \
+ __sanitizer_syscall_pre_impl_pciconfig_write( \
(long)(bus), (long)(dfn), (long)(off), (long)(len), (long)(buf))
#define __sanitizer_syscall_post_pciconfig_write(res, bus, dfn, off, len, buf) \
__sanitizer_syscall_post_impl_pciconfig_write( \
res, (long)(bus), (long)(dfn), (long)(off), (long)(len), (long)(buf))
-#define __sanitizer_syscall_pre_swapon(specialfile, swap_flags) \
+#define __sanitizer_syscall_pre_swapon(specialfile, swap_flags) \
__sanitizer_syscall_pre_impl_swapon((long)(specialfile), (long)(swap_flags))
-#define __sanitizer_syscall_post_swapon(res, specialfile, swap_flags) \
- __sanitizer_syscall_post_impl_swapon(res, (long)(specialfile), \
+#define __sanitizer_syscall_post_swapon(res, specialfile, swap_flags) \
+ __sanitizer_syscall_post_impl_swapon(res, (long)(specialfile), \
(long)(swap_flags))
-#define __sanitizer_syscall_pre_swapoff(specialfile) \
+#define __sanitizer_syscall_pre_swapoff(specialfile) \
__sanitizer_syscall_pre_impl_swapoff((long)(specialfile))
-#define __sanitizer_syscall_post_swapoff(res, specialfile) \
+#define __sanitizer_syscall_post_swapoff(res, specialfile) \
__sanitizer_syscall_post_impl_swapoff(res, (long)(specialfile))
-#define __sanitizer_syscall_pre_sysctl(args) \
+#define __sanitizer_syscall_pre_sysctl(args) \
__sanitizer_syscall_pre_impl_sysctl((long)(args))
-#define __sanitizer_syscall_post_sysctl(res, args) \
+#define __sanitizer_syscall_post_sysctl(res, args) \
__sanitizer_syscall_post_impl_sysctl(res, (long)(args))
-#define __sanitizer_syscall_pre_sysinfo(info) \
+#define __sanitizer_syscall_pre_sysinfo(info) \
__sanitizer_syscall_pre_impl_sysinfo((long)(info))
-#define __sanitizer_syscall_post_sysinfo(res, info) \
+#define __sanitizer_syscall_post_sysinfo(res, info) \
__sanitizer_syscall_post_impl_sysinfo(res, (long)(info))
-#define __sanitizer_syscall_pre_sysfs(option, arg1, arg2) \
+#define __sanitizer_syscall_pre_sysfs(option, arg1, arg2) \
__sanitizer_syscall_pre_impl_sysfs((long)(option), (long)(arg1), (long)(arg2))
-#define __sanitizer_syscall_post_sysfs(res, option, arg1, arg2) \
- __sanitizer_syscall_post_impl_sysfs(res, (long)(option), (long)(arg1), \
+#define __sanitizer_syscall_post_sysfs(res, option, arg1, arg2) \
+ __sanitizer_syscall_post_impl_sysfs(res, (long)(option), (long)(arg1), \
(long)(arg2))
-#define __sanitizer_syscall_pre_syslog(type, buf, len) \
+#define __sanitizer_syscall_pre_syslog(type, buf, len) \
__sanitizer_syscall_pre_impl_syslog((long)(type), (long)(buf), (long)(len))
-#define __sanitizer_syscall_post_syslog(res, type, buf, len) \
- __sanitizer_syscall_post_impl_syslog(res, (long)(type), (long)(buf), \
+#define __sanitizer_syscall_post_syslog(res, type, buf, len) \
+ __sanitizer_syscall_post_impl_syslog(res, (long)(type), (long)(buf), \
(long)(len))
-#define __sanitizer_syscall_pre_uselib(library) \
+#define __sanitizer_syscall_pre_uselib(library) \
__sanitizer_syscall_pre_impl_uselib((long)(library))
-#define __sanitizer_syscall_post_uselib(res, library) \
+#define __sanitizer_syscall_post_uselib(res, library) \
__sanitizer_syscall_post_impl_uselib(res, (long)(library))
-#define __sanitizer_syscall_pre_ni_syscall() \
+#define __sanitizer_syscall_pre_ni_syscall() \
__sanitizer_syscall_pre_impl_ni_syscall()
-#define __sanitizer_syscall_post_ni_syscall(res) \
+#define __sanitizer_syscall_post_ni_syscall(res) \
__sanitizer_syscall_post_impl_ni_syscall(res)
-#define __sanitizer_syscall_pre_ptrace(request, pid, addr, data) \
- __sanitizer_syscall_pre_impl_ptrace((long)(request), (long)(pid), \
+#define __sanitizer_syscall_pre_ptrace(request, pid, addr, data) \
+ __sanitizer_syscall_pre_impl_ptrace((long)(request), (long)(pid), \
(long)(addr), (long)(data))
-#define __sanitizer_syscall_post_ptrace(res, request, pid, addr, data) \
- __sanitizer_syscall_post_impl_ptrace(res, (long)(request), (long)(pid), \
+#define __sanitizer_syscall_post_ptrace(res, request, pid, addr, data) \
+ __sanitizer_syscall_post_impl_ptrace(res, (long)(request), (long)(pid), \
(long)(addr), (long)(data))
-#define __sanitizer_syscall_pre_add_key(_type, _description, _payload, plen, \
- destringid) \
- __sanitizer_syscall_pre_impl_add_key((long)(_type), (long)(_description), \
- (long)(_payload), (long)(plen), \
+#define __sanitizer_syscall_pre_add_key(_type, _description, _payload, plen, \
+ destringid) \
+ __sanitizer_syscall_pre_impl_add_key((long)(_type), (long)(_description), \
+ (long)(_payload), (long)(plen), \
(long)(destringid))
-#define __sanitizer_syscall_post_add_key(res, _type, _description, _payload, \
- plen, destringid) \
- __sanitizer_syscall_post_impl_add_key( \
- res, (long)(_type), (long)(_description), (long)(_payload), \
+#define __sanitizer_syscall_post_add_key(res, _type, _description, _payload, \
+ plen, destringid) \
+ __sanitizer_syscall_post_impl_add_key( \
+ res, (long)(_type), (long)(_description), (long)(_payload), \
(long)(plen), (long)(destringid))
-#define __sanitizer_syscall_pre_request_key(_type, _description, \
- _callout_info, destringid) \
- __sanitizer_syscall_pre_impl_request_key( \
- (long)(_type), (long)(_description), (long)(_callout_info), \
+#define __sanitizer_syscall_pre_request_key(_type, _description, \
+ _callout_info, destringid) \
+ __sanitizer_syscall_pre_impl_request_key( \
+ (long)(_type), (long)(_description), (long)(_callout_info), \
(long)(destringid))
-#define __sanitizer_syscall_post_request_key(res, _type, _description, \
- _callout_info, destringid) \
- __sanitizer_syscall_post_impl_request_key( \
- res, (long)(_type), (long)(_description), (long)(_callout_info), \
+#define __sanitizer_syscall_post_request_key(res, _type, _description, \
+ _callout_info, destringid) \
+ __sanitizer_syscall_post_impl_request_key( \
+ res, (long)(_type), (long)(_description), (long)(_callout_info), \
(long)(destringid))
#define __sanitizer_syscall_pre_keyctl(cmd, arg2, arg3, arg4, arg5) \
__sanitizer_syscall_pre_impl_keyctl((long)(cmd), (long)(arg2), (long)(arg3), \
(long)(arg4), (long)(arg5))
-#define __sanitizer_syscall_post_keyctl(res, cmd, arg2, arg3, arg4, arg5) \
- __sanitizer_syscall_post_impl_keyctl(res, (long)(cmd), (long)(arg2), \
- (long)(arg3), (long)(arg4), \
+#define __sanitizer_syscall_post_keyctl(res, cmd, arg2, arg3, arg4, arg5) \
+ __sanitizer_syscall_post_impl_keyctl(res, (long)(cmd), (long)(arg2), \
+ (long)(arg3), (long)(arg4), \
(long)(arg5))
-#define __sanitizer_syscall_pre_ioprio_set(which, who, ioprio) \
- __sanitizer_syscall_pre_impl_ioprio_set((long)(which), (long)(who), \
+#define __sanitizer_syscall_pre_ioprio_set(which, who, ioprio) \
+ __sanitizer_syscall_pre_impl_ioprio_set((long)(which), (long)(who), \
(long)(ioprio))
-#define __sanitizer_syscall_post_ioprio_set(res, which, who, ioprio) \
- __sanitizer_syscall_post_impl_ioprio_set(res, (long)(which), (long)(who), \
+#define __sanitizer_syscall_post_ioprio_set(res, which, who, ioprio) \
+ __sanitizer_syscall_post_impl_ioprio_set(res, (long)(which), (long)(who), \
(long)(ioprio))
-#define __sanitizer_syscall_pre_ioprio_get(which, who) \
+#define __sanitizer_syscall_pre_ioprio_get(which, who) \
__sanitizer_syscall_pre_impl_ioprio_get((long)(which), (long)(who))
-#define __sanitizer_syscall_post_ioprio_get(res, which, who) \
+#define __sanitizer_syscall_post_ioprio_get(res, which, who) \
__sanitizer_syscall_post_impl_ioprio_get(res, (long)(which), (long)(who))
-#define __sanitizer_syscall_pre_set_mempolicy(mode, nmask, maxnode) \
- __sanitizer_syscall_pre_impl_set_mempolicy((long)(mode), (long)(nmask), \
+#define __sanitizer_syscall_pre_set_mempolicy(mode, nmask, maxnode) \
+ __sanitizer_syscall_pre_impl_set_mempolicy((long)(mode), (long)(nmask), \
(long)(maxnode))
-#define __sanitizer_syscall_post_set_mempolicy(res, mode, nmask, maxnode) \
- __sanitizer_syscall_post_impl_set_mempolicy(res, (long)(mode), \
+#define __sanitizer_syscall_post_set_mempolicy(res, mode, nmask, maxnode) \
+ __sanitizer_syscall_post_impl_set_mempolicy(res, (long)(mode), \
(long)(nmask), (long)(maxnode))
-#define __sanitizer_syscall_pre_migrate_pages(pid, maxnode, from, to) \
- __sanitizer_syscall_pre_impl_migrate_pages((long)(pid), (long)(maxnode), \
+#define __sanitizer_syscall_pre_migrate_pages(pid, maxnode, from, to) \
+ __sanitizer_syscall_pre_impl_migrate_pages((long)(pid), (long)(maxnode), \
(long)(from), (long)(to))
-#define __sanitizer_syscall_post_migrate_pages(res, pid, maxnode, from, to) \
- __sanitizer_syscall_post_impl_migrate_pages( \
+#define __sanitizer_syscall_post_migrate_pages(res, pid, maxnode, from, to) \
+ __sanitizer_syscall_post_impl_migrate_pages( \
res, (long)(pid), (long)(maxnode), (long)(from), (long)(to))
-#define __sanitizer_syscall_pre_move_pages(pid, nr_pages, pages, nodes, \
- status, flags) \
- __sanitizer_syscall_pre_impl_move_pages((long)(pid), (long)(nr_pages), \
- (long)(pages), (long)(nodes), \
+#define __sanitizer_syscall_pre_move_pages(pid, nr_pages, pages, nodes, \
+ status, flags) \
+ __sanitizer_syscall_pre_impl_move_pages((long)(pid), (long)(nr_pages), \
+ (long)(pages), (long)(nodes), \
(long)(status), (long)(flags))
#define __sanitizer_syscall_post_move_pages(res, pid, nr_pages, pages, nodes, \
status, flags) \
__sanitizer_syscall_pre_impl_mbind((long)(start), (long)(len), (long)(mode), \
(long)(nmask), (long)(maxnode), \
(long)(flags))
-#define __sanitizer_syscall_post_mbind(res, start, len, mode, nmask, maxnode, \
- flags) \
- __sanitizer_syscall_post_impl_mbind(res, (long)(start), (long)(len), \
- (long)(mode), (long)(nmask), \
+#define __sanitizer_syscall_post_mbind(res, start, len, mode, nmask, maxnode, \
+ flags) \
+ __sanitizer_syscall_post_impl_mbind(res, (long)(start), (long)(len), \
+ (long)(mode), (long)(nmask), \
(long)(maxnode), (long)(flags))
-#define __sanitizer_syscall_pre_get_mempolicy(policy, nmask, maxnode, addr, \
- flags) \
- __sanitizer_syscall_pre_impl_get_mempolicy((long)(policy), (long)(nmask), \
- (long)(maxnode), (long)(addr), \
+#define __sanitizer_syscall_pre_get_mempolicy(policy, nmask, maxnode, addr, \
+ flags) \
+ __sanitizer_syscall_pre_impl_get_mempolicy((long)(policy), (long)(nmask), \
+ (long)(maxnode), (long)(addr), \
(long)(flags))
-#define __sanitizer_syscall_post_get_mempolicy(res, policy, nmask, maxnode, \
- addr, flags) \
- __sanitizer_syscall_post_impl_get_mempolicy(res, (long)(policy), \
- (long)(nmask), (long)(maxnode), \
+#define __sanitizer_syscall_post_get_mempolicy(res, policy, nmask, maxnode, \
+ addr, flags) \
+ __sanitizer_syscall_post_impl_get_mempolicy(res, (long)(policy), \
+ (long)(nmask), (long)(maxnode), \
(long)(addr), (long)(flags))
-#define __sanitizer_syscall_pre_inotify_init() \
+#define __sanitizer_syscall_pre_inotify_init() \
__sanitizer_syscall_pre_impl_inotify_init()
-#define __sanitizer_syscall_post_inotify_init(res) \
+#define __sanitizer_syscall_post_inotify_init(res) \
__sanitizer_syscall_post_impl_inotify_init(res)
-#define __sanitizer_syscall_pre_inotify_init1(flags) \
+#define __sanitizer_syscall_pre_inotify_init1(flags) \
__sanitizer_syscall_pre_impl_inotify_init1((long)(flags))
-#define __sanitizer_syscall_post_inotify_init1(res, flags) \
+#define __sanitizer_syscall_post_inotify_init1(res, flags) \
__sanitizer_syscall_post_impl_inotify_init1(res, (long)(flags))
-#define __sanitizer_syscall_pre_inotify_add_watch(fd, path, mask) \
- __sanitizer_syscall_pre_impl_inotify_add_watch((long)(fd), (long)(path), \
+#define __sanitizer_syscall_pre_inotify_add_watch(fd, path, mask) \
+ __sanitizer_syscall_pre_impl_inotify_add_watch((long)(fd), (long)(path), \
(long)(mask))
-#define __sanitizer_syscall_post_inotify_add_watch(res, fd, path, mask) \
- __sanitizer_syscall_post_impl_inotify_add_watch(res, (long)(fd), \
+#define __sanitizer_syscall_post_inotify_add_watch(res, fd, path, mask) \
+ __sanitizer_syscall_post_impl_inotify_add_watch(res, (long)(fd), \
(long)(path), (long)(mask))
-#define __sanitizer_syscall_pre_inotify_rm_watch(fd, wd) \
+#define __sanitizer_syscall_pre_inotify_rm_watch(fd, wd) \
__sanitizer_syscall_pre_impl_inotify_rm_watch((long)(fd), (long)(wd))
-#define __sanitizer_syscall_post_inotify_rm_watch(res, fd, wd) \
+#define __sanitizer_syscall_post_inotify_rm_watch(res, fd, wd) \
__sanitizer_syscall_post_impl_inotify_rm_watch(res, (long)(fd), (long)(wd))
-#define __sanitizer_syscall_pre_spu_run(fd, unpc, ustatus) \
- __sanitizer_syscall_pre_impl_spu_run((long)(fd), (long)(unpc), \
+#define __sanitizer_syscall_pre_spu_run(fd, unpc, ustatus) \
+ __sanitizer_syscall_pre_impl_spu_run((long)(fd), (long)(unpc), \
(long)(ustatus))
-#define __sanitizer_syscall_post_spu_run(res, fd, unpc, ustatus) \
- __sanitizer_syscall_post_impl_spu_run(res, (long)(fd), (long)(unpc), \
+#define __sanitizer_syscall_post_spu_run(res, fd, unpc, ustatus) \
+ __sanitizer_syscall_post_impl_spu_run(res, (long)(fd), (long)(unpc), \
(long)(ustatus))
-#define __sanitizer_syscall_pre_spu_create(name, flags, mode, fd) \
- __sanitizer_syscall_pre_impl_spu_create((long)(name), (long)(flags), \
+#define __sanitizer_syscall_pre_spu_create(name, flags, mode, fd) \
+ __sanitizer_syscall_pre_impl_spu_create((long)(name), (long)(flags), \
(long)(mode), (long)(fd))
-#define __sanitizer_syscall_post_spu_create(res, name, flags, mode, fd) \
- __sanitizer_syscall_post_impl_spu_create(res, (long)(name), (long)(flags), \
+#define __sanitizer_syscall_post_spu_create(res, name, flags, mode, fd) \
+ __sanitizer_syscall_post_impl_spu_create(res, (long)(name), (long)(flags), \
(long)(mode), (long)(fd))
-#define __sanitizer_syscall_pre_mknodat(dfd, filename, mode, dev) \
- __sanitizer_syscall_pre_impl_mknodat((long)(dfd), (long)(filename), \
+#define __sanitizer_syscall_pre_mknodat(dfd, filename, mode, dev) \
+ __sanitizer_syscall_pre_impl_mknodat((long)(dfd), (long)(filename), \
(long)(mode), (long)(dev))
-#define __sanitizer_syscall_post_mknodat(res, dfd, filename, mode, dev) \
- __sanitizer_syscall_post_impl_mknodat(res, (long)(dfd), (long)(filename), \
+#define __sanitizer_syscall_post_mknodat(res, dfd, filename, mode, dev) \
+ __sanitizer_syscall_post_impl_mknodat(res, (long)(dfd), (long)(filename), \
(long)(mode), (long)(dev))
-#define __sanitizer_syscall_pre_mkdirat(dfd, pathname, mode) \
- __sanitizer_syscall_pre_impl_mkdirat((long)(dfd), (long)(pathname), \
+#define __sanitizer_syscall_pre_mkdirat(dfd, pathname, mode) \
+ __sanitizer_syscall_pre_impl_mkdirat((long)(dfd), (long)(pathname), \
(long)(mode))
-#define __sanitizer_syscall_post_mkdirat(res, dfd, pathname, mode) \
- __sanitizer_syscall_post_impl_mkdirat(res, (long)(dfd), (long)(pathname), \
+#define __sanitizer_syscall_post_mkdirat(res, dfd, pathname, mode) \
+ __sanitizer_syscall_post_impl_mkdirat(res, (long)(dfd), (long)(pathname), \
(long)(mode))
-#define __sanitizer_syscall_pre_unlinkat(dfd, pathname, flag) \
- __sanitizer_syscall_pre_impl_unlinkat((long)(dfd), (long)(pathname), \
+#define __sanitizer_syscall_pre_unlinkat(dfd, pathname, flag) \
+ __sanitizer_syscall_pre_impl_unlinkat((long)(dfd), (long)(pathname), \
(long)(flag))
-#define __sanitizer_syscall_post_unlinkat(res, dfd, pathname, flag) \
- __sanitizer_syscall_post_impl_unlinkat(res, (long)(dfd), (long)(pathname), \
+#define __sanitizer_syscall_post_unlinkat(res, dfd, pathname, flag) \
+ __sanitizer_syscall_post_impl_unlinkat(res, (long)(dfd), (long)(pathname), \
(long)(flag))
-#define __sanitizer_syscall_pre_symlinkat(oldname, newdfd, newname) \
- __sanitizer_syscall_pre_impl_symlinkat((long)(oldname), (long)(newdfd), \
+#define __sanitizer_syscall_pre_symlinkat(oldname, newdfd, newname) \
+ __sanitizer_syscall_pre_impl_symlinkat((long)(oldname), (long)(newdfd), \
(long)(newname))
-#define __sanitizer_syscall_post_symlinkat(res, oldname, newdfd, newname) \
- __sanitizer_syscall_post_impl_symlinkat(res, (long)(oldname), \
+#define __sanitizer_syscall_post_symlinkat(res, oldname, newdfd, newname) \
+ __sanitizer_syscall_post_impl_symlinkat(res, (long)(oldname), \
(long)(newdfd), (long)(newname))
-#define __sanitizer_syscall_pre_linkat(olddfd, oldname, newdfd, newname, \
- flags) \
- __sanitizer_syscall_pre_impl_linkat((long)(olddfd), (long)(oldname), \
- (long)(newdfd), (long)(newname), \
+#define __sanitizer_syscall_pre_linkat(olddfd, oldname, newdfd, newname, \
+ flags) \
+ __sanitizer_syscall_pre_impl_linkat((long)(olddfd), (long)(oldname), \
+ (long)(newdfd), (long)(newname), \
(long)(flags))
#define __sanitizer_syscall_post_linkat(res, olddfd, oldname, newdfd, newname, \
flags) \
__sanitizer_syscall_post_impl_linkat(res, (long)(olddfd), (long)(oldname), \
(long)(newdfd), (long)(newname), \
(long)(flags))
-#define __sanitizer_syscall_pre_renameat(olddfd, oldname, newdfd, newname) \
- __sanitizer_syscall_pre_impl_renameat((long)(olddfd), (long)(oldname), \
+#define __sanitizer_syscall_pre_renameat(olddfd, oldname, newdfd, newname) \
+ __sanitizer_syscall_pre_impl_renameat((long)(olddfd), (long)(oldname), \
(long)(newdfd), (long)(newname))
#define __sanitizer_syscall_post_renameat(res, olddfd, oldname, newdfd, \
newname) \
__sanitizer_syscall_post_impl_renameat(res, (long)(olddfd), (long)(oldname), \
(long)(newdfd), (long)(newname))
-#define __sanitizer_syscall_pre_futimesat(dfd, filename, utimes) \
- __sanitizer_syscall_pre_impl_futimesat((long)(dfd), (long)(filename), \
+#define __sanitizer_syscall_pre_futimesat(dfd, filename, utimes) \
+ __sanitizer_syscall_pre_impl_futimesat((long)(dfd), (long)(filename), \
(long)(utimes))
-#define __sanitizer_syscall_post_futimesat(res, dfd, filename, utimes) \
- __sanitizer_syscall_post_impl_futimesat(res, (long)(dfd), (long)(filename), \
+#define __sanitizer_syscall_post_futimesat(res, dfd, filename, utimes) \
+ __sanitizer_syscall_post_impl_futimesat(res, (long)(dfd), (long)(filename), \
(long)(utimes))
-#define __sanitizer_syscall_pre_faccessat(dfd, filename, mode) \
- __sanitizer_syscall_pre_impl_faccessat((long)(dfd), (long)(filename), \
+#define __sanitizer_syscall_pre_faccessat(dfd, filename, mode) \
+ __sanitizer_syscall_pre_impl_faccessat((long)(dfd), (long)(filename), \
(long)(mode))
-#define __sanitizer_syscall_post_faccessat(res, dfd, filename, mode) \
- __sanitizer_syscall_post_impl_faccessat(res, (long)(dfd), (long)(filename), \
+#define __sanitizer_syscall_post_faccessat(res, dfd, filename, mode) \
+ __sanitizer_syscall_post_impl_faccessat(res, (long)(dfd), (long)(filename), \
(long)(mode))
-#define __sanitizer_syscall_pre_fchmodat(dfd, filename, mode) \
- __sanitizer_syscall_pre_impl_fchmodat((long)(dfd), (long)(filename), \
+#define __sanitizer_syscall_pre_fchmodat(dfd, filename, mode) \
+ __sanitizer_syscall_pre_impl_fchmodat((long)(dfd), (long)(filename), \
(long)(mode))
-#define __sanitizer_syscall_post_fchmodat(res, dfd, filename, mode) \
- __sanitizer_syscall_post_impl_fchmodat(res, (long)(dfd), (long)(filename), \
+#define __sanitizer_syscall_post_fchmodat(res, dfd, filename, mode) \
+ __sanitizer_syscall_post_impl_fchmodat(res, (long)(dfd), (long)(filename), \
(long)(mode))
-#define __sanitizer_syscall_pre_fchownat(dfd, filename, user, group, flag) \
- __sanitizer_syscall_pre_impl_fchownat((long)(dfd), (long)(filename), \
- (long)(user), (long)(group), \
+#define __sanitizer_syscall_pre_fchownat(dfd, filename, user, group, flag) \
+ __sanitizer_syscall_pre_impl_fchownat((long)(dfd), (long)(filename), \
+ (long)(user), (long)(group), \
(long)(flag))
-#define __sanitizer_syscall_post_fchownat(res, dfd, filename, user, group, \
- flag) \
- __sanitizer_syscall_post_impl_fchownat(res, (long)(dfd), (long)(filename), \
- (long)(user), (long)(group), \
+#define __sanitizer_syscall_post_fchownat(res, dfd, filename, user, group, \
+ flag) \
+ __sanitizer_syscall_post_impl_fchownat(res, (long)(dfd), (long)(filename), \
+ (long)(user), (long)(group), \
(long)(flag))
-#define __sanitizer_syscall_pre_openat(dfd, filename, flags, mode) \
- __sanitizer_syscall_pre_impl_openat((long)(dfd), (long)(filename), \
+#define __sanitizer_syscall_pre_openat(dfd, filename, flags, mode) \
+ __sanitizer_syscall_pre_impl_openat((long)(dfd), (long)(filename), \
(long)(flags), (long)(mode))
-#define __sanitizer_syscall_post_openat(res, dfd, filename, flags, mode) \
- __sanitizer_syscall_post_impl_openat(res, (long)(dfd), (long)(filename), \
+#define __sanitizer_syscall_post_openat(res, dfd, filename, flags, mode) \
+ __sanitizer_syscall_post_impl_openat(res, (long)(dfd), (long)(filename), \
(long)(flags), (long)(mode))
-#define __sanitizer_syscall_pre_newfstatat(dfd, filename, statbuf, flag) \
- __sanitizer_syscall_pre_impl_newfstatat((long)(dfd), (long)(filename), \
+#define __sanitizer_syscall_pre_newfstatat(dfd, filename, statbuf, flag) \
+ __sanitizer_syscall_pre_impl_newfstatat((long)(dfd), (long)(filename), \
(long)(statbuf), (long)(flag))
#define __sanitizer_syscall_post_newfstatat(res, dfd, filename, statbuf, flag) \
__sanitizer_syscall_post_impl_newfstatat(res, (long)(dfd), (long)(filename), \
(long)(statbuf), (long)(flag))
-#define __sanitizer_syscall_pre_fstatat64(dfd, filename, statbuf, flag) \
- __sanitizer_syscall_pre_impl_fstatat64((long)(dfd), (long)(filename), \
+#define __sanitizer_syscall_pre_fstatat64(dfd, filename, statbuf, flag) \
+ __sanitizer_syscall_pre_impl_fstatat64((long)(dfd), (long)(filename), \
(long)(statbuf), (long)(flag))
-#define __sanitizer_syscall_post_fstatat64(res, dfd, filename, statbuf, flag) \
- __sanitizer_syscall_post_impl_fstatat64(res, (long)(dfd), (long)(filename), \
+#define __sanitizer_syscall_post_fstatat64(res, dfd, filename, statbuf, flag) \
+ __sanitizer_syscall_post_impl_fstatat64(res, (long)(dfd), (long)(filename), \
(long)(statbuf), (long)(flag))
-#define __sanitizer_syscall_pre_readlinkat(dfd, path, buf, bufsiz) \
- __sanitizer_syscall_pre_impl_readlinkat((long)(dfd), (long)(path), \
+#define __sanitizer_syscall_pre_readlinkat(dfd, path, buf, bufsiz) \
+ __sanitizer_syscall_pre_impl_readlinkat((long)(dfd), (long)(path), \
(long)(buf), (long)(bufsiz))
-#define __sanitizer_syscall_post_readlinkat(res, dfd, path, buf, bufsiz) \
- __sanitizer_syscall_post_impl_readlinkat(res, (long)(dfd), (long)(path), \
+#define __sanitizer_syscall_post_readlinkat(res, dfd, path, buf, bufsiz) \
+ __sanitizer_syscall_post_impl_readlinkat(res, (long)(dfd), (long)(path), \
(long)(buf), (long)(bufsiz))
-#define __sanitizer_syscall_pre_utimensat(dfd, filename, utimes, flags) \
- __sanitizer_syscall_pre_impl_utimensat((long)(dfd), (long)(filename), \
+#define __sanitizer_syscall_pre_utimensat(dfd, filename, utimes, flags) \
+ __sanitizer_syscall_pre_impl_utimensat((long)(dfd), (long)(filename), \
(long)(utimes), (long)(flags))
-#define __sanitizer_syscall_post_utimensat(res, dfd, filename, utimes, flags) \
- __sanitizer_syscall_post_impl_utimensat(res, (long)(dfd), (long)(filename), \
+#define __sanitizer_syscall_post_utimensat(res, dfd, filename, utimes, flags) \
+ __sanitizer_syscall_post_impl_utimensat(res, (long)(dfd), (long)(filename), \
(long)(utimes), (long)(flags))
-#define __sanitizer_syscall_pre_unshare(unshare_flags) \
+#define __sanitizer_syscall_pre_unshare(unshare_flags) \
__sanitizer_syscall_pre_impl_unshare((long)(unshare_flags))
-#define __sanitizer_syscall_post_unshare(res, unshare_flags) \
+#define __sanitizer_syscall_post_unshare(res, unshare_flags) \
__sanitizer_syscall_post_impl_unshare(res, (long)(unshare_flags))
-#define __sanitizer_syscall_pre_splice(fd_in, off_in, fd_out, off_out, len, \
- flags) \
- __sanitizer_syscall_pre_impl_splice((long)(fd_in), (long)(off_in), \
- (long)(fd_out), (long)(off_out), \
+#define __sanitizer_syscall_pre_splice(fd_in, off_in, fd_out, off_out, len, \
+ flags) \
+ __sanitizer_syscall_pre_impl_splice((long)(fd_in), (long)(off_in), \
+ (long)(fd_out), (long)(off_out), \
(long)(len), (long)(flags))
-#define __sanitizer_syscall_post_splice(res, fd_in, off_in, fd_out, off_out, \
- len, flags) \
- __sanitizer_syscall_post_impl_splice(res, (long)(fd_in), (long)(off_in), \
- (long)(fd_out), (long)(off_out), \
+#define __sanitizer_syscall_post_splice(res, fd_in, off_in, fd_out, off_out, \
+ len, flags) \
+ __sanitizer_syscall_post_impl_splice(res, (long)(fd_in), (long)(off_in), \
+ (long)(fd_out), (long)(off_out), \
(long)(len), (long)(flags))
-#define __sanitizer_syscall_pre_vmsplice(fd, iov, nr_segs, flags) \
- __sanitizer_syscall_pre_impl_vmsplice((long)(fd), (long)(iov), \
+#define __sanitizer_syscall_pre_vmsplice(fd, iov, nr_segs, flags) \
+ __sanitizer_syscall_pre_impl_vmsplice((long)(fd), (long)(iov), \
(long)(nr_segs), (long)(flags))
-#define __sanitizer_syscall_post_vmsplice(res, fd, iov, nr_segs, flags) \
- __sanitizer_syscall_post_impl_vmsplice(res, (long)(fd), (long)(iov), \
+#define __sanitizer_syscall_post_vmsplice(res, fd, iov, nr_segs, flags) \
+ __sanitizer_syscall_post_impl_vmsplice(res, (long)(fd), (long)(iov), \
(long)(nr_segs), (long)(flags))
-#define __sanitizer_syscall_pre_tee(fdin, fdout, len, flags) \
- __sanitizer_syscall_pre_impl_tee((long)(fdin), (long)(fdout), (long)(len), \
+#define __sanitizer_syscall_pre_tee(fdin, fdout, len, flags) \
+ __sanitizer_syscall_pre_impl_tee((long)(fdin), (long)(fdout), (long)(len), \
(long)(flags))
-#define __sanitizer_syscall_post_tee(res, fdin, fdout, len, flags) \
- __sanitizer_syscall_post_impl_tee(res, (long)(fdin), (long)(fdout), \
+#define __sanitizer_syscall_post_tee(res, fdin, fdout, len, flags) \
+ __sanitizer_syscall_post_impl_tee(res, (long)(fdin), (long)(fdout), \
(long)(len), (long)(flags))
-#define __sanitizer_syscall_pre_get_robust_list(pid, head_ptr, len_ptr) \
- __sanitizer_syscall_pre_impl_get_robust_list((long)(pid), (long)(head_ptr), \
+#define __sanitizer_syscall_pre_get_robust_list(pid, head_ptr, len_ptr) \
+ __sanitizer_syscall_pre_impl_get_robust_list((long)(pid), (long)(head_ptr), \
(long)(len_ptr))
-#define __sanitizer_syscall_post_get_robust_list(res, pid, head_ptr, len_ptr) \
- __sanitizer_syscall_post_impl_get_robust_list( \
+#define __sanitizer_syscall_post_get_robust_list(res, pid, head_ptr, len_ptr) \
+ __sanitizer_syscall_post_impl_get_robust_list( \
res, (long)(pid), (long)(head_ptr), (long)(len_ptr))
-#define __sanitizer_syscall_pre_set_robust_list(head, len) \
+#define __sanitizer_syscall_pre_set_robust_list(head, len) \
__sanitizer_syscall_pre_impl_set_robust_list((long)(head), (long)(len))
-#define __sanitizer_syscall_post_set_robust_list(res, head, len) \
+#define __sanitizer_syscall_post_set_robust_list(res, head, len) \
__sanitizer_syscall_post_impl_set_robust_list(res, (long)(head), (long)(len))
-#define __sanitizer_syscall_pre_getcpu(cpu, node, cache) \
+#define __sanitizer_syscall_pre_getcpu(cpu, node, cache) \
__sanitizer_syscall_pre_impl_getcpu((long)(cpu), (long)(node), (long)(cache))
-#define __sanitizer_syscall_post_getcpu(res, cpu, node, cache) \
- __sanitizer_syscall_post_impl_getcpu(res, (long)(cpu), (long)(node), \
+#define __sanitizer_syscall_post_getcpu(res, cpu, node, cache) \
+ __sanitizer_syscall_post_impl_getcpu(res, (long)(cpu), (long)(node), \
(long)(cache))
-#define __sanitizer_syscall_pre_signalfd(ufd, user_mask, sizemask) \
- __sanitizer_syscall_pre_impl_signalfd((long)(ufd), (long)(user_mask), \
+#define __sanitizer_syscall_pre_signalfd(ufd, user_mask, sizemask) \
+ __sanitizer_syscall_pre_impl_signalfd((long)(ufd), (long)(user_mask), \
(long)(sizemask))
-#define __sanitizer_syscall_post_signalfd(res, ufd, user_mask, sizemask) \
- __sanitizer_syscall_post_impl_signalfd(res, (long)(ufd), (long)(user_mask), \
+#define __sanitizer_syscall_post_signalfd(res, ufd, user_mask, sizemask) \
+ __sanitizer_syscall_post_impl_signalfd(res, (long)(ufd), (long)(user_mask), \
(long)(sizemask))
-#define __sanitizer_syscall_pre_signalfd4(ufd, user_mask, sizemask, flags) \
- __sanitizer_syscall_pre_impl_signalfd4((long)(ufd), (long)(user_mask), \
+#define __sanitizer_syscall_pre_signalfd4(ufd, user_mask, sizemask, flags) \
+ __sanitizer_syscall_pre_impl_signalfd4((long)(ufd), (long)(user_mask), \
(long)(sizemask), (long)(flags))
#define __sanitizer_syscall_post_signalfd4(res, ufd, user_mask, sizemask, \
flags) \
__sanitizer_syscall_post_impl_signalfd4(res, (long)(ufd), (long)(user_mask), \
(long)(sizemask), (long)(flags))
-#define __sanitizer_syscall_pre_timerfd_create(clockid, flags) \
+#define __sanitizer_syscall_pre_timerfd_create(clockid, flags) \
__sanitizer_syscall_pre_impl_timerfd_create((long)(clockid), (long)(flags))
-#define __sanitizer_syscall_post_timerfd_create(res, clockid, flags) \
- __sanitizer_syscall_post_impl_timerfd_create(res, (long)(clockid), \
+#define __sanitizer_syscall_post_timerfd_create(res, clockid, flags) \
+ __sanitizer_syscall_post_impl_timerfd_create(res, (long)(clockid), \
(long)(flags))
-#define __sanitizer_syscall_pre_timerfd_settime(ufd, flags, utmr, otmr) \
- __sanitizer_syscall_pre_impl_timerfd_settime((long)(ufd), (long)(flags), \
+#define __sanitizer_syscall_pre_timerfd_settime(ufd, flags, utmr, otmr) \
+ __sanitizer_syscall_pre_impl_timerfd_settime((long)(ufd), (long)(flags), \
(long)(utmr), (long)(otmr))
-#define __sanitizer_syscall_post_timerfd_settime(res, ufd, flags, utmr, otmr) \
- __sanitizer_syscall_post_impl_timerfd_settime( \
+#define __sanitizer_syscall_post_timerfd_settime(res, ufd, flags, utmr, otmr) \
+ __sanitizer_syscall_post_impl_timerfd_settime( \
res, (long)(ufd), (long)(flags), (long)(utmr), (long)(otmr))
-#define __sanitizer_syscall_pre_timerfd_gettime(ufd, otmr) \
+#define __sanitizer_syscall_pre_timerfd_gettime(ufd, otmr) \
__sanitizer_syscall_pre_impl_timerfd_gettime((long)(ufd), (long)(otmr))
-#define __sanitizer_syscall_post_timerfd_gettime(res, ufd, otmr) \
+#define __sanitizer_syscall_post_timerfd_gettime(res, ufd, otmr) \
__sanitizer_syscall_post_impl_timerfd_gettime(res, (long)(ufd), (long)(otmr))
-#define __sanitizer_syscall_pre_eventfd(count) \
+#define __sanitizer_syscall_pre_eventfd(count) \
__sanitizer_syscall_pre_impl_eventfd((long)(count))
-#define __sanitizer_syscall_post_eventfd(res, count) \
+#define __sanitizer_syscall_post_eventfd(res, count) \
__sanitizer_syscall_post_impl_eventfd(res, (long)(count))
-#define __sanitizer_syscall_pre_eventfd2(count, flags) \
+#define __sanitizer_syscall_pre_eventfd2(count, flags) \
__sanitizer_syscall_pre_impl_eventfd2((long)(count), (long)(flags))
-#define __sanitizer_syscall_post_eventfd2(res, count, flags) \
+#define __sanitizer_syscall_post_eventfd2(res, count, flags) \
__sanitizer_syscall_post_impl_eventfd2(res, (long)(count), (long)(flags))
-#define __sanitizer_syscall_pre_old_readdir(arg0, arg1, arg2) \
- __sanitizer_syscall_pre_impl_old_readdir((long)(arg0), (long)(arg1), \
+#define __sanitizer_syscall_pre_old_readdir(arg0, arg1, arg2) \
+ __sanitizer_syscall_pre_impl_old_readdir((long)(arg0), (long)(arg1), \
(long)(arg2))
-#define __sanitizer_syscall_post_old_readdir(res, arg0, arg1, arg2) \
- __sanitizer_syscall_post_impl_old_readdir(res, (long)(arg0), (long)(arg1), \
+#define __sanitizer_syscall_post_old_readdir(res, arg0, arg1, arg2) \
+ __sanitizer_syscall_post_impl_old_readdir(res, (long)(arg0), (long)(arg1), \
(long)(arg2))
-#define __sanitizer_syscall_pre_pselect6(arg0, arg1, arg2, arg3, arg4, arg5) \
- __sanitizer_syscall_pre_impl_pselect6((long)(arg0), (long)(arg1), \
- (long)(arg2), (long)(arg3), \
+#define __sanitizer_syscall_pre_pselect6(arg0, arg1, arg2, arg3, arg4, arg5) \
+ __sanitizer_syscall_pre_impl_pselect6((long)(arg0), (long)(arg1), \
+ (long)(arg2), (long)(arg3), \
(long)(arg4), (long)(arg5))
-#define __sanitizer_syscall_post_pselect6(res, arg0, arg1, arg2, arg3, arg4, \
- arg5) \
- __sanitizer_syscall_post_impl_pselect6(res, (long)(arg0), (long)(arg1), \
- (long)(arg2), (long)(arg3), \
+#define __sanitizer_syscall_post_pselect6(res, arg0, arg1, arg2, arg3, arg4, \
+ arg5) \
+ __sanitizer_syscall_post_impl_pselect6(res, (long)(arg0), (long)(arg1), \
+ (long)(arg2), (long)(arg3), \
(long)(arg4), (long)(arg5))
#define __sanitizer_syscall_pre_ppoll(arg0, arg1, arg2, arg3, arg4) \
__sanitizer_syscall_pre_impl_ppoll((long)(arg0), (long)(arg1), (long)(arg2), \
(long)(arg3), (long)(arg4))
-#define __sanitizer_syscall_post_ppoll(res, arg0, arg1, arg2, arg3, arg4) \
- __sanitizer_syscall_post_impl_ppoll(res, (long)(arg0), (long)(arg1), \
- (long)(arg2), (long)(arg3), \
+#define __sanitizer_syscall_post_ppoll(res, arg0, arg1, arg2, arg3, arg4) \
+ __sanitizer_syscall_post_impl_ppoll(res, (long)(arg0), (long)(arg1), \
+ (long)(arg2), (long)(arg3), \
(long)(arg4))
-#define __sanitizer_syscall_pre_syncfs(fd) \
+#define __sanitizer_syscall_pre_syncfs(fd) \
__sanitizer_syscall_pre_impl_syncfs((long)(fd))
-#define __sanitizer_syscall_post_syncfs(res, fd) \
+#define __sanitizer_syscall_post_syncfs(res, fd) \
__sanitizer_syscall_post_impl_syncfs(res, (long)(fd))
#define __sanitizer_syscall_pre_perf_event_open(attr_uptr, pid, cpu, group_fd, \
flags) \
__sanitizer_syscall_pre_impl_perf_event_open((long)(attr_uptr), (long)(pid), \
(long)(cpu), (long)(group_fd), \
(long)(flags))
-#define __sanitizer_syscall_post_perf_event_open(res, attr_uptr, pid, cpu, \
- group_fd, flags) \
- __sanitizer_syscall_post_impl_perf_event_open( \
- res, (long)(attr_uptr), (long)(pid), (long)(cpu), (long)(group_fd), \
+#define __sanitizer_syscall_post_perf_event_open(res, attr_uptr, pid, cpu, \
+ group_fd, flags) \
+ __sanitizer_syscall_post_impl_perf_event_open( \
+ res, (long)(attr_uptr), (long)(pid), (long)(cpu), (long)(group_fd), \
(long)(flags))
-#define __sanitizer_syscall_pre_mmap_pgoff(addr, len, prot, flags, fd, pgoff) \
- __sanitizer_syscall_pre_impl_mmap_pgoff((long)(addr), (long)(len), \
- (long)(prot), (long)(flags), \
+#define __sanitizer_syscall_pre_mmap_pgoff(addr, len, prot, flags, fd, pgoff) \
+ __sanitizer_syscall_pre_impl_mmap_pgoff((long)(addr), (long)(len), \
+ (long)(prot), (long)(flags), \
(long)(fd), (long)(pgoff))
-#define __sanitizer_syscall_post_mmap_pgoff(res, addr, len, prot, flags, fd, \
- pgoff) \
- __sanitizer_syscall_post_impl_mmap_pgoff(res, (long)(addr), (long)(len), \
- (long)(prot), (long)(flags), \
+#define __sanitizer_syscall_post_mmap_pgoff(res, addr, len, prot, flags, fd, \
+ pgoff) \
+ __sanitizer_syscall_post_impl_mmap_pgoff(res, (long)(addr), (long)(len), \
+ (long)(prot), (long)(flags), \
(long)(fd), (long)(pgoff))
-#define __sanitizer_syscall_pre_old_mmap(arg) \
+#define __sanitizer_syscall_pre_old_mmap(arg) \
__sanitizer_syscall_pre_impl_old_mmap((long)(arg))
-#define __sanitizer_syscall_post_old_mmap(res, arg) \
+#define __sanitizer_syscall_post_old_mmap(res, arg) \
__sanitizer_syscall_post_impl_old_mmap(res, (long)(arg))
-#define __sanitizer_syscall_pre_name_to_handle_at(dfd, name, handle, mnt_id, \
- flag) \
- __sanitizer_syscall_pre_impl_name_to_handle_at( \
+#define __sanitizer_syscall_pre_name_to_handle_at(dfd, name, handle, mnt_id, \
+ flag) \
+ __sanitizer_syscall_pre_impl_name_to_handle_at( \
(long)(dfd), (long)(name), (long)(handle), (long)(mnt_id), (long)(flag))
-#define __sanitizer_syscall_post_name_to_handle_at(res, dfd, name, handle, \
- mnt_id, flag) \
- __sanitizer_syscall_post_impl_name_to_handle_at( \
- res, (long)(dfd), (long)(name), (long)(handle), (long)(mnt_id), \
+#define __sanitizer_syscall_post_name_to_handle_at(res, dfd, name, handle, \
+ mnt_id, flag) \
+ __sanitizer_syscall_post_impl_name_to_handle_at( \
+ res, (long)(dfd), (long)(name), (long)(handle), (long)(mnt_id), \
(long)(flag))
-#define __sanitizer_syscall_pre_open_by_handle_at(mountdirfd, handle, flags) \
- __sanitizer_syscall_pre_impl_open_by_handle_at( \
+#define __sanitizer_syscall_pre_open_by_handle_at(mountdirfd, handle, flags) \
+ __sanitizer_syscall_pre_impl_open_by_handle_at( \
(long)(mountdirfd), (long)(handle), (long)(flags))
-#define __sanitizer_syscall_post_open_by_handle_at(res, mountdirfd, handle, \
- flags) \
- __sanitizer_syscall_post_impl_open_by_handle_at( \
+#define __sanitizer_syscall_post_open_by_handle_at(res, mountdirfd, handle, \
+ flags) \
+ __sanitizer_syscall_post_impl_open_by_handle_at( \
res, (long)(mountdirfd), (long)(handle), (long)(flags))
-#define __sanitizer_syscall_pre_setns(fd, nstype) \
+#define __sanitizer_syscall_pre_setns(fd, nstype) \
__sanitizer_syscall_pre_impl_setns((long)(fd), (long)(nstype))
-#define __sanitizer_syscall_post_setns(res, fd, nstype) \
+#define __sanitizer_syscall_post_setns(res, fd, nstype) \
__sanitizer_syscall_post_impl_setns(res, (long)(fd), (long)(nstype))
-#define __sanitizer_syscall_pre_process_vm_readv(pid, lvec, liovcnt, rvec, \
- riovcnt, flags) \
- __sanitizer_syscall_pre_impl_process_vm_readv( \
- (long)(pid), (long)(lvec), (long)(liovcnt), (long)(rvec), \
+#define __sanitizer_syscall_pre_process_vm_readv(pid, lvec, liovcnt, rvec, \
+ riovcnt, flags) \
+ __sanitizer_syscall_pre_impl_process_vm_readv( \
+ (long)(pid), (long)(lvec), (long)(liovcnt), (long)(rvec), \
(long)(riovcnt), (long)(flags))
-#define __sanitizer_syscall_post_process_vm_readv(res, pid, lvec, liovcnt, \
- rvec, riovcnt, flags) \
- __sanitizer_syscall_post_impl_process_vm_readv( \
- res, (long)(pid), (long)(lvec), (long)(liovcnt), (long)(rvec), \
+#define __sanitizer_syscall_post_process_vm_readv(res, pid, lvec, liovcnt, \
+ rvec, riovcnt, flags) \
+ __sanitizer_syscall_post_impl_process_vm_readv( \
+ res, (long)(pid), (long)(lvec), (long)(liovcnt), (long)(rvec), \
(long)(riovcnt), (long)(flags))
-#define __sanitizer_syscall_pre_process_vm_writev(pid, lvec, liovcnt, rvec, \
- riovcnt, flags) \
- __sanitizer_syscall_pre_impl_process_vm_writev( \
- (long)(pid), (long)(lvec), (long)(liovcnt), (long)(rvec), \
+#define __sanitizer_syscall_pre_process_vm_writev(pid, lvec, liovcnt, rvec, \
+ riovcnt, flags) \
+ __sanitizer_syscall_pre_impl_process_vm_writev( \
+ (long)(pid), (long)(lvec), (long)(liovcnt), (long)(rvec), \
(long)(riovcnt), (long)(flags))
-#define __sanitizer_syscall_post_process_vm_writev(res, pid, lvec, liovcnt, \
- rvec, riovcnt, flags) \
- __sanitizer_syscall_post_impl_process_vm_writev( \
- res, (long)(pid), (long)(lvec), (long)(liovcnt), (long)(rvec), \
+#define __sanitizer_syscall_post_process_vm_writev(res, pid, lvec, liovcnt, \
+ rvec, riovcnt, flags) \
+ __sanitizer_syscall_post_impl_process_vm_writev( \
+ res, (long)(pid), (long)(lvec), (long)(liovcnt), (long)(rvec), \
(long)(riovcnt), (long)(flags))
-#define __sanitizer_syscall_pre_fork() \
- __sanitizer_syscall_pre_impl_fork()
-#define __sanitizer_syscall_post_fork(res) \
+#define __sanitizer_syscall_pre_fork() __sanitizer_syscall_pre_impl_fork()
+#define __sanitizer_syscall_post_fork(res) \
__sanitizer_syscall_post_impl_fork(res)
-#define __sanitizer_syscall_pre_vfork() \
- __sanitizer_syscall_pre_impl_vfork()
-#define __sanitizer_syscall_post_vfork(res) \
+#define __sanitizer_syscall_pre_vfork() __sanitizer_syscall_pre_impl_vfork()
+#define __sanitizer_syscall_post_vfork(res) \
__sanitizer_syscall_post_impl_vfork(res)
#define __sanitizer_syscall_pre_sigaction(signum, act, oldact) \
__sanitizer_syscall_pre_impl_sigaction((long)signum, (long)act, (long)oldact)
void __sanitizer_syscall_post_impl_epoll_pwait(long res, long epfd, long events,
long maxevents, long timeout,
long sigmask, long sigsetsize);
+void __sanitizer_syscall_pre_impl_epoll_pwait2(long epfd, long events,
+ long maxevents, long timeout,
+ long sigmask, long sigsetsize);
+void __sanitizer_syscall_post_impl_epoll_pwait2(long res, long epfd,
+ long events, long maxevents,
+ long timeout, long sigmask,
+ long sigsetsize);
void __sanitizer_syscall_pre_impl_gethostname(long name, long len);
void __sanitizer_syscall_post_impl_gethostname(long res, long name, long len);
void __sanitizer_syscall_pre_impl_sethostname(long name, long len);
void __sanitizer_syscall_pre_impl_sigaltstack(long ss, long oss);
void __sanitizer_syscall_post_impl_sigaltstack(long res, long ss, long oss);
#ifdef __cplusplus
-} // extern "C"
+} // extern "C"
#endif
-#endif // SANITIZER_LINUX_SYSCALL_HOOKS_H
+#endif // SANITIZER_LINUX_SYSCALL_HOOKS_H
/* Tell MSan about newly destroyed memory. Mark memory as uninitialized. */
void __sanitizer_dtor_callback(const volatile void* data, size_t size);
+ void __sanitizer_dtor_callback_fields(const volatile void *data, size_t size);
+ void __sanitizer_dtor_callback_vptr(const volatile void *data);
/* This function may be optionally provided by user and should return
a string containing Msan runtime options. See msan_flags.h for details. */
// if TSan should exit as if issues were detected.
int __tsan_on_finalize(int failed);
+// Release TSan internal memory in a best-effort manner.
+void __tsan_flush_memory();
+
#ifdef __cplusplus
} // extern "C"
#endif
add_subdirectory(builtins)
endif()
-if(COMPILER_RT_BUILD_CRT AND COMPILER_RT_HAS_CRT)
+if(COMPILER_RT_BUILD_CRT)
add_subdirectory(crt)
endif()
function(compiler_rt_build_runtime runtime)
string(TOUPPER ${runtime} runtime_uppercase)
if(COMPILER_RT_HAS_${runtime_uppercase})
- add_subdirectory(${runtime})
if(${runtime} STREQUAL tsan)
add_subdirectory(tsan/dd)
endif()
- if(${runtime} STREQUAL scudo)
+ if(${runtime} STREQUAL scudo_standalone)
add_subdirectory(scudo/standalone)
+ else()
+ add_subdirectory(${runtime})
endif()
endif()
endfunction()
asan_new_delete.cpp
)
+set(ASAN_STATIC_SOURCES
+ asan_rtl_static.cpp
+ )
+
+if (NOT WIN32 AND NOT APPLE)
+ list(APPEND ASAN_STATIC_SOURCES
+ asan_rtl_x86_64.S
+ )
+endif()
+
set(ASAN_PREINIT_SOURCES
asan_preinit.cpp
)
append_rtti_flag(OFF ASAN_CFLAGS)
+# Silence warnings in system headers with MSVC.
+if(NOT CLANG_CL)
+ append_list_if(COMPILER_RT_HAS_EXTERNAL_FLAG "/experimental:external /external:W0 /external:anglebrackets" ASAN_CFLAGS)
+endif()
+
+# Too many existing bugs, needs cleanup.
+append_list_if(COMPILER_RT_HAS_WNO_FORMAT -Wno-format ASAN_CFLAGS)
+
set(ASAN_DYNAMIC_LINK_FLAGS ${SANITIZER_COMMON_LINK_FLAGS})
if(ANDROID)
-ftls-model=initial-exec ASAN_DYNAMIC_CFLAGS)
append_list_if(MSVC /DEBUG ASAN_DYNAMIC_LINK_FLAGS)
-set(ASAN_DYNAMIC_LIBS ${SANITIZER_CXX_ABI_LIBRARIES} ${SANITIZER_COMMON_LINK_LIBS})
+set(ASAN_DYNAMIC_LIBS
+ ${COMPILER_RT_UNWINDER_LINK_LIBS}
+ ${SANITIZER_CXX_ABI_LIBRARIES}
+ ${SANITIZER_COMMON_LINK_LIBS})
append_list_if(COMPILER_RT_HAS_LIBDL dl ASAN_DYNAMIC_LIBS)
append_list_if(COMPILER_RT_HAS_LIBRT rt ASAN_DYNAMIC_LIBS)
ADDITIONAL_HEADERS ${ASAN_HEADERS}
CFLAGS ${ASAN_CFLAGS}
DEFS ${ASAN_COMMON_DEFINITIONS})
+ add_compiler_rt_object_libraries(RTAsan_static
+ ARCHS ${ASAN_SUPPORTED_ARCH}
+ SOURCES ${ASAN_STATIC_SOURCES}
+ ADDITIONAL_HEADERS ${ASAN_HEADERS}
+ CFLAGS ${ASAN_CFLAGS}
+ DEFS ${ASAN_COMMON_DEFINITIONS})
add_compiler_rt_object_libraries(RTAsan_preinit
ARCHS ${ASAN_SUPPORTED_ARCH}
SOURCES ${ASAN_PREINIT_SOURCES}
LINK_FLAGS ${WEAK_SYMBOL_LINK_FLAGS}
DEFS ${ASAN_DYNAMIC_DEFINITIONS}
PARENT_TARGET asan)
+
+ add_compiler_rt_runtime(clang_rt.asan_static
+ STATIC
+ ARCHS ${ASAN_SUPPORTED_ARCH}
+ OBJECT_LIBS RTAsan_static
+ CFLAGS ${ASAN_CFLAGS}
+ DEFS ${ASAN_COMMON_DEFINITIONS}
+ PARENT_TARGET asan)
else()
# Build separate libraries for each target.
DEFS ${ASAN_COMMON_DEFINITIONS}
PARENT_TARGET asan)
+ add_compiler_rt_runtime(clang_rt.asan_static
+ STATIC
+ ARCHS ${ASAN_SUPPORTED_ARCH}
+ OBJECT_LIBS RTAsan_static
+ CFLAGS ${ASAN_CFLAGS}
+ DEFS ${ASAN_COMMON_DEFINITIONS}
+ PARENT_TARGET asan)
+
add_compiler_rt_runtime(clang_rt.asan-preinit
STATIC
ARCHS ${ASAN_SUPPORTED_ARCH}
add_compiler_rt_object_libraries(AsanWeakInterception
${SANITIZER_COMMON_SUPPORTED_OS}
ARCHS ${arch}
- SOURCES asan_win_weak_interception.cpp
+ SOURCES
+ asan_win_weak_interception.cpp
CFLAGS ${ASAN_CFLAGS} -DSANITIZER_DYNAMIC
DEFS ${ASAN_COMMON_DEFINITIONS})
set(ASAN_DYNAMIC_WEAK_INTERCEPTION
disabled.quarantine_size_mb = 0;
disabled.thread_local_quarantine_size_kb = 0;
// Redzone must be at least Max(16, granularity) bytes long.
- disabled.min_redzone = Max(16, (int)SHADOW_GRANULARITY);
+ disabled.min_redzone = Max(16, (int)ASAN_SHADOW_GRANULARITY);
disabled.max_redzone = disabled.min_redzone;
disabled.alloc_dealloc_mismatch = false;
disabled.may_return_null = true;
public:
uptr UsedSize() const {
- uptr R = user_requested_size_lo;
- if (sizeof(uptr) > sizeof(user_requested_size_lo))
- R += (uptr)user_requested_size_hi << (8 * sizeof(user_requested_size_lo));
- return R;
+ static_assert(sizeof(user_requested_size_lo) == 4,
+ "Expression below requires this");
+ return FIRST_32_SECOND_64(0, ((uptr)user_requested_size_hi << 32)) +
+ user_requested_size_lo;
}
void SetUsedSize(uptr size) {
user_requested_size_lo = size;
- if (sizeof(uptr) > sizeof(user_requested_size_lo)) {
- size >>= (8 * sizeof(user_requested_size_lo));
- user_requested_size_hi = size;
- CHECK_EQ(user_requested_size_hi, size);
- }
+ static_assert(sizeof(user_requested_size_lo) == 4,
+ "Expression below requires this");
+ user_requested_size_hi = FIRST_32_SECOND_64(0, size >> 32);
+ CHECK_EQ(UsedSize(), size);
}
void SetAllocContext(u32 tid, u32 stack) {
CHECK_EQ(old_chunk_state, CHUNK_QUARANTINE);
}
- PoisonShadow(m->Beg(),
- RoundUpTo(m->UsedSize(), SHADOW_GRANULARITY),
+ PoisonShadow(m->Beg(), RoundUpTo(m->UsedSize(), ASAN_SHADOW_GRANULARITY),
kAsanHeapLeftRedzoneMagic);
// Statistics.
QuarantineCache fallback_quarantine_cache;
uptr max_user_defined_malloc_size;
- atomic_uint8_t rss_limit_exceeded;
// ------------------- Options --------------------------
atomic_uint16_t min_redzone;
: kMaxAllowedMallocSize;
}
- bool RssLimitExceeded() {
- return atomic_load(&rss_limit_exceeded, memory_order_relaxed);
- }
-
- void SetRssLimitExceeded(bool limit_exceeded) {
- atomic_store(&rss_limit_exceeded, limit_exceeded, memory_order_relaxed);
- }
-
void RePoisonChunk(uptr chunk) {
// This could be a user-facing chunk (with redzones), or some internal
// housekeeping chunk, like TransferBatch. Start by assuming the former.
if (chunk < beg && beg < end && end <= chunk_end) {
// Looks like a valid AsanChunk in use, poison redzones only.
PoisonShadow(chunk, beg - chunk, kAsanHeapLeftRedzoneMagic);
- uptr end_aligned_down = RoundDownTo(end, SHADOW_GRANULARITY);
+ uptr end_aligned_down = RoundDownTo(end, ASAN_SHADOW_GRANULARITY);
FastPoisonShadowPartialRightRedzone(
end_aligned_down, end - end_aligned_down,
chunk_end - end_aligned_down, kAsanHeapLeftRedzoneMagic);
AllocType alloc_type, bool can_fill) {
if (UNLIKELY(!asan_inited))
AsanInitFromRtl();
- if (RssLimitExceeded()) {
+ if (UNLIKELY(IsRssLimitExceeded())) {
if (AllocatorMayReturnNull())
return nullptr;
ReportRssLimitExceeded(stack);
}
Flags &fl = *flags();
CHECK(stack);
- const uptr min_alignment = SHADOW_GRANULARITY;
+ const uptr min_alignment = ASAN_SHADOW_GRANULARITY;
const uptr user_requested_alignment_log =
ComputeUserRequestedAlignmentLog(alignment);
if (alignment < min_alignment)
size > max_user_defined_malloc_size) {
if (AllocatorMayReturnNull()) {
Report("WARNING: AddressSanitizer failed to allocate 0x%zx bytes\n",
- (void*)size);
+ size);
return nullptr;
}
uptr malloc_limit =
m->SetAllocContext(t ? t->tid() : kMainTid, StackDepotPut(*stack));
uptr size_rounded_down_to_granularity =
- RoundDownTo(size, SHADOW_GRANULARITY);
+ RoundDownTo(size, ASAN_SHADOW_GRANULARITY);
// Unpoison the bulk of the memory region.
if (size_rounded_down_to_granularity)
PoisonShadow(user_beg, size_rounded_down_to_granularity, 0);
if (size != size_rounded_down_to_granularity && CanPoisonMemory()) {
u8 *shadow =
(u8 *)MemToShadow(user_beg + size_rounded_down_to_granularity);
- *shadow = fl.poison_partial ? (size & (SHADOW_GRANULARITY - 1)) : 0;
+ *shadow = fl.poison_partial ? (size & (ASAN_SHADOW_GRANULARITY - 1)) : 0;
}
AsanStats &thread_stats = GetCurrentThreadStats();
CHECK_LE(alloc_beg + sizeof(LargeChunkHeader), chunk_beg);
reinterpret_cast<LargeChunkHeader *>(alloc_beg)->Set(m);
}
- ASAN_MALLOC_HOOK(res, size);
+ RunMallocHooks(res, size);
return res;
}
}
// Poison the region.
- PoisonShadow(m->Beg(),
- RoundUpTo(m->UsedSize(), SHADOW_GRANULARITY),
+ PoisonShadow(m->Beg(), RoundUpTo(m->UsedSize(), ASAN_SHADOW_GRANULARITY),
kAsanHeapFreeMagic);
AsanStats &thread_stats = GetCurrentThreadStats();
return;
}
- ASAN_FREE_HOOK(ptr);
+ RunFreeHooks(ptr);
// Must mark the chunk as quarantined before any changes to its metadata.
// Do not quarantine given chunk if we failed to set CHUNK_QUARANTINE flag.
sptr offset = 0;
if (!m1 || AsanChunkView(m1).AddrIsAtLeft(addr, 1, &offset)) {
// The address is in the chunk's left redzone, so maybe it is actually
- // a right buffer overflow from the other chunk to the left.
- // Search a bit to the left to see if there is another chunk.
+ // a right buffer overflow from the other chunk before.
+ // Search a bit before to see if there is another chunk.
AsanChunk *m2 = nullptr;
for (uptr l = 1; l < GetPageSizeCached(); l++) {
m2 = GetAsanChunkByAddr(addr - l);
quarantine.PrintStats();
}
- void ForceLock() ACQUIRE(fallback_mutex) {
+ void ForceLock() SANITIZER_ACQUIRE(fallback_mutex) {
allocator.ForceLock();
fallback_mutex.Lock();
}
- void ForceUnlock() RELEASE(fallback_mutex) {
+ void ForceUnlock() SANITIZER_RELEASE(fallback_mutex) {
fallback_mutex.Unlock();
allocator.ForceUnlock();
}
return (AllocType)chunk_->alloc_type;
}
-static StackTrace GetStackTraceFromId(u32 id) {
- CHECK(id);
- StackTrace res = StackDepotGet(id);
- CHECK(res.trace);
- return res;
-}
-
u32 AsanChunkView::GetAllocStackId() const {
u32 tid = 0;
u32 stack = 0;
return stack;
}
-StackTrace AsanChunkView::GetAllocStack() const {
- return GetStackTraceFromId(GetAllocStackId());
-}
-
-StackTrace AsanChunkView::GetFreeStack() const {
- return GetStackTraceFromId(GetFreeStackId());
-}
-
void InitializeAllocator(const AllocatorOptions &options) {
instance.InitLinkerInitialized(options);
}
return instance.AllocationSize(reinterpret_cast<uptr>(ptr));
}
-void asan_mz_force_lock() NO_THREAD_SAFETY_ANALYSIS { instance.ForceLock(); }
-
-void asan_mz_force_unlock() NO_THREAD_SAFETY_ANALYSIS {
- instance.ForceUnlock();
+void asan_mz_force_lock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
+ instance.ForceLock();
}
-void AsanSoftRssLimitExceededCallback(bool limit_exceeded) {
- instance.SetRssLimitExceeded(limit_exceeded);
+void asan_mz_force_unlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
+ instance.ForceUnlock();
}
} // namespace __asan
}
uptr GetUserBegin(uptr chunk) {
+ // FIXME: All usecases provide chunk address, GetAsanChunkByAddrFastLocked is
+ // not needed.
__asan::AsanChunk *m = __asan::instance.GetAsanChunkByAddrFastLocked(chunk);
return m ? m->Beg() : 0;
}
return kIgnoreObjectSuccess;
}
-void GetAdditionalThreadContextPtrs(ThreadContextBase *tctx, void *ptrs) {
- // Look for the arg pointer of threads that have been created or are running.
- // This is necessary to prevent false positive leaks due to the AsanThread
- // holding the only live reference to a heap object. This can happen because
- // the `pthread_create()` interceptor doesn't wait for the child thread to
- // start before returning and thus loosing the the only live reference to the
- // heap object on the stack.
-
- __asan::AsanThreadContext *atctx =
- reinterpret_cast<__asan::AsanThreadContext *>(tctx);
- __asan::AsanThread *asan_thread = atctx->thread;
-
- // Note ThreadStatusRunning is required because there is a small window where
- // the thread status switches to `ThreadStatusRunning` but the `arg` pointer
- // still isn't on the stack yet.
- if (atctx->status != ThreadStatusCreated &&
- atctx->status != ThreadStatusRunning)
- return;
-
- uptr thread_arg = reinterpret_cast<uptr>(asan_thread->get_arg());
- if (!thread_arg)
- return;
-
- auto ptrsVec = reinterpret_cast<InternalMmapVector<uptr> *>(ptrs);
- ptrsVec->push_back(thread_arg);
-}
-
} // namespace __lsan
// ---------------------- Interface ---------------- {{{1
GET_STACK_TRACE_MALLOC;
return instance.UpdateAllocationStack((uptr)addr, &stack);
}
-
-#if !SANITIZER_SUPPORTS_WEAK_HOOKS
-// Provide default (no-op) implementation of malloc hooks.
-SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_malloc_hook,
- void *ptr, uptr size) {
- (void)ptr;
- (void)size;
-}
-
-SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_free_hook, void *ptr) {
- (void)ptr;
-}
-#endif
bool Eq(const AsanChunkView &c) const { return chunk_ == c.chunk_; }
u32 GetAllocStackId() const;
u32 GetFreeStackId() const;
- StackTrace GetAllocStack() const;
- StackTrace GetFreeStack() const;
AllocType GetAllocType() const;
bool AddrIsInside(uptr addr, uptr access_size, sptr *offset) const {
if (addr >= Beg() && (addr + access_size) <= End()) {
const uptr kAllocatorSpace = ~(uptr)0;
const uptr kAllocatorSize = 0x2000000000ULL; // 128G.
typedef VeryDenseSizeClassMap SizeClassMap;
-# elif defined(__aarch64__)
-// AArch64/SANITIZER_CAN_USE_ALLOCATOR64 is only for 42-bit VMA
-// so no need to different values for different VMA.
-const uptr kAllocatorSpace = 0x10000000000ULL;
-const uptr kAllocatorSize = 0x10000000000ULL; // 3T.
-typedef DefaultSizeClassMap SizeClassMap;
#elif defined(__sparc__)
const uptr kAllocatorSpace = ~(uptr)0;
const uptr kAllocatorSize = 0x20000000000ULL; // 2T.
#include "asan_mapping.h"
#include "asan_report.h"
#include "asan_thread.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
namespace {
using namespace __asan;
StackTrace stack(nullptr, 0);
if (alloc_stack) {
if (chunk.AllocTid() == kInvalidTid) return 0;
- stack = chunk.GetAllocStack();
+ stack = StackDepotGet(chunk.GetAllocStackId());
if (thread_id) *thread_id = chunk.AllocTid();
} else {
if (chunk.FreeTid() == kInvalidTid) return 0;
- stack = chunk.GetFreeStack();
+ stack = StackDepotGet(chunk.GetFreeStackId());
if (thread_id) *thread_id = chunk.FreeTid();
}
SANITIZER_INTERFACE_ATTRIBUTE
void __asan_get_shadow_mapping(uptr *shadow_scale, uptr *shadow_offset) {
if (shadow_scale)
- *shadow_scale = SHADOW_SCALE;
+ *shadow_scale = ASAN_SHADOW_SCALE;
if (shadow_offset)
- *shadow_offset = SHADOW_OFFSET;
+ *shadow_offset = ASAN_SHADOW_OFFSET;
}
str.append("%s", d.Location());
switch (descr.access_type) {
case kAccessTypeLeft:
- str.append("%p is located %zd bytes to the left of",
+ str.append("%p is located %zd bytes before",
(void *)descr.bad_addr, descr.offset);
break;
case kAccessTypeRight:
- str.append("%p is located %zd bytes to the right of",
+ str.append("%p is located %zd bytes after",
(void *)descr.bad_addr, descr.offset);
break;
case kAccessTypeInside:
}
str.append("'");
if (var.line > 0) {
- str.append(" (line %d)", var.line);
+ str.append(" (line %zd)", var.line);
}
if (pos_descr) {
Decorator d;
Decorator d;
str.append("%s", d.Location());
if (addr < g.beg) {
- str.append("%p is located %zd bytes to the left", (void *)addr,
+ str.append("%p is located %zd bytes before", (void *)addr,
g.beg - addr);
} else if (addr + access_size > g.beg + g.size) {
if (addr < g.beg + g.size) addr = g.beg + g.size;
- str.append("%p is located %zd bytes to the right", (void *)addr,
+ str.append("%p is located %zd bytes after", (void *)addr,
addr - (g.beg + g.size));
} else {
// Can it happen?
- str.append("%p is located %zd bytes inside", (void *)addr, addr - g.beg);
+ str.append("%p is located %zd bytes inside of", (void *)addr, addr - g.beg);
}
- str.append(" of global variable '%s' defined in '",
+ str.append(" global variable '%s' defined in '",
MaybeDemangleGlobalName(g.name));
PrintGlobalLocation(&str, g);
str.append("' (0x%zx) of size %zu\n", g.beg, g.size);
}
void ShadowAddressDescription::Print() const {
- Printf("Address %p is located in the %s area.\n", addr, ShadowNames[kind]);
+ Printf("Address %p is located in the %s area.\n", (void *)addr,
+ ShadowNames[kind]);
}
void GlobalAddressDescription::Print(const char *bug_type) const {
void StackAddressDescription::Print() const {
Decorator d;
Printf("%s", d.Location());
- Printf("Address %p is located in stack of thread %s", addr,
+ Printf("Address %p is located in stack of thread %s", (void *)addr,
AsanThreadIdAndName(tid).c_str());
if (!frame_descr) {
void WildAddressDescription::Print() const {
Printf("Address %p is a wild pointer inside of access range of size %p.\n",
- addr, access_size);
+ (void *)addr, (void *)access_size);
}
void PrintAddressDescription(uptr addr, uptr access_size,
void ErrorDoubleFree::Print() {
Decorator d;
Printf("%s", d.Error());
- Report(
- "ERROR: AddressSanitizer: attempting %s on %p in thread %s:\n",
- scariness.GetDescription(), addr_description.addr,
- AsanThreadIdAndName(tid).c_str());
+ Report("ERROR: AddressSanitizer: attempting %s on %p in thread %s:\n",
+ scariness.GetDescription(), (void *)addr_description.addr,
+ AsanThreadIdAndName(tid).c_str());
Printf("%s", d.Default());
scariness.Print();
GET_STACK_TRACE_FATAL(second_free_stack->trace[0],
void ErrorNewDeleteTypeMismatch::Print() {
Decorator d;
Printf("%s", d.Error());
- Report(
- "ERROR: AddressSanitizer: %s on %p in thread %s:\n",
- scariness.GetDescription(), addr_description.addr,
- AsanThreadIdAndName(tid).c_str());
+ Report("ERROR: AddressSanitizer: %s on %p in thread %s:\n",
+ scariness.GetDescription(), (void *)addr_description.addr,
+ AsanThreadIdAndName(tid).c_str());
Printf("%s object passed to delete has wrong type:\n", d.Default());
if (delete_size != 0) {
Printf(
Report(
"ERROR: AddressSanitizer: attempting free on address "
"which was not malloc()-ed: %p in thread %s\n",
- addr_description.Address(), AsanThreadIdAndName(tid).c_str());
+ (void *)addr_description.Address(), AsanThreadIdAndName(tid).c_str());
Printf("%s", d.Default());
CHECK_GT(free_stack->size, 0);
scariness.Print();
Printf("%s", d.Error());
Report("ERROR: AddressSanitizer: %s (%s vs %s) on %p\n",
scariness.GetDescription(), alloc_names[alloc_type],
- dealloc_names[dealloc_type], addr_description.Address());
+ dealloc_names[dealloc_type], (void *)addr_description.Address());
Printf("%s", d.Default());
CHECK_GT(dealloc_stack->size, 0);
scariness.Print();
Report(
"ERROR: AddressSanitizer: attempting to call malloc_usable_size() for "
"pointer which is not owned: %p\n",
- addr_description.Address());
+ (void *)addr_description.Address());
Printf("%s", d.Default());
stack->Print();
addr_description.Print();
Report(
"ERROR: AddressSanitizer: attempting to call "
"__sanitizer_get_allocated_size() for pointer which is not owned: %p\n",
- addr_description.Address());
+ (void *)addr_description.Address());
Printf("%s", d.Default());
stack->Print();
addr_description.Print();
void ErrorOutOfMemory::Print() {
Decorator d;
Printf("%s", d.Error());
- Report(
- "ERROR: AddressSanitizer: allocator is out of memory trying to allocate "
- "0x%zx bytes\n", requested_size);
+ ERROR_OOM("allocator is trying to allocate 0x%zx bytes\n", requested_size);
Printf("%s", d.Default());
stack->Print();
PrintHintAllocatorCannotReturnNull();
Report(
"ERROR: AddressSanitizer: %s: memory ranges [%p,%p) and [%p, %p) "
"overlap\n",
- bug_type, addr1_description.Address(),
- addr1_description.Address() + length1, addr2_description.Address(),
- addr2_description.Address() + length2);
+ bug_type, (void *)addr1_description.Address(),
+ (void *)(addr1_description.Address() + length1),
+ (void *)addr2_description.Address(),
+ (void *)(addr2_description.Address() + length2));
Printf("%s", d.Default());
scariness.Print();
stack->Print();
" end : %p\n"
" old_mid : %p\n"
" new_mid : %p\n",
- beg, end, old_mid, new_mid);
- uptr granularity = SHADOW_GRANULARITY;
+ (void *)beg, (void *)end, (void *)old_mid, (void *)new_mid);
+ uptr granularity = ASAN_SHADOW_GRANULARITY;
if (!IsAligned(beg, granularity))
- Report("ERROR: beg is not aligned by %d\n", granularity);
+ Report("ERROR: beg is not aligned by %zu\n", granularity);
+ stack->Print();
+ ReportErrorSummary(scariness.GetDescription(), stack);
+}
+
+void ErrorBadParamsToAnnotateDoubleEndedContiguousContainer::Print() {
+ Report(
+ "ERROR: AddressSanitizer: bad parameters to "
+ "__sanitizer_annotate_double_ended_contiguous_container:\n"
+ " storage_beg : %p\n"
+ " storage_end : %p\n"
+ " old_container_beg : %p\n"
+ " old_container_end : %p\n"
+ " new_container_beg : %p\n"
+ " new_container_end : %p\n",
+ (void *)storage_beg, (void *)storage_end, (void *)old_container_beg,
+ (void *)old_container_end, (void *)new_container_beg,
+ (void *)new_container_end);
+ uptr granularity = ASAN_SHADOW_GRANULARITY;
+ if (!IsAligned(storage_beg, granularity))
+ Report("ERROR: storage_beg is not aligned by %zu\n", granularity);
stack->Print();
ReportErrorSummary(scariness.GetDescription(), stack);
}
Decorator d;
Printf("%s", d.Error());
Report("ERROR: AddressSanitizer: %s (%p):\n", scariness.GetDescription(),
- global1.beg);
+ (void *)global1.beg);
Printf("%s", d.Default());
InternalScopedString g1_loc;
InternalScopedString g2_loc;
Decorator d;
Printf("%s", d.Error());
Report("ERROR: AddressSanitizer: %s: %p %p\n", scariness.GetDescription(),
- addr1_description.Address(), addr2_description.Address());
+ (void *)addr1_description.Address(),
+ (void *)addr2_description.Address());
Printf("%s", d.Default());
GET_STACK_TRACE_FATAL(pc, bp);
stack.Print();
if (AddrIsInMem(addr)) {
u8 *shadow_addr = (u8 *)MemToShadow(addr);
// If we are accessing 16 bytes, look at the second shadow byte.
- if (*shadow_addr == 0 && access_size > SHADOW_GRANULARITY) shadow_addr++;
+ if (*shadow_addr == 0 && access_size > ASAN_SHADOW_GRANULARITY)
+ shadow_addr++;
// If we are in the partial right redzone, look at the next shadow byte.
if (*shadow_addr > 0 && *shadow_addr < 128) shadow_addr++;
bool far_from_bounds = false;
str->append(
"Shadow byte legend (one shadow byte represents %d "
"application bytes):\n",
- (int)SHADOW_GRANULARITY);
+ (int)ASAN_SHADOW_GRANULARITY);
PrintShadowByte(str, " Addressable: ", 0);
str->append(" Partially addressable: ");
- for (u8 i = 1; i < SHADOW_GRANULARITY; i++) PrintShadowByte(str, "", i, " ");
+ for (u8 i = 1; i < ASAN_SHADOW_GRANULARITY; i++)
+ PrintShadowByte(str, "", i, " ");
str->append("\n");
PrintShadowByte(str, " Heap left redzone: ",
kAsanHeapLeftRedzoneMagic);
static void PrintShadowBytes(InternalScopedString *str, const char *before,
u8 *bytes, u8 *guilty, uptr n) {
Decorator d;
- if (before) str->append("%s%p:", before, bytes);
+ if (before)
+ str->append("%s%p:", before,
+ (void *)ShadowToMem(reinterpret_cast<uptr>(bytes)));
for (uptr i = 0; i < n; i++) {
u8 *p = bytes + i;
const char *before =
Printf("%s", d.Error());
uptr addr = addr_description.Address();
Report("ERROR: AddressSanitizer: %s on address %p at pc %p bp %p sp %p\n",
- bug_descr, (void *)addr, pc, bp, sp);
+ bug_descr, (void *)addr, (void *)pc, (void *)bp, (void *)sp);
Printf("%s", d.Default());
Printf("%s%s of size %zu at %p thread %s%s\n", d.Access(),
scariness.Scare(10, "null-deref");
} else if (signal.addr == signal.pc) {
scariness.Scare(60, "wild-jump");
- } else if (signal.write_flag == SignalContext::WRITE) {
+ } else if (signal.write_flag == SignalContext::Write) {
scariness.Scare(30, "wild-addr-write");
- } else if (signal.write_flag == SignalContext::READ) {
+ } else if (signal.write_flag == SignalContext::Read) {
scariness.Scare(20, "wild-addr-read");
} else {
scariness.Scare(25, "wild-addr");
void Print();
};
+struct ErrorBadParamsToAnnotateDoubleEndedContiguousContainer : ErrorBase {
+ const BufferedStackTrace *stack;
+ uptr storage_beg, storage_end, old_container_beg, old_container_end,
+ new_container_beg, new_container_end;
+
+ ErrorBadParamsToAnnotateDoubleEndedContiguousContainer() = default; // (*)
+ ErrorBadParamsToAnnotateDoubleEndedContiguousContainer(
+ u32 tid, BufferedStackTrace *stack_, uptr storage_beg_, uptr storage_end_,
+ uptr old_container_beg_, uptr old_container_end_, uptr new_container_beg_,
+ uptr new_container_end_)
+ : ErrorBase(tid, 10,
+ "bad-__sanitizer_annotate_double_ended_contiguous_container"),
+ stack(stack_),
+ storage_beg(storage_beg_),
+ storage_end(storage_end_),
+ old_container_beg(old_container_beg_),
+ old_container_end(old_container_end_),
+ new_container_beg(new_container_beg_),
+ new_container_end(new_container_end_) {}
+ void Print();
+};
+
struct ErrorODRViolation : ErrorBase {
__asan_global global1, global2;
u32 stack_id1, stack_id2;
u8 shadow_val;
ErrorGeneric() = default; // (*)
- ErrorGeneric(u32 tid, uptr addr, uptr pc_, uptr bp_, uptr sp_, bool is_write_,
+ ErrorGeneric(u32 tid, uptr pc_, uptr bp_, uptr sp_, uptr addr, bool is_write_,
uptr access_size_);
void Print();
};
// clang-format off
-#define ASAN_FOR_EACH_ERROR_KIND(macro) \
- macro(DeadlySignal) \
- macro(DoubleFree) \
- macro(NewDeleteTypeMismatch) \
- macro(FreeNotMalloced) \
- macro(AllocTypeMismatch) \
- macro(MallocUsableSizeNotOwned) \
- macro(SanitizerGetAllocatedSizeNotOwned) \
- macro(CallocOverflow) \
- macro(ReallocArrayOverflow) \
- macro(PvallocOverflow) \
- macro(InvalidAllocationAlignment) \
- macro(InvalidAlignedAllocAlignment) \
- macro(InvalidPosixMemalignAlignment) \
- macro(AllocationSizeTooBig) \
- macro(RssLimitExceeded) \
- macro(OutOfMemory) \
- macro(StringFunctionMemoryRangesOverlap) \
- macro(StringFunctionSizeOverflow) \
- macro(BadParamsToAnnotateContiguousContainer) \
- macro(ODRViolation) \
- macro(InvalidPointerPair) \
+#define ASAN_FOR_EACH_ERROR_KIND(macro) \
+ macro(DeadlySignal) \
+ macro(DoubleFree) \
+ macro(NewDeleteTypeMismatch) \
+ macro(FreeNotMalloced) \
+ macro(AllocTypeMismatch) \
+ macro(MallocUsableSizeNotOwned) \
+ macro(SanitizerGetAllocatedSizeNotOwned) \
+ macro(CallocOverflow) \
+ macro(ReallocArrayOverflow) \
+ macro(PvallocOverflow) \
+ macro(InvalidAllocationAlignment) \
+ macro(InvalidAlignedAllocAlignment) \
+ macro(InvalidPosixMemalignAlignment) \
+ macro(AllocationSizeTooBig) \
+ macro(RssLimitExceeded) \
+ macro(OutOfMemory) \
+ macro(StringFunctionMemoryRangesOverlap) \
+ macro(StringFunctionSizeOverflow) \
+ macro(BadParamsToAnnotateContiguousContainer) \
+ macro(BadParamsToAnnotateDoubleEndedContiguousContainer) \
+ macro(ODRViolation) \
+ macro(InvalidPointerPair) \
macro(Generic)
// clang-format on
// For small size classes inline PoisonShadow for better performance.
ALWAYS_INLINE void SetShadow(uptr ptr, uptr size, uptr class_id, u64 magic) {
u64 *shadow = reinterpret_cast<u64*>(MemToShadow(ptr));
- if (SHADOW_SCALE == 3 && class_id <= 6) {
- // This code expects SHADOW_SCALE=3.
+ if (ASAN_SHADOW_SCALE == 3 && class_id <= 6) {
+ // This code expects ASAN_SHADOW_SCALE=3.
for (uptr i = 0; i < (((uptr)1) << class_id); i++) {
shadow[i] = magic;
// Make sure this does not become memset.
: MmapOrDie(size, "FakeStack"));
res->stack_size_log_ = stack_size_log;
u8 *p = reinterpret_cast<u8 *>(res);
- VReport(1, "T%d: FakeStack created: %p -- %p stack_size_log: %zd; "
+ VReport(1,
+ "T%d: FakeStack created: %p -- %p stack_size_log: %zd; "
"mmapped %zdK, noreserve=%d \n",
- GetCurrentTidOrInvalid(), p,
- p + FakeStack::RequiredSize(stack_size_log), stack_size_log,
+ GetCurrentTidOrInvalid(), (void *)p,
+ (void *)(p + FakeStack::RequiredSize(stack_size_log)), stack_size_log,
size >> 10, flags()->uar_noreserve);
return res;
}
// We do it based on their 'real_stack' values -- everything that is lower
// than the current real_stack is garbage.
NOINLINE void FakeStack::GC(uptr real_stack) {
- uptr collected = 0;
for (uptr class_id = 0; class_id < kNumberOfSizeClasses; class_id++) {
u8 *flags = GetFlags(stack_size_log(), class_id);
for (uptr i = 0, n = NumberOfFrames(stack_size_log(), class_id); i < n;
GetFrame(stack_size_log(), class_id, i));
if (ff->real_stack < real_stack) {
flags[i] = 0;
- collected++;
}
}
}
uptr LeftRedzoneAddr = addr - kAllocaRedzoneSize;
uptr PartialRzAddr = addr + size;
uptr RightRzAddr = (PartialRzAddr + kAllocaRedzoneMask) & ~kAllocaRedzoneMask;
- uptr PartialRzAligned = PartialRzAddr & ~(SHADOW_GRANULARITY - 1);
+ uptr PartialRzAligned = PartialRzAddr & ~(ASAN_SHADOW_GRANULARITY - 1);
FastPoisonShadow(LeftRedzoneAddr, kAllocaRedzoneSize, kAsanAllocaLeftMagic);
FastPoisonShadowPartialRightRedzone(
- PartialRzAligned, PartialRzAddr % SHADOW_GRANULARITY,
+ PartialRzAligned, PartialRzAddr % ASAN_SHADOW_GRANULARITY,
RightRzAddr - PartialRzAligned, kAsanAllocaRightMagic);
FastPoisonShadow(RightRzAddr, kAllocaRedzoneSize, kAsanAllocaRightMagic);
}
SANITIZER_INTERFACE_ATTRIBUTE
void __asan_allocas_unpoison(uptr top, uptr bottom) {
if ((!top) || (top > bottom)) return;
- REAL(memset)(reinterpret_cast<void*>(MemToShadow(top)), 0,
- (bottom - top) / SHADOW_GRANULARITY);
+ REAL(memset)
+ (reinterpret_cast<void *>(MemToShadow(top)), 0,
+ (bottom - top) / ASAN_SHADOW_GRANULARITY);
}
} // extern "C"
RegisterCommonFlags(&ubsan_parser);
#endif
- if (SANITIZER_MAC) {
+ if (SANITIZER_APPLE) {
// Support macOS MallocScribble and MallocPreScribble:
// <https://developer.apple.com/library/content/documentation/Performance/
// Conceptual/ManagingMemory/Articles/MallocDebug.html>
SanitizerToolName);
Die();
}
- // Ensure that redzone is at least SHADOW_GRANULARITY.
- if (f->redzone < (int)SHADOW_GRANULARITY)
- f->redzone = SHADOW_GRANULARITY;
+ // Ensure that redzone is at least ASAN_SHADOW_GRANULARITY.
+ if (f->redzone < (int)ASAN_SHADOW_GRANULARITY)
+ f->redzone = ASAN_SHADOW_GRANULARITY;
// Make "strict_init_order" imply "check_initialization_order".
// TODO(samsonov): Use a single runtime flag for an init-order checker.
if (f->strict_init_order) {
"to find more errors.")
ASAN_FLAG(bool, replace_intrin, true,
"If set, uses custom wrappers for memset/memcpy/memmove intrinsics.")
-ASAN_FLAG(bool, detect_stack_use_after_return, false,
+ASAN_FLAG(bool, detect_stack_use_after_return,
+ SANITIZER_LINUX && !SANITIZER_ANDROID,
"Enables stack-use-after-return checking at run-time.")
-ASAN_FLAG(int, min_uar_stack_size_log, 16, // We can't do smaller anyway.
+ASAN_FLAG(int, min_uar_stack_size_log, 16, // We can't do smaller anyway.
"Minimum fake stack size log.")
ASAN_FLAG(int, max_uar_stack_size_log,
20, // 1Mb per size class, i.e. ~11Mb per thread
int, sleep_after_init, 0,
"Number of seconds to sleep after AddressSanitizer is initialized. "
"Useful for debugging purposes (e.g. when one needs to attach gdb).")
+ASAN_FLAG(
+ int, sleep_before_init, 0,
+ "Number of seconds to sleep before AddressSanitizer starts initializing. "
+ "Useful for debugging purposes (e.g. when one needs to attach gdb).")
ASAN_FLAG(bool, check_malloc_usable_size, true,
"Allows the users to work around the bug in Nvidia drivers prior to "
"295.*.")
// https://github.com/google/sanitizers/issues/309
// TODO(glider,timurrrr): Fix known issues and enable this back.
ASAN_FLAG(bool, alloc_dealloc_mismatch,
- !SANITIZER_MAC && !SANITIZER_WINDOWS && !SANITIZER_ANDROID,
+ !SANITIZER_APPLE && !SANITIZER_WINDOWS && !SANITIZER_ANDROID,
"Report errors on malloc/delete, new/free, new/delete[], etc.")
ASAN_FLAG(bool, new_delete_type_mismatch, true,
#include "sanitizer_common/sanitizer_fuchsia.h"
#if SANITIZER_FUCHSIA
-#include "asan_interceptors.h"
-#include "asan_internal.h"
-#include "asan_stack.h"
-#include "asan_thread.h"
-
#include <limits.h>
#include <zircon/sanitizer.h>
#include <zircon/syscalls.h>
#include <zircon/threads.h>
+# include "asan_interceptors.h"
+# include "asan_internal.h"
+# include "asan_stack.h"
+# include "asan_thread.h"
+# include "lsan/lsan_common.h"
+
namespace __asan {
// The system already set up the shadow memory for us.
// AsanInitInternal->InitializeHighMemEnd (asan_rtl.cpp).
// Just do some additional sanity checks here.
void InitializeShadowMemory() {
- if (Verbosity()) PrintAddressSpaceLayout();
+ if (Verbosity())
+ PrintAddressSpaceLayout();
// Make sure SHADOW_OFFSET doesn't use __asan_shadow_memory_dynamic_address.
__asan_shadow_memory_dynamic_address = kDefaultShadowSentinel;
UNIMPLEMENTED();
}
-bool PlatformUnpoisonStacks() { return false; }
+bool PlatformUnpoisonStacks() {
+ // The current sp might not point to the default stack. This
+ // could be because we are in a crash stack from fuzzing for example.
+ // Unpoison the default stack and the current stack page.
+ AsanThread *curr_thread = GetCurrentThread();
+ CHECK(curr_thread != nullptr);
+ uptr top = curr_thread->stack_top();
+ uptr bottom = curr_thread->stack_bottom();
+ // The default stack grows from top to bottom. (bottom < top).
+
+ uptr local_stack = reinterpret_cast<uptr>(__builtin_frame_address(0));
+ if (local_stack >= bottom && local_stack <= top) {
+ // The current stack is the default stack.
+ // We only need to unpoison from where we are using until the end.
+ bottom = RoundDownTo(local_stack, GetPageSize());
+ UnpoisonStack(bottom, top, "default");
+ } else {
+ // The current stack is not the default stack.
+ // Unpoison the entire default stack and the current stack page.
+ UnpoisonStack(bottom, top, "default");
+ bottom = RoundDownTo(local_stack, GetPageSize());
+ top = bottom + GetPageSize();
+ UnpoisonStack(bottom, top, "unknown");
+ return true;
+ }
+
+ return false;
+}
// We can use a plain thread_local variable for TSD.
static thread_local void *per_thread;
// Shared setup between thread creation and startup for the initial thread.
static AsanThread *CreateAsanThread(StackTrace *stack, u32 parent_tid,
- uptr user_id, bool detached,
- const char *name) {
+ bool detached, const char *name) {
// In lieu of AsanThread::Create.
AsanThread *thread = (AsanThread *)MmapOrDie(AsanThreadMmapSize(), __func__);
AsanThreadContext::CreateThreadContextArgs args = {thread, stack};
- u32 tid =
- asanThreadRegistry().CreateThread(user_id, detached, parent_tid, &args);
+ u32 tid = asanThreadRegistry().CreateThread(0, detached, parent_tid, &args);
asanThreadRegistry().SetThreadName(tid, name);
return thread;
CHECK_NE(__sanitizer::MainThreadStackBase, 0);
CHECK_GT(__sanitizer::MainThreadStackSize, 0);
AsanThread *t = CreateAsanThread(
- nullptr, 0, reinterpret_cast<uptr>(self), true,
+ nullptr, 0, true,
_zx_object_get_property(thrd_get_zx_handle(self), ZX_PROP_NAME, name,
sizeof(name)) == ZX_OK
? name
uptr stack_size) {
EnsureMainThreadIDIsCorrect();
// Strict init-order checking is thread-hostile.
- if (flags()->strict_init_order) StopInitOrderChecking();
+ if (flags()->strict_init_order)
+ StopInitOrderChecking();
GET_STACK_TRACE_THREAD;
u32 parent_tid = GetCurrentTidOrInvalid();
- AsanThread *thread =
- CreateAsanThread(&stack, parent_tid, user_id, detached, name);
+ AsanThread *thread = CreateAsanThread(&stack, parent_tid, detached, name);
// On other systems, AsanThread::Init() is called from the new
// thread itself. But on Fuchsia we already know the stack address
__sanitizer_fill_shadow(p, size, 0, 0);
}
+// On Fuchsia, leak detection is done by a special hook after atexit hooks.
+// So this doesn't install any atexit hook like on other platforms.
+void InstallAtExitCheckLeaks() {}
+
} // namespace __asan
+namespace __lsan {
+
+bool UseExitcodeOnLeak() { return __asan::flags()->halt_on_error; }
+
+} // namespace __lsan
+
// These are declared (in extern "C") by <zircon/sanitizer.h>.
// The system runtime will call our definitions directly.
ListOfGlobals *next;
};
-static BlockingMutex mu_for_globals(LINKER_INITIALIZED);
+static Mutex mu_for_globals;
static LowLevelAllocator allocator_for_globals;
static ListOfGlobals *list_of_all_globals;
}
ALWAYS_INLINE void PoisonRedZones(const Global &g) {
- uptr aligned_size = RoundUpTo(g.size, SHADOW_GRANULARITY);
+ uptr aligned_size = RoundUpTo(g.size, ASAN_SHADOW_GRANULARITY);
FastPoisonShadow(g.beg + aligned_size, g.size_with_redzone - aligned_size,
kAsanGlobalRedzoneMagic);
if (g.size != aligned_size) {
FastPoisonShadowPartialRightRedzone(
- g.beg + RoundDownTo(g.size, SHADOW_GRANULARITY),
- g.size % SHADOW_GRANULARITY,
- SHADOW_GRANULARITY,
+ g.beg + RoundDownTo(g.size, ASAN_SHADOW_GRANULARITY),
+ g.size % ASAN_SHADOW_GRANULARITY, ASAN_SHADOW_GRANULARITY,
kAsanGlobalRedzoneMagic);
}
}
Report(
"%s Global[%p]: beg=%p size=%zu/%zu name=%s module=%s dyn_init=%zu "
"odr_indicator=%p\n",
- prefix, &g, (void *)g.beg, g.size, g.size_with_redzone, g.name,
+ prefix, (void *)&g, (void *)g.beg, g.size, g.size_with_redzone, g.name,
g.module_name, g.has_dynamic_init, (void *)g.odr_indicator);
- if (g.location) {
- Report(" location (%p): name=%s[%p], %d %d\n", g.location,
- g.location->filename, g.location->filename, g.location->line_no,
- g.location->column_no);
+
+ DataInfo info;
+ Symbolizer::GetOrInit()->SymbolizeData(g.beg, &info);
+ if (info.line != 0) {
+ Report(" location: name=%s, %d\n", info.file, static_cast<int>(info.line));
}
}
int GetGlobalsForAddress(uptr addr, Global *globals, u32 *reg_sites,
int max_globals) {
if (!flags()->report_globals) return 0;
- BlockingMutexLock lock(&mu_for_globals);
+ Lock lock(&mu_for_globals);
int res = 0;
for (ListOfGlobals *l = list_of_all_globals; l; l = l->next) {
const Global &g = *l->g;
}
void StopInitOrderChecking() {
- BlockingMutexLock lock(&mu_for_globals);
+ Lock lock(&mu_for_globals);
if (!flags()->check_initialization_order || !dynamic_init_globals)
return;
flags()->check_initialization_order = false;
(char *)g.beg);
}
-static const char *GlobalFilename(const __asan_global &g) {
- const char *res = g.module_name;
- // Prefer the filename from source location, if is available.
- if (g.location) res = g.location->filename;
- CHECK(res);
- return res;
-}
-
void PrintGlobalLocation(InternalScopedString *str, const __asan_global &g) {
- str->append("%s", GlobalFilename(g));
- if (!g.location) return;
- if (g.location->line_no) str->append(":%d", g.location->line_no);
- if (g.location->column_no) str->append(":%d", g.location->column_no);
+ DataInfo info;
+ Symbolizer::GetOrInit()->SymbolizeData(g.beg, &info);
+
+ if (info.line != 0) {
+ str->append("%s:%d", info.file, static_cast<int>(info.line));
+ } else {
+ str->append("%s", g.module_name);
+ }
}
} // namespace __asan
if (!flags()->report_globals) return;
GET_STACK_TRACE_MALLOC;
u32 stack_id = StackDepotPut(stack);
- BlockingMutexLock lock(&mu_for_globals);
+ Lock lock(&mu_for_globals);
if (!global_registration_site_vector) {
global_registration_site_vector =
new (allocator_for_globals) GlobalRegistrationSiteVector;
global_registration_site_vector->push_back(site);
if (flags()->report_globals >= 2) {
PRINT_CURRENT_STACK();
- Printf("=== ID %d; %p %p\n", stack_id, &globals[0], &globals[n - 1]);
+ Printf("=== ID %d; %p %p\n", stack_id, (void *)&globals[0],
+ (void *)&globals[n - 1]);
}
for (uptr i = 0; i < n; i++) {
if (SANITIZER_WINDOWS && globals[i].beg == 0) {
// We must do this when a shared objects gets dlclosed.
void __asan_unregister_globals(__asan_global *globals, uptr n) {
if (!flags()->report_globals) return;
- BlockingMutexLock lock(&mu_for_globals);
+ Lock lock(&mu_for_globals);
for (uptr i = 0; i < n; i++) {
if (SANITIZER_WINDOWS && globals[i].beg == 0) {
// Skip globals that look like padding from the MSVC incremental linker.
bool strict_init_order = flags()->strict_init_order;
CHECK(module_name);
CHECK(asan_inited);
- BlockingMutexLock lock(&mu_for_globals);
+ Lock lock(&mu_for_globals);
if (flags()->report_globals >= 3)
Printf("DynInitPoison module: %s\n", module_name);
for (uptr i = 0, n = dynamic_init_globals->size(); i < n; ++i) {
!dynamic_init_globals)
return;
CHECK(asan_inited);
- BlockingMutexLock lock(&mu_for_globals);
+ Lock lock(&mu_for_globals);
// FIXME: Optionally report that we're unpoisoning globals from a module.
for (uptr i = 0, n = dynamic_init_globals->size(); i < n; ++i) {
DynInitGlobal &dyn_g = (*dynamic_init_globals)[i];
ASAN_READ_RANGE((ctx), (s), \
common_flags()->strict_string_checks ? (len) + 1 : (n))
-#define ASAN_READ_STRING(ctx, s, n) \
- ASAN_READ_STRING_OF_LEN((ctx), (s), REAL(strlen)(s), (n))
+# define ASAN_READ_STRING(ctx, s, n) \
+ ASAN_READ_STRING_OF_LEN((ctx), (s), internal_strlen(s), (n))
static inline uptr MaybeRealStrnlen(const char *s, uptr maxlen) {
#if SANITIZER_INTERCEPT_STRNLEN
do { \
if (asan_init_is_running) \
return REAL(func)(__VA_ARGS__); \
- if (SANITIZER_MAC && UNLIKELY(!asan_inited)) \
+ if (SANITIZER_APPLE && UNLIKELY(!asan_inited)) \
return REAL(func)(__VA_ARGS__); \
ENSURE_ASAN_INITED(); \
} while (false)
#define COMMON_INTERCEPTOR_BLOCK_REAL(name) REAL(name)
// Strict init-order checking is dlopen-hostile:
// https://github.com/google/sanitizers/issues/178
-#define COMMON_INTERCEPTOR_ON_DLOPEN(filename, flag) \
- do { \
- if (flags()->strict_init_order) \
- StopInitOrderChecking(); \
- CheckNoDeepBind(filename, flag); \
- } while (false)
-#define COMMON_INTERCEPTOR_ON_EXIT(ctx) OnExit()
-#define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, handle)
-#define COMMON_INTERCEPTOR_LIBRARY_UNLOADED()
-#define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED (!asan_inited)
-#define COMMON_INTERCEPTOR_GET_TLS_RANGE(begin, end) \
- if (AsanThread *t = GetCurrentThread()) { \
- *begin = t->tls_begin(); \
- *end = t->tls_end(); \
- } else { \
- *begin = *end = 0; \
- }
+# define COMMON_INTERCEPTOR_DLOPEN(filename, flag) \
+ ({ \
+ if (flags()->strict_init_order) \
+ StopInitOrderChecking(); \
+ CheckNoDeepBind(filename, flag); \
+ REAL(dlopen)(filename, flag); \
+ })
+# define COMMON_INTERCEPTOR_ON_EXIT(ctx) OnExit()
+# define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, handle)
+# define COMMON_INTERCEPTOR_LIBRARY_UNLOADED()
+# define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED (!asan_inited)
+# define COMMON_INTERCEPTOR_GET_TLS_RANGE(begin, end) \
+ if (AsanThread *t = GetCurrentThread()) { \
+ *begin = t->tls_begin(); \
+ *end = t->tls_end(); \
+ } else { \
+ *begin = *end = 0; \
+ }
#define COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size) \
do { \
#if ASAN_INTERCEPT_SWAPCONTEXT
static void ClearShadowMemoryForContextStack(uptr stack, uptr ssize) {
+ // Only clear if we know the stack. This should be true only for contexts
+ // created with makecontext().
+ if (!ssize)
+ return;
// Align to page size.
uptr PageSize = GetPageSizeCached();
- uptr bottom = stack & ~(PageSize - 1);
+ uptr bottom = RoundDownTo(stack, PageSize);
+ if (!AddrIsInMem(bottom))
+ return;
ssize += stack - bottom;
ssize = RoundUpTo(ssize, PageSize);
- static const uptr kMaxSaneContextStackSize = 1 << 22; // 4 Mb
- if (AddrIsInMem(bottom) && ssize && ssize <= kMaxSaneContextStackSize) {
- PoisonShadow(bottom, ssize, 0);
- }
+ PoisonShadow(bottom, ssize, 0);
+}
+
+INTERCEPTOR(void, makecontext, struct ucontext_t *ucp, void (*func)(), int argc,
+ ...) {
+ va_list ap;
+ uptr args[64];
+ // We don't know a better way to forward ... into REAL function. We can
+ // increase args size if neccecary.
+ CHECK_LE(argc, ARRAY_SIZE(args));
+ internal_memset(args, 0, sizeof(args));
+ va_start(ap, argc);
+ for (int i = 0; i < argc; ++i) args[i] = va_arg(ap, uptr);
+ va_end(ap);
+
+# define ENUMERATE_ARRAY_4(start) \
+ args[start], args[start + 1], args[start + 2], args[start + 3]
+# define ENUMERATE_ARRAY_16(start) \
+ ENUMERATE_ARRAY_4(start), ENUMERATE_ARRAY_4(start + 4), \
+ ENUMERATE_ARRAY_4(start + 8), ENUMERATE_ARRAY_4(start + 12)
+# define ENUMERATE_ARRAY_64() \
+ ENUMERATE_ARRAY_16(0), ENUMERATE_ARRAY_16(16), ENUMERATE_ARRAY_16(32), \
+ ENUMERATE_ARRAY_16(48)
+
+ REAL(makecontext)
+ ((struct ucontext_t *)ucp, func, argc, ENUMERATE_ARRAY_64());
+
+# undef ENUMERATE_ARRAY_4
+# undef ENUMERATE_ARRAY_16
+# undef ENUMERATE_ARRAY_64
+
+ // Sign the stack so we can identify it for unpoisoning.
+ SignContextStack(ucp);
}
INTERCEPTOR(int, swapcontext, struct ucontext_t *oucp,
uptr stack, ssize;
ReadContextStack(ucp, &stack, &ssize);
ClearShadowMemoryForContextStack(stack, ssize);
-#if __has_attribute(__indirect_return__) && \
- (defined(__x86_64__) || defined(__i386__))
+
+# if __has_attribute(__indirect_return__) && \
+ (defined(__x86_64__) || defined(__i386__))
int (*real_swapcontext)(struct ucontext_t *, struct ucontext_t *)
- __attribute__((__indirect_return__))
- = REAL(swapcontext);
+ __attribute__((__indirect_return__)) = REAL(swapcontext);
int res = real_swapcontext(oucp, ucp);
-#else
+# else
int res = REAL(swapcontext)(oucp, ucp);
-#endif
+# endif
// swapcontext technically does not return, but program may swap context to
// "oucp" later, that would look as if swapcontext() returned 0.
// We need to clear shadow for ucp once again, as it may be in arbitrary
INTERCEPTOR(char*, index, const char *string, int c)
ALIAS(WRAPPER_NAME(strchr));
# else
-# if SANITIZER_MAC
+# if SANITIZER_APPLE
DECLARE_REAL(char*, index, const char *string, int c)
OVERRIDE_FUNCTION(index, strchr);
# else
ASAN_INTERCEPTOR_ENTER(ctx, strcat);
ENSURE_ASAN_INITED();
if (flags()->replace_str) {
- uptr from_length = REAL(strlen)(from);
+ uptr from_length = internal_strlen(from);
ASAN_READ_RANGE(ctx, from, from_length + 1);
- uptr to_length = REAL(strlen)(to);
+ uptr to_length = internal_strlen(to);
ASAN_READ_STRING_OF_LEN(ctx, to, to_length, to_length);
ASAN_WRITE_RANGE(ctx, to + to_length, from_length + 1);
// If the copying actually happens, the |from| string should not overlap
uptr from_length = MaybeRealStrnlen(from, size);
uptr copy_length = Min(size, from_length + 1);
ASAN_READ_RANGE(ctx, from, copy_length);
- uptr to_length = REAL(strlen)(to);
+ uptr to_length = internal_strlen(to);
ASAN_READ_STRING_OF_LEN(ctx, to, to_length, to_length);
ASAN_WRITE_RANGE(ctx, to + to_length, from_length + 1);
if (from_length > 0) {
INTERCEPTOR(char *, strcpy, char *to, const char *from) {
void *ctx;
ASAN_INTERCEPTOR_ENTER(ctx, strcpy);
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
if (UNLIKELY(!asan_inited))
return REAL(strcpy)(to, from);
#endif
}
ENSURE_ASAN_INITED();
if (flags()->replace_str) {
- uptr from_size = REAL(strlen)(from) + 1;
+ uptr from_size = internal_strlen(from) + 1;
CHECK_RANGES_OVERLAP("strcpy", to, from_size, from, from_size);
ASAN_READ_RANGE(ctx, from, from_size);
ASAN_WRITE_RANGE(ctx, to, from_size);
ASAN_INTERCEPTOR_ENTER(ctx, strdup);
if (UNLIKELY(!asan_inited)) return internal_strdup(s);
ENSURE_ASAN_INITED();
- uptr length = REAL(strlen)(s);
+ uptr length = internal_strlen(s);
if (flags()->replace_str) {
ASAN_READ_RANGE(ctx, s, length + 1);
}
ASAN_INTERCEPTOR_ENTER(ctx, strdup);
if (UNLIKELY(!asan_inited)) return internal_strdup(s);
ENSURE_ASAN_INITED();
- uptr length = REAL(strlen)(s);
+ uptr length = internal_strlen(s);
if (flags()->replace_str) {
ASAN_READ_RANGE(ctx, s, length + 1);
}
INTERCEPTOR(int, atoi, const char *nptr) {
void *ctx;
ASAN_INTERCEPTOR_ENTER(ctx, atoi);
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
if (UNLIKELY(!asan_inited)) return REAL(atoi)(nptr);
#endif
ENSURE_ASAN_INITED();
INTERCEPTOR(long, atol, const char *nptr) {
void *ctx;
ASAN_INTERCEPTOR_ENTER(ctx, atol);
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
if (UNLIKELY(!asan_inited)) return REAL(atol)(nptr);
#endif
ENSURE_ASAN_INITED();
#if ASAN_INTERCEPT___CXA_ATEXIT
INTERCEPTOR(int, __cxa_atexit, void (*func)(void *), void *arg,
void *dso_handle) {
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
if (UNLIKELY(!asan_inited)) return REAL(__cxa_atexit)(func, arg, dso_handle);
#endif
ENSURE_ASAN_INITED();
#if CAN_SANITIZE_LEAKS
__lsan::ScopedInterceptorDisabler disabler;
#endif
- // Avoid calling real atexit as it is unrechable on at least on Linux.
+ // Avoid calling real atexit as it is unreachable on at least on Linux.
int res = REAL(__cxa_atexit)((void (*)(void *a))func, nullptr, nullptr);
REAL(__cxa_atexit)(AtCxaAtexit, nullptr, nullptr);
return res;
// Intecept jump-related functions.
ASAN_INTERCEPT_FUNC(longjmp);
-#if ASAN_INTERCEPT_SWAPCONTEXT
+# if ASAN_INTERCEPT_SWAPCONTEXT
ASAN_INTERCEPT_FUNC(swapcontext);
-#endif
-#if ASAN_INTERCEPT__LONGJMP
+ ASAN_INTERCEPT_FUNC(makecontext);
+# endif
+# if ASAN_INTERCEPT__LONGJMP
ASAN_INTERCEPT_FUNC(_longjmp);
#endif
#if ASAN_INTERCEPT___LONGJMP_CHK
#if SANITIZER_LINUX && \
(defined(__arm__) || defined(__aarch64__) || defined(__i386__) || \
- defined(__x86_64__) || SANITIZER_RISCV64)
+ defined(__x86_64__) || SANITIZER_RISCV64 || SANITIZER_LOONGARCH64)
# define ASAN_INTERCEPT_VFORK 1
#else
# define ASAN_INTERCEPT_VFORK 0
DECLARE_REAL(uptr, strnlen, const char *s, uptr maxlen)
DECLARE_REAL(char*, strstr, const char *s1, const char *s2)
-#if !SANITIZER_MAC
-#define ASAN_INTERCEPT_FUNC(name) \
- do { \
- if (!INTERCEPT_FUNCTION(name)) \
- VReport(1, "AddressSanitizer: failed to intercept '%s'\n", #name); \
- } while (0)
-#define ASAN_INTERCEPT_FUNC_VER(name, ver) \
- do { \
- if (!INTERCEPT_FUNCTION_VER(name, ver)) \
- VReport(1, "AddressSanitizer: failed to intercept '%s@@%s'\n", #name, \
- #ver); \
- } while (0)
-#define ASAN_INTERCEPT_FUNC_VER_UNVERSIONED_FALLBACK(name, ver) \
- do { \
- if (!INTERCEPT_FUNCTION_VER(name, ver) && !INTERCEPT_FUNCTION(name)) \
- VReport(1, "AddressSanitizer: failed to intercept '%s@@%s' or '%s'\n", \
- #name, #ver, #name); \
- } while (0)
-
-#else
+# if !SANITIZER_APPLE
+# define ASAN_INTERCEPT_FUNC(name) \
+ do { \
+ if (!INTERCEPT_FUNCTION(name)) \
+ VReport(1, "AddressSanitizer: failed to intercept '%s'\n", #name); \
+ } while (0)
+# define ASAN_INTERCEPT_FUNC_VER(name, ver) \
+ do { \
+ if (!INTERCEPT_FUNCTION_VER(name, ver)) \
+ VReport(1, "AddressSanitizer: failed to intercept '%s@@%s'\n", \
+ #name, ver); \
+ } while (0)
+# define ASAN_INTERCEPT_FUNC_VER_UNVERSIONED_FALLBACK(name, ver) \
+ do { \
+ if (!INTERCEPT_FUNCTION_VER(name, ver) && !INTERCEPT_FUNCTION(name)) \
+ VReport(1, \
+ "AddressSanitizer: failed to intercept '%s@@%s' or '%s'\n", \
+ #name, ver, #name); \
+ } while (0)
+
+# else
// OS X interceptors don't need to be initialized with INTERCEPT_FUNCTION.
-#define ASAN_INTERCEPT_FUNC(name)
-#endif // SANITIZER_MAC
+# define ASAN_INTERCEPT_FUNC(name)
+# endif // SANITIZER_APPLE
#endif // !SANITIZER_FUCHSIA
#include "asan_mapping.h"
#include "interception/interception.h"
-DECLARE_REAL(void*, memcpy, void *to, const void *from, uptr size)
-DECLARE_REAL(void*, memset, void *block, int c, uptr size)
+DECLARE_REAL(void *, memcpy, void *to, const void *from, uptr size)
+DECLARE_REAL(void *, memset, void *block, int c, uptr size)
namespace __asan {
// Return true if we can quickly decide that the region is unpoisoned.
// We assume that a redzone is at least 16 bytes.
static inline bool QuickCheckForUnpoisonedRegion(uptr beg, uptr size) {
- if (size == 0) return true;
- if (size <= 32)
- return !AddressIsPoisoned(beg) &&
- !AddressIsPoisoned(beg + size - 1) &&
- !AddressIsPoisoned(beg + size / 2);
- if (size <= 64)
- return !AddressIsPoisoned(beg) &&
- !AddressIsPoisoned(beg + size / 4) &&
- !AddressIsPoisoned(beg + size - 1) &&
- !AddressIsPoisoned(beg + 3 * size / 4) &&
- !AddressIsPoisoned(beg + size / 2);
- return false;
+ if (UNLIKELY(size == 0 || size > sizeof(uptr) * ASAN_SHADOW_GRANULARITY))
+ return !size;
+
+ uptr last = beg + size - 1;
+ uptr shadow_first = MEM_TO_SHADOW(beg);
+ uptr shadow_last = MEM_TO_SHADOW(last);
+ uptr uptr_first = RoundDownTo(shadow_first, sizeof(uptr));
+ uptr uptr_last = RoundDownTo(shadow_last, sizeof(uptr));
+ if (LIKELY(((*reinterpret_cast<const uptr *>(uptr_first) |
+ *reinterpret_cast<const uptr *>(uptr_last)) == 0)))
+ return true;
+ u8 shadow = AddressIsPoisoned(last);
+ for (; shadow_first < shadow_last; ++shadow_first)
+ shadow |= *((u8 *)shadow_first);
+ return !shadow;
}
struct AsanInterceptorContext {
// that no extra frames are created, and stack trace contains
// relevant information only.
// We check all shadow bytes.
-#define ACCESS_MEMORY_RANGE(ctx, offset, size, isWrite) do { \
- uptr __offset = (uptr)(offset); \
- uptr __size = (uptr)(size); \
- uptr __bad = 0; \
- if (__offset > __offset + __size) { \
- GET_STACK_TRACE_FATAL_HERE; \
- ReportStringFunctionSizeOverflow(__offset, __size, &stack); \
- } \
- if (!QuickCheckForUnpoisonedRegion(__offset, __size) && \
- (__bad = __asan_region_is_poisoned(__offset, __size))) { \
- AsanInterceptorContext *_ctx = (AsanInterceptorContext *)ctx; \
- bool suppressed = false; \
- if (_ctx) { \
- suppressed = IsInterceptorSuppressed(_ctx->interceptor_name); \
- if (!suppressed && HaveStackTraceBasedSuppressions()) { \
- GET_STACK_TRACE_FATAL_HERE; \
- suppressed = IsStackTraceSuppressed(&stack); \
- } \
- } \
- if (!suppressed) { \
- GET_CURRENT_PC_BP_SP; \
- ReportGenericError(pc, bp, sp, __bad, isWrite, __size, 0, false);\
- } \
- } \
+#define ACCESS_MEMORY_RANGE(ctx, offset, size, isWrite) \
+ do { \
+ uptr __offset = (uptr)(offset); \
+ uptr __size = (uptr)(size); \
+ uptr __bad = 0; \
+ if (UNLIKELY(__offset > __offset + __size)) { \
+ GET_STACK_TRACE_FATAL_HERE; \
+ ReportStringFunctionSizeOverflow(__offset, __size, &stack); \
+ } \
+ if (UNLIKELY(!QuickCheckForUnpoisonedRegion(__offset, __size)) && \
+ (__bad = __asan_region_is_poisoned(__offset, __size))) { \
+ AsanInterceptorContext *_ctx = (AsanInterceptorContext *)ctx; \
+ bool suppressed = false; \
+ if (_ctx) { \
+ suppressed = IsInterceptorSuppressed(_ctx->interceptor_name); \
+ if (!suppressed && HaveStackTraceBasedSuppressions()) { \
+ GET_STACK_TRACE_FATAL_HERE; \
+ suppressed = IsStackTraceSuppressed(&stack); \
+ } \
+ } \
+ if (!suppressed) { \
+ GET_CURRENT_PC_BP_SP; \
+ ReportGenericError(pc, bp, sp, __bad, isWrite, __size, 0, false); \
+ } \
+ } \
} while (0)
// memcpy is called during __asan_init() from the internals of printf(...).
// We do not treat memcpy with to==from as a bug.
// See http://llvm.org/bugs/show_bug.cgi?id=11763.
-#define ASAN_MEMCPY_IMPL(ctx, to, from, size) \
- do { \
- if (UNLIKELY(!asan_inited)) return internal_memcpy(to, from, size); \
- if (asan_init_is_running) { \
- return REAL(memcpy)(to, from, size); \
- } \
- ENSURE_ASAN_INITED(); \
- if (flags()->replace_intrin) { \
- if (to != from) { \
- CHECK_RANGES_OVERLAP("memcpy", to, size, from, size); \
- } \
- ASAN_READ_RANGE(ctx, from, size); \
- ASAN_WRITE_RANGE(ctx, to, size); \
- } \
- return REAL(memcpy)(to, from, size); \
+#define ASAN_MEMCPY_IMPL(ctx, to, from, size) \
+ do { \
+ if (LIKELY(replace_intrin_cached)) { \
+ if (LIKELY(to != from)) { \
+ CHECK_RANGES_OVERLAP("memcpy", to, size, from, size); \
+ } \
+ ASAN_READ_RANGE(ctx, from, size); \
+ ASAN_WRITE_RANGE(ctx, to, size); \
+ } else if (UNLIKELY(!asan_inited)) { \
+ return internal_memcpy(to, from, size); \
+ } \
+ return REAL(memcpy)(to, from, size); \
} while (0)
// memset is called inside Printf.
-#define ASAN_MEMSET_IMPL(ctx, block, c, size) \
- do { \
- if (UNLIKELY(!asan_inited)) return internal_memset(block, c, size); \
- if (asan_init_is_running) { \
- return REAL(memset)(block, c, size); \
- } \
- ENSURE_ASAN_INITED(); \
- if (flags()->replace_intrin) { \
- ASAN_WRITE_RANGE(ctx, block, size); \
- } \
- return REAL(memset)(block, c, size); \
+#define ASAN_MEMSET_IMPL(ctx, block, c, size) \
+ do { \
+ if (LIKELY(replace_intrin_cached)) { \
+ ASAN_WRITE_RANGE(ctx, block, size); \
+ } else if (UNLIKELY(!asan_inited)) { \
+ return internal_memset(block, c, size); \
+ } \
+ return REAL(memset)(block, c, size); \
} while (0)
-#define ASAN_MEMMOVE_IMPL(ctx, to, from, size) \
- do { \
- if (UNLIKELY(!asan_inited)) return internal_memmove(to, from, size); \
- ENSURE_ASAN_INITED(); \
- if (flags()->replace_intrin) { \
- ASAN_READ_RANGE(ctx, from, size); \
- ASAN_WRITE_RANGE(ctx, to, size); \
- } \
- return internal_memmove(to, from, size); \
+#define ASAN_MEMMOVE_IMPL(ctx, to, from, size) \
+ do { \
+ if (LIKELY(replace_intrin_cached)) { \
+ ASAN_READ_RANGE(ctx, from, size); \
+ ASAN_WRITE_RANGE(ctx, to, size); \
+ } \
+ return internal_memmove(to, from, size); \
} while (0)
#define ASAN_READ_RANGE(ctx, offset, size) \
do { \
const char *offset1 = (const char *)_offset1; \
const char *offset2 = (const char *)_offset2; \
- if (RangesOverlap(offset1, length1, offset2, length2)) { \
+ if (UNLIKELY(RangesOverlap(offset1, length1, offset2, length2))) { \
GET_STACK_TRACE_FATAL_HERE; \
bool suppressed = IsInterceptorSuppressed(name); \
if (!suppressed && HaveStackTraceBasedSuppressions()) { \
#include "sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S"
#include "sanitizer_common/sanitizer_common_interceptors_vfork_arm.inc.S"
#include "sanitizer_common/sanitizer_common_interceptors_vfork_i386.inc.S"
+#include "sanitizer_common/sanitizer_common_interceptors_vfork_loongarch64.inc.S"
#include "sanitizer_common/sanitizer_common_interceptors_vfork_riscv64.inc.S"
#include "sanitizer_common/sanitizer_common_interceptors_vfork_x86_64.inc.S"
#endif
//===----------------------------------------------------------------------===//
// Asan interface list.
//===----------------------------------------------------------------------===//
+
INTERFACE_FUNCTION(__asan_addr_is_in_fake_stack)
INTERFACE_FUNCTION(__asan_address_is_poisoned)
INTERFACE_FUNCTION(__asan_after_dynamic_init)
INTERFACE_FUNCTION(__asan_set_death_callback)
INTERFACE_FUNCTION(__asan_set_error_report_callback)
INTERFACE_FUNCTION(__asan_set_shadow_00)
+INTERFACE_FUNCTION(__asan_set_shadow_01)
+INTERFACE_FUNCTION(__asan_set_shadow_02)
+INTERFACE_FUNCTION(__asan_set_shadow_03)
+INTERFACE_FUNCTION(__asan_set_shadow_04)
+INTERFACE_FUNCTION(__asan_set_shadow_05)
+INTERFACE_FUNCTION(__asan_set_shadow_06)
+INTERFACE_FUNCTION(__asan_set_shadow_07)
INTERFACE_FUNCTION(__asan_set_shadow_f1)
INTERFACE_FUNCTION(__asan_set_shadow_f2)
INTERFACE_FUNCTION(__asan_set_shadow_f3)
const char *module_name; // Module name as a C string. This pointer is a
// unique identifier of a module.
uptr has_dynamic_init; // Non-zero if the global has dynamic initializer.
- __asan_global_source_location *location; // Source location of a global,
- // or NULL if it is unknown.
+ uptr windows_padding; // TODO: Figure out how to remove this padding
+ // that's simply here to make the MSVC incremental
+ // linker happy...
uptr odr_indicator; // The address of the ODR indicator symbol.
};
SANITIZER_INTERFACE_ATTRIBUTE
void __asan_set_shadow_00(uptr addr, uptr size);
SANITIZER_INTERFACE_ATTRIBUTE
+ void __asan_set_shadow_01(uptr addr, uptr size);
+ SANITIZER_INTERFACE_ATTRIBUTE
+ void __asan_set_shadow_02(uptr addr, uptr size);
+ SANITIZER_INTERFACE_ATTRIBUTE
+ void __asan_set_shadow_03(uptr addr, uptr size);
+ SANITIZER_INTERFACE_ATTRIBUTE
+ void __asan_set_shadow_04(uptr addr, uptr size);
+ SANITIZER_INTERFACE_ATTRIBUTE
+ void __asan_set_shadow_05(uptr addr, uptr size);
+ SANITIZER_INTERFACE_ATTRIBUTE
+ void __asan_set_shadow_06(uptr addr, uptr size);
+ SANITIZER_INTERFACE_ATTRIBUTE
+ void __asan_set_shadow_07(uptr addr, uptr size);
+ SANITIZER_INTERFACE_ATTRIBUTE
void __asan_set_shadow_f1(uptr addr, uptr size);
SANITIZER_INTERFACE_ATTRIBUTE
void __asan_set_shadow_f2(uptr addr, uptr size);
#include "asan_interface_internal.h"
#include "sanitizer_common/sanitizer_common.h"
#include "sanitizer_common/sanitizer_internal_defs.h"
-#include "sanitizer_common/sanitizer_stacktrace.h"
#include "sanitizer_common/sanitizer_libc.h"
+#include "sanitizer_common/sanitizer_stacktrace.h"
#if __has_feature(address_sanitizer) || defined(__SANITIZE_ADDRESS__)
-# error "The AddressSanitizer run-time should not be"
- " instrumented by AddressSanitizer"
+# error \
+ "The AddressSanitizer run-time should not be instrumented by AddressSanitizer"
#endif
// Build-time configuration options.
// If set, asan will intercept C++ exception api call(s).
#ifndef ASAN_HAS_EXCEPTIONS
-# define ASAN_HAS_EXCEPTIONS 1
+# define ASAN_HAS_EXCEPTIONS 1
#endif
// If set, values like allocator chunk size, as well as defaults for some flags
#endif
#ifndef ASAN_DYNAMIC
-# ifdef PIC
-# define ASAN_DYNAMIC 1
-# else
-# define ASAN_DYNAMIC 0
-# endif
+# ifdef PIC
+# define ASAN_DYNAMIC 1
+# else
+# define ASAN_DYNAMIC 0
+# endif
#endif
// All internal functions in asan reside inside the __asan namespace
void AsanOnDeadlySignal(int, void *siginfo, void *context);
+void SignContextStack(void *context);
void ReadContextStack(void *context, uptr *stack, uptr *ssize);
void StopInitOrderChecking();
// `dlopen()` specific initialization inside this function.
bool HandleDlopenInit();
-// Add convenient macro for interface functions that may be represented as
-// weak hooks.
-#define ASAN_MALLOC_HOOK(ptr, size) \
- do { \
- if (&__sanitizer_malloc_hook) __sanitizer_malloc_hook(ptr, size); \
- RunMallocHooks(ptr, size); \
- } while (false)
-#define ASAN_FREE_HOOK(ptr) \
- do { \
- if (&__sanitizer_free_hook) __sanitizer_free_hook(ptr); \
- RunFreeHooks(ptr); \
- } while (false)
+void InstallAtExitCheckLeaks();
+
#define ASAN_ON_ERROR() \
- if (&__asan_on_error) __asan_on_error()
+ if (&__asan_on_error) \
+ __asan_on_error()
extern int asan_inited;
// Used to avoid infinite recursion in __asan_init().
extern bool asan_init_is_running;
+extern bool replace_intrin_cached;
extern void (*death_callback)(void);
-// These magic values are written to shadow for better error reporting.
+// These magic values are written to shadow for better error
+// reporting.
const int kAsanHeapLeftRedzoneMagic = 0xfa;
const int kAsanHeapFreeMagic = 0xfd;
const int kAsanStackLeftRedzoneMagic = 0xf1;
#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \
SANITIZER_SOLARIS
-#include "asan_interceptors.h"
-#include "asan_internal.h"
-#include "asan_premap_shadow.h"
-#include "asan_thread.h"
-#include "sanitizer_common/sanitizer_flags.h"
-#include "sanitizer_common/sanitizer_freebsd.h"
-#include "sanitizer_common/sanitizer_libc.h"
-#include "sanitizer_common/sanitizer_procmaps.h"
-
-#include <sys/time.h>
-#include <sys/resource.h>
-#include <sys/mman.h>
-#include <sys/syscall.h>
-#include <sys/types.h>
-#include <dlfcn.h>
-#include <fcntl.h>
-#include <limits.h>
-#include <pthread.h>
-#include <stdio.h>
-#include <unistd.h>
-#include <unwind.h>
-
-#if SANITIZER_FREEBSD
-#include <sys/link_elf.h>
-#endif
-
-#if SANITIZER_SOLARIS
-#include <link.h>
-#endif
-
-#if SANITIZER_ANDROID || SANITIZER_FREEBSD || SANITIZER_SOLARIS
-#include <ucontext.h>
-extern "C" void* _DYNAMIC;
-#elif SANITIZER_NETBSD
-#include <link_elf.h>
-#include <ucontext.h>
+# include <dlfcn.h>
+# include <fcntl.h>
+# include <limits.h>
+# include <pthread.h>
+# include <stdio.h>
+# include <sys/mman.h>
+# include <sys/resource.h>
+# include <sys/syscall.h>
+# include <sys/time.h>
+# include <sys/types.h>
+# include <unistd.h>
+# include <unwind.h>
+
+# include "asan_interceptors.h"
+# include "asan_internal.h"
+# include "asan_premap_shadow.h"
+# include "asan_thread.h"
+# include "sanitizer_common/sanitizer_flags.h"
+# include "sanitizer_common/sanitizer_freebsd.h"
+# include "sanitizer_common/sanitizer_hash.h"
+# include "sanitizer_common/sanitizer_libc.h"
+# include "sanitizer_common/sanitizer_procmaps.h"
+
+# if SANITIZER_FREEBSD
+# include <sys/link_elf.h>
+# endif
+
+# if SANITIZER_SOLARIS
+# include <link.h>
+# endif
+
+# if SANITIZER_ANDROID || SANITIZER_FREEBSD || SANITIZER_SOLARIS
+# include <ucontext.h>
+extern "C" void *_DYNAMIC;
+# elif SANITIZER_NETBSD
+# include <link_elf.h>
+# include <ucontext.h>
extern Elf_Dyn _DYNAMIC;
-#else
-#include <sys/ucontext.h>
-#include <link.h>
+# else
+# include <link.h>
+# include <sys/ucontext.h>
extern ElfW(Dyn) _DYNAMIC[];
-#endif
+# endif
// x86-64 FreeBSD 9.2 and older define 'ucontext_t' incorrectly in
// 32-bit mode.
-#if SANITIZER_FREEBSD && (SANITIZER_WORDSIZE == 32) && \
- __FreeBSD_version <= 902001 // v9.2
-#define ucontext_t xucontext_t
-#endif
+# if SANITIZER_FREEBSD && (SANITIZER_WORDSIZE == 32) && \
+ __FreeBSD_version <= 902001 // v9.2
+# define ucontext_t xucontext_t
+# endif
typedef enum {
ASAN_RT_VERSION_UNDEFINED = 0,
// FIXME: perhaps also store abi version here?
extern "C" {
SANITIZER_INTERFACE_ATTRIBUTE
-asan_rt_version_t __asan_rt_version;
+asan_rt_version_t __asan_rt_version;
}
namespace __asan {
void InitializePlatformInterceptors() {}
void InitializePlatformExceptionHandlers() {}
-bool IsSystemHeapAddress (uptr addr) { return false; }
+bool IsSystemHeapAddress(uptr addr) { return false; }
void *AsanDoesNotSupportStaticLinkage() {
// This will fail to link with -static.
return &_DYNAMIC;
}
-#if ASAN_PREMAP_SHADOW
+# if ASAN_PREMAP_SHADOW
uptr FindPremappedShadowStart(uptr shadow_size_bytes) {
uptr granularity = GetMmapGranularity();
uptr shadow_start = reinterpret_cast<uptr>(&__asan_shadow);
UnmapFromTo(shadow_start + shadow_size, shadow_start + premap_shadow_size);
return shadow_start;
}
-#endif
+# endif
uptr FindDynamicShadowStart() {
uptr shadow_size_bytes = MemToShadowSize(kHighMemEnd);
-#if ASAN_PREMAP_SHADOW
+# if ASAN_PREMAP_SHADOW
if (!PremapShadowFailed())
return FindPremappedShadowStart(shadow_size_bytes);
-#endif
+# endif
- return MapDynamicShadow(shadow_size_bytes, SHADOW_SCALE,
+ return MapDynamicShadow(shadow_size_bytes, ASAN_SHADOW_SCALE,
/*min_shadow_base_alignment*/ 0, kHighMemEnd);
}
ReleaseMemoryPagesToOS(MemToShadow(p), MemToShadow(p + size));
}
-#if SANITIZER_ANDROID
+# if SANITIZER_ANDROID
// FIXME: should we do anything for Android?
void AsanCheckDynamicRTPrereqs() {}
void AsanCheckIncompatibleRT() {}
-#else
+# else
static int FindFirstDSOCallback(struct dl_phdr_info *info, size_t size,
void *data) {
- VReport(2, "info->dlpi_name = %s\tinfo->dlpi_addr = %p\n",
- info->dlpi_name, info->dlpi_addr);
+ VReport(2, "info->dlpi_name = %s\tinfo->dlpi_addr = %p\n", info->dlpi_name,
+ (void *)info->dlpi_addr);
- // Continue until the first dynamic library is found
- if (!info->dlpi_name || info->dlpi_name[0] == 0)
- return 0;
-
- // Ignore vDSO
- if (internal_strncmp(info->dlpi_name, "linux-", sizeof("linux-") - 1) == 0)
- return 0;
+ const char **name = (const char **)data;
-#if SANITIZER_FREEBSD || SANITIZER_NETBSD
// Ignore first entry (the main program)
- char **p = (char **)data;
- if (!(*p)) {
- *p = (char *)-1;
+ if (!*name) {
+ *name = "";
return 0;
}
-#endif
-#if SANITIZER_SOLARIS
- // Ignore executable on Solaris
- if (info->dlpi_addr == 0)
+# if SANITIZER_LINUX
+ // Ignore vDSO. glibc versions earlier than 2.15 (and some patched
+ // by distributors) return an empty name for the vDSO entry, so
+ // detect this as well.
+ if (!info->dlpi_name[0] ||
+ internal_strncmp(info->dlpi_name, "linux-", sizeof("linux-") - 1) == 0)
return 0;
-#endif
+# endif
- *(const char **)data = info->dlpi_name;
+ *name = info->dlpi_name;
return 1;
}
static bool IsDynamicRTName(const char *libname) {
return internal_strstr(libname, "libclang_rt.asan") ||
- internal_strstr(libname, "libasan.so");
+ internal_strstr(libname, "libasan.so");
}
static void ReportIncompatibleRT() {
// Ensure that dynamic RT is the first DSO in the list
const char *first_dso_name = nullptr;
dl_iterate_phdr(FindFirstDSOCallback, &first_dso_name);
- if (first_dso_name && !IsDynamicRTName(first_dso_name)) {
- Report("ASan runtime does not come first in initial library list; "
- "you should either link runtime to your application or "
- "manually preload it with LD_PRELOAD.\n");
+ if (first_dso_name && first_dso_name[0] && !IsDynamicRTName(first_dso_name)) {
+ Report(
+ "ASan runtime does not come first in initial library list; "
+ "you should either link runtime to your application or "
+ "manually preload it with LD_PRELOAD.\n");
Die();
}
}
// as early as possible, otherwise ASan interceptors could bind to
// the functions in dynamic ASan runtime instead of the functions in
// system libraries, causing crashes later in ASan initialization.
- MemoryMappingLayout proc_maps(/*cache_enabled*/true);
+ MemoryMappingLayout proc_maps(/*cache_enabled*/ true);
char filename[PATH_MAX];
MemoryMappedSegment segment(filename, sizeof(filename));
while (proc_maps.Next(&segment)) {
if (IsDynamicRTName(segment.filename)) {
- Report("Your application is linked against "
- "incompatible ASan runtimes.\n");
+ Report(
+ "Your application is linked against "
+ "incompatible ASan runtimes.\n");
Die();
}
}
}
}
}
-#endif // SANITIZER_ANDROID
+# endif // SANITIZER_ANDROID
-#if !SANITIZER_ANDROID
-void ReadContextStack(void *context, uptr *stack, uptr *ssize) {
- ucontext_t *ucp = (ucontext_t*)context;
- *stack = (uptr)ucp->uc_stack.ss_sp;
- *ssize = ucp->uc_stack.ss_size;
+# if ASAN_INTERCEPT_SWAPCONTEXT
+constexpr u32 kAsanContextStackFlagsMagic = 0x51260eea;
+
+static int HashContextStack(const ucontext_t &ucp) {
+ MurMur2Hash64Builder hash(kAsanContextStackFlagsMagic);
+ hash.add(reinterpret_cast<uptr>(ucp.uc_stack.ss_sp));
+ hash.add(ucp.uc_stack.ss_size);
+ return static_cast<int>(hash.get());
}
-#else
-void ReadContextStack(void *context, uptr *stack, uptr *ssize) {
- UNIMPLEMENTED();
+
+void SignContextStack(void *context) {
+ ucontext_t *ucp = reinterpret_cast<ucontext_t *>(context);
+ ucp->uc_stack.ss_flags = HashContextStack(*ucp);
}
-#endif
-void *AsanDlSymNext(const char *sym) {
- return dlsym(RTLD_NEXT, sym);
+void ReadContextStack(void *context, uptr *stack, uptr *ssize) {
+ const ucontext_t *ucp = reinterpret_cast<const ucontext_t *>(context);
+ if (HashContextStack(*ucp) == ucp->uc_stack.ss_flags) {
+ *stack = reinterpret_cast<uptr>(ucp->uc_stack.ss_sp);
+ *ssize = ucp->uc_stack.ss_size;
+ return;
+ }
+ *stack = 0;
+ *ssize = 0;
}
+# endif // ASAN_INTERCEPT_SWAPCONTEXT
+
+void *AsanDlSymNext(const char *sym) { return dlsym(RTLD_NEXT, sym); }
bool HandleDlopenInit() {
// Not supported on this platform.
return false;
}
-} // namespace __asan
+} // namespace __asan
#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD ||
// SANITIZER_SOLARIS
//===----------------------------------------------------------------------===//
#include "sanitizer_common/sanitizer_platform.h"
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
#include "asan_interceptors.h"
#include "asan_internal.h"
}
uptr FindDynamicShadowStart() {
- return MapDynamicShadow(MemToShadowSize(kHighMemEnd), SHADOW_SCALE,
+ return MapDynamicShadow(MemToShadowSize(kHighMemEnd), ASAN_SHADOW_SCALE,
/*min_shadow_base_alignment*/ 0, kHighMemEnd);
}
ReleaseMemoryPagesToOS(MemToShadow(p), MemToShadow(p + size));
}
-void ReadContextStack(void *context, uptr *stack, uptr *ssize) {
- UNIMPLEMENTED();
-}
-
// Support for the following functions from libdispatch on Mac OS:
// dispatch_async_f()
// dispatch_async()
}
#endif
-#endif // SANITIZER_MAC
+#endif // SANITIZER_APPLE
# include "asan_interceptors.h"
# include "asan_internal.h"
# include "asan_stack.h"
+# include "lsan/lsan_common.h"
# include "sanitizer_common/sanitizer_allocator_checks.h"
+# include "sanitizer_common/sanitizer_allocator_dlsym.h"
# include "sanitizer_common/sanitizer_errno.h"
# include "sanitizer_common/sanitizer_tls_get_addr.h"
// ---------------------- Replacement functions ---------------- {{{1
using namespace __asan;
-static uptr allocated_for_dlsym;
-static uptr last_dlsym_alloc_size_in_words;
-static const uptr kDlsymAllocPoolSize = 1024;
-static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize];
-
-static inline bool IsInDlsymAllocPool(const void *ptr) {
- uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
- return off < allocated_for_dlsym * sizeof(alloc_memory_for_dlsym[0]);
-}
-
-static void *AllocateFromLocalPool(uptr size_in_bytes) {
- uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize;
- void *mem = (void*)&alloc_memory_for_dlsym[allocated_for_dlsym];
- last_dlsym_alloc_size_in_words = size_in_words;
- allocated_for_dlsym += size_in_words;
- CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize);
- return mem;
-}
-
-static void DeallocateFromLocalPool(const void *ptr) {
- // Hack: since glibc 2.27 dlsym no longer uses stack-allocated memory to store
- // error messages and instead uses malloc followed by free. To avoid pool
- // exhaustion due to long object filenames, handle that special case here.
- uptr prev_offset = allocated_for_dlsym - last_dlsym_alloc_size_in_words;
- void *prev_mem = (void*)&alloc_memory_for_dlsym[prev_offset];
- if (prev_mem == ptr) {
- REAL(memset)(prev_mem, 0, last_dlsym_alloc_size_in_words * kWordSize);
- allocated_for_dlsym = prev_offset;
- last_dlsym_alloc_size_in_words = 0;
+struct DlsymAlloc : public DlSymAllocator<DlsymAlloc> {
+ static bool UseImpl() { return asan_init_is_running; }
+ static void OnAllocate(const void *ptr, uptr size) {
+# if CAN_SANITIZE_LEAKS
+ // Suppress leaks from dlerror(). Previously dlsym hack on global array was
+ // used by leak sanitizer as a root region.
+ __lsan_register_root_region(ptr, size);
+# endif
}
-}
-
-static int PosixMemalignFromLocalPool(void **memptr, uptr alignment,
- uptr size_in_bytes) {
- if (UNLIKELY(!CheckPosixMemalignAlignment(alignment)))
- return errno_EINVAL;
-
- CHECK(alignment >= kWordSize);
-
- uptr addr = (uptr)&alloc_memory_for_dlsym[allocated_for_dlsym];
- uptr aligned_addr = RoundUpTo(addr, alignment);
- uptr aligned_size = RoundUpTo(size_in_bytes, kWordSize);
-
- uptr *end_mem = (uptr*)(aligned_addr + aligned_size);
- uptr allocated = end_mem - alloc_memory_for_dlsym;
- if (allocated >= kDlsymAllocPoolSize)
- return errno_ENOMEM;
-
- allocated_for_dlsym = allocated;
- *memptr = (void*)aligned_addr;
- return 0;
-}
-
-static inline bool MaybeInDlsym() {
- // Fuchsia doesn't use dlsym-based interceptors.
- return !SANITIZER_FUCHSIA && asan_init_is_running;
-}
-
-static inline bool UseLocalPool() { return MaybeInDlsym(); }
-
-static void *ReallocFromLocalPool(void *ptr, uptr size) {
- const uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
- const uptr copy_size = Min(size, kDlsymAllocPoolSize - offset);
- void *new_ptr;
- if (UNLIKELY(UseLocalPool())) {
- new_ptr = AllocateFromLocalPool(size);
- } else {
- ENSURE_ASAN_INITED();
- GET_STACK_TRACE_MALLOC;
- new_ptr = asan_malloc(size, &stack);
+ static void OnFree(const void *ptr, uptr size) {
+# if CAN_SANITIZE_LEAKS
+ __lsan_unregister_root_region(ptr, size);
+# endif
}
- internal_memcpy(new_ptr, ptr, copy_size);
- return new_ptr;
-}
+};
INTERCEPTOR(void, free, void *ptr) {
- if (UNLIKELY(IsInDlsymAllocPool(ptr))) {
- DeallocateFromLocalPool(ptr);
- return;
- }
+ if (DlsymAlloc::PointerIsMine(ptr))
+ return DlsymAlloc::Free(ptr);
GET_STACK_TRACE_FREE;
asan_free(ptr, &stack, FROM_MALLOC);
}
#if SANITIZER_INTERCEPT_CFREE
INTERCEPTOR(void, cfree, void *ptr) {
- if (UNLIKELY(IsInDlsymAllocPool(ptr)))
- return;
+ if (DlsymAlloc::PointerIsMine(ptr))
+ return DlsymAlloc::Free(ptr);
GET_STACK_TRACE_FREE;
asan_free(ptr, &stack, FROM_MALLOC);
}
#endif // SANITIZER_INTERCEPT_CFREE
INTERCEPTOR(void*, malloc, uptr size) {
- if (UNLIKELY(UseLocalPool()))
- // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym.
- return AllocateFromLocalPool(size);
+ if (DlsymAlloc::Use())
+ return DlsymAlloc::Allocate(size);
ENSURE_ASAN_INITED();
GET_STACK_TRACE_MALLOC;
return asan_malloc(size, &stack);
}
INTERCEPTOR(void*, calloc, uptr nmemb, uptr size) {
- if (UNLIKELY(UseLocalPool()))
- // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
- return AllocateFromLocalPool(nmemb * size);
+ if (DlsymAlloc::Use())
+ return DlsymAlloc::Callocate(nmemb, size);
ENSURE_ASAN_INITED();
GET_STACK_TRACE_MALLOC;
return asan_calloc(nmemb, size, &stack);
}
INTERCEPTOR(void*, realloc, void *ptr, uptr size) {
- if (UNLIKELY(IsInDlsymAllocPool(ptr)))
- return ReallocFromLocalPool(ptr, size);
- if (UNLIKELY(UseLocalPool()))
- return AllocateFromLocalPool(size);
+ if (DlsymAlloc::Use() || DlsymAlloc::PointerIsMine(ptr))
+ return DlsymAlloc::Realloc(ptr, size);
ENSURE_ASAN_INITED();
GET_STACK_TRACE_MALLOC;
return asan_realloc(ptr, size, &stack);
#endif // SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO
INTERCEPTOR(int, posix_memalign, void **memptr, uptr alignment, uptr size) {
- if (UNLIKELY(UseLocalPool()))
- return PosixMemalignFromLocalPool(memptr, alignment, size);
GET_STACK_TRACE_MALLOC;
return asan_posix_memalign(memptr, alignment, size, &stack);
}
//===----------------------------------------------------------------------===//
#include "sanitizer_common/sanitizer_platform.h"
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
#include "asan_interceptors.h"
#include "asan_report.h"
#ifndef ASAN_MAPPING_H
#define ASAN_MAPPING_H
-#include "asan_internal.h"
+#include "sanitizer_common/sanitizer_platform.h"
// The full explanation of the memory mapping could be found here:
// https://github.com/google/sanitizers/wiki/AddressSanitizerAlgorithm
// || `[0x0080000000000, 0x008ffffffffff]` || LowShadow ||
// || `[0x0000000000000, 0x007ffffffffff]` || LowMem ||
//
+// Default Linux/LoongArch64 (47-bit VMA) mapping:
+// || `[0x500000000000, 0x7fffffffffff]` || HighMem ||
+// || `[0x4a0000000000, 0x4fffffffffff]` || HighShadow ||
+// || `[0x480000000000, 0x49ffffffffff]` || ShadowGap ||
+// || `[0x400000000000, 0x47ffffffffff]` || LowShadow ||
+// || `[0x000000000000, 0x3fffffffffff]` || LowMem ||
+//
// Shadow mapping on FreeBSD/x86-64 with SHADOW_OFFSET == 0x400000000000:
// || `[0x500000000000, 0x7fffffffffff]` || HighMem ||
// || `[0x4a0000000000, 0x4fffffffffff]` || HighShadow ||
// || `[0x30000000, 0x35ffffff]` || LowShadow ||
// || `[0x00000000, 0x2fffffff]` || LowMem ||
-#if defined(ASAN_SHADOW_SCALE)
-static const u64 kDefaultShadowScale = ASAN_SHADOW_SCALE;
-#else
-static const u64 kDefaultShadowScale = 3;
-#endif
-static const u64 kDefaultShadowSentinel = ~(uptr)0;
-static const u64 kDefaultShadowOffset32 = 1ULL << 29; // 0x20000000
-static const u64 kDefaultShadowOffset64 = 1ULL << 44;
-static const u64 kDefaultShort64bitShadowOffset =
- 0x7FFFFFFF & (~0xFFFULL << kDefaultShadowScale); // < 2G.
-static const u64 kAArch64_ShadowOffset64 = 1ULL << 36;
-static const u64 kRiscv64_ShadowOffset64 = 0xd55550000;
-static const u64 kMIPS32_ShadowOffset32 = 0x0aaa0000;
-static const u64 kMIPS64_ShadowOffset64 = 1ULL << 37;
-static const u64 kPPC64_ShadowOffset64 = 1ULL << 44;
-static const u64 kSystemZ_ShadowOffset64 = 1ULL << 52;
-static const u64 kSPARC64_ShadowOffset64 = 1ULL << 43; // 0x80000000000
-static const u64 kFreeBSD_ShadowOffset32 = 1ULL << 30; // 0x40000000
-static const u64 kFreeBSD_ShadowOffset64 = 1ULL << 46; // 0x400000000000
-static const u64 kNetBSD_ShadowOffset32 = 1ULL << 30; // 0x40000000
-static const u64 kNetBSD_ShadowOffset64 = 1ULL << 46; // 0x400000000000
-static const u64 kWindowsShadowOffset32 = 3ULL << 28; // 0x30000000
-
-#define SHADOW_SCALE kDefaultShadowScale
+#define ASAN_SHADOW_SCALE 3
#if SANITIZER_FUCHSIA
-# define SHADOW_OFFSET (0)
+# define ASAN_SHADOW_OFFSET_CONST (0)
#elif SANITIZER_WORDSIZE == 32
# if SANITIZER_ANDROID
-# define SHADOW_OFFSET __asan_shadow_memory_dynamic_address
+# define ASAN_SHADOW_OFFSET_DYNAMIC
# elif defined(__mips__)
-# define SHADOW_OFFSET kMIPS32_ShadowOffset32
+# define ASAN_SHADOW_OFFSET_CONST 0x0aaa0000
# elif SANITIZER_FREEBSD
-# define SHADOW_OFFSET kFreeBSD_ShadowOffset32
+# define ASAN_SHADOW_OFFSET_CONST 0x40000000
# elif SANITIZER_NETBSD
-# define SHADOW_OFFSET kNetBSD_ShadowOffset32
+# define ASAN_SHADOW_OFFSET_CONST 0x40000000
# elif SANITIZER_WINDOWS
-# define SHADOW_OFFSET kWindowsShadowOffset32
+# define ASAN_SHADOW_OFFSET_CONST 0x30000000
# elif SANITIZER_IOS
-# define SHADOW_OFFSET __asan_shadow_memory_dynamic_address
+# define ASAN_SHADOW_OFFSET_DYNAMIC
# else
-# define SHADOW_OFFSET kDefaultShadowOffset32
+# define ASAN_SHADOW_OFFSET_CONST 0x20000000
# endif
#else
# if SANITIZER_IOS
-# define SHADOW_OFFSET __asan_shadow_memory_dynamic_address
-# elif SANITIZER_MAC && defined(__aarch64__)
-# define SHADOW_OFFSET __asan_shadow_memory_dynamic_address
-#elif SANITIZER_RISCV64
-#define SHADOW_OFFSET kRiscv64_ShadowOffset64
+# define ASAN_SHADOW_OFFSET_DYNAMIC
+# elif SANITIZER_APPLE && defined(__aarch64__)
+# define ASAN_SHADOW_OFFSET_DYNAMIC
+# elif SANITIZER_FREEBSD && defined(__aarch64__)
+# define ASAN_SHADOW_OFFSET_CONST 0x0000800000000000
+# elif SANITIZER_RISCV64
+# define ASAN_SHADOW_OFFSET_CONST 0x0000000d55550000
# elif defined(__aarch64__)
-# define SHADOW_OFFSET kAArch64_ShadowOffset64
+# define ASAN_SHADOW_OFFSET_CONST 0x0000001000000000
# elif defined(__powerpc64__)
-# define SHADOW_OFFSET kPPC64_ShadowOffset64
+# define ASAN_SHADOW_OFFSET_CONST 0x0000100000000000
# elif defined(__s390x__)
-# define SHADOW_OFFSET kSystemZ_ShadowOffset64
+# define ASAN_SHADOW_OFFSET_CONST 0x0010000000000000
# elif SANITIZER_FREEBSD
-# define SHADOW_OFFSET kFreeBSD_ShadowOffset64
+# define ASAN_SHADOW_OFFSET_CONST 0x0000400000000000
# elif SANITIZER_NETBSD
-# define SHADOW_OFFSET kNetBSD_ShadowOffset64
-# elif SANITIZER_MAC
-# define SHADOW_OFFSET kDefaultShadowOffset64
+# define ASAN_SHADOW_OFFSET_CONST 0x0000400000000000
+# elif SANITIZER_APPLE
+# define ASAN_SHADOW_OFFSET_CONST 0x0000100000000000
# elif defined(__mips64)
-# define SHADOW_OFFSET kMIPS64_ShadowOffset64
-#elif defined(__sparc__)
-#define SHADOW_OFFSET kSPARC64_ShadowOffset64
+# define ASAN_SHADOW_OFFSET_CONST 0x0000002000000000
+# elif defined(__sparc__)
+# define ASAN_SHADOW_OFFSET_CONST 0x0000080000000000
+# elif SANITIZER_LOONGARCH64
+# define ASAN_SHADOW_OFFSET_CONST 0x0000400000000000
# elif SANITIZER_WINDOWS64
-# define SHADOW_OFFSET __asan_shadow_memory_dynamic_address
+# define ASAN_SHADOW_OFFSET_DYNAMIC
# else
-# define SHADOW_OFFSET kDefaultShort64bitShadowOffset
+# if ASAN_SHADOW_SCALE != 3
+# error "Value below is based on shadow scale = 3."
+# error "Original formula was: 0x7FFFFFFF & (~0xFFFULL << SHADOW_SCALE)."
+# endif
+# define ASAN_SHADOW_OFFSET_CONST 0x000000007fff8000
# endif
#endif
-#if SANITIZER_ANDROID && defined(__arm__)
-# define ASAN_PREMAP_SHADOW 1
-#else
-# define ASAN_PREMAP_SHADOW 0
-#endif
+#if defined(__cplusplus)
+# include "asan_internal.h"
-#define SHADOW_GRANULARITY (1ULL << SHADOW_SCALE)
+static const u64 kDefaultShadowSentinel = ~(uptr)0;
-#define DO_ASAN_MAPPING_PROFILE 0 // Set to 1 to profile the functions below.
+# if defined(ASAN_SHADOW_OFFSET_CONST)
+static const u64 kConstShadowOffset = ASAN_SHADOW_OFFSET_CONST;
+# define ASAN_SHADOW_OFFSET kConstShadowOffset
+# elif defined(ASAN_SHADOW_OFFSET_DYNAMIC)
+# define ASAN_SHADOW_OFFSET __asan_shadow_memory_dynamic_address
+# else
+# error "ASAN_SHADOW_OFFSET can't be determined."
+# endif
-#if DO_ASAN_MAPPING_PROFILE
-# define PROFILE_ASAN_MAPPING() AsanMappingProfile[__LINE__]++;
-#else
-# define PROFILE_ASAN_MAPPING()
-#endif
+# if SANITIZER_ANDROID && defined(__arm__)
+# define ASAN_PREMAP_SHADOW 1
+# else
+# define ASAN_PREMAP_SHADOW 0
+# endif
+
+# define ASAN_SHADOW_GRANULARITY (1ULL << ASAN_SHADOW_SCALE)
+
+# define DO_ASAN_MAPPING_PROFILE 0 // Set to 1 to profile the functions below.
+
+# if DO_ASAN_MAPPING_PROFILE
+# define PROFILE_ASAN_MAPPING() AsanMappingProfile[__LINE__]++;
+# else
+# define PROFILE_ASAN_MAPPING()
+# endif
// If 1, all shadow boundaries are constants.
// Don't set to 1 other than for testing.
-#define ASAN_FIXED_MAPPING 0
+# define ASAN_FIXED_MAPPING 0
namespace __asan {
extern uptr AsanMappingProfile[];
-#if ASAN_FIXED_MAPPING
+# if ASAN_FIXED_MAPPING
// Fixed mapping for 64-bit Linux. Mostly used for performance comparison
// with non-fixed mapping. As of r175253 (Feb 2013) the performance
// difference between fixed and non-fixed mapping is below the noise level.
static uptr kHighMemEnd = 0x7fffffffffffULL;
-static uptr kMidMemBeg = 0x3000000000ULL;
-static uptr kMidMemEnd = 0x4fffffffffULL;
-#else
+static uptr kMidMemBeg = 0x3000000000ULL;
+static uptr kMidMemEnd = 0x4fffffffffULL;
+# else
extern uptr kHighMemEnd, kMidMemBeg, kMidMemEnd; // Initialized in __asan_init.
-#endif
+# endif
} // namespace __asan
-#if defined(__sparc__) && SANITIZER_WORDSIZE == 64
-# include "asan_mapping_sparc64.h"
-#else
-#define MEM_TO_SHADOW(mem) (((mem) >> SHADOW_SCALE) + (SHADOW_OFFSET))
+# if defined(__sparc__) && SANITIZER_WORDSIZE == 64
+# include "asan_mapping_sparc64.h"
+# else
+# define MEM_TO_SHADOW(mem) \
+ (((mem) >> ASAN_SHADOW_SCALE) + (ASAN_SHADOW_OFFSET))
+# define SHADOW_TO_MEM(mem) \
+ (((mem) - (ASAN_SHADOW_OFFSET)) << (ASAN_SHADOW_SCALE))
-#define kLowMemBeg 0
-#define kLowMemEnd (SHADOW_OFFSET ? SHADOW_OFFSET - 1 : 0)
+# define kLowMemBeg 0
+# define kLowMemEnd (ASAN_SHADOW_OFFSET ? ASAN_SHADOW_OFFSET - 1 : 0)
-#define kLowShadowBeg SHADOW_OFFSET
-#define kLowShadowEnd MEM_TO_SHADOW(kLowMemEnd)
+# define kLowShadowBeg ASAN_SHADOW_OFFSET
+# define kLowShadowEnd MEM_TO_SHADOW(kLowMemEnd)
-#define kHighMemBeg (MEM_TO_SHADOW(kHighMemEnd) + 1)
+# define kHighMemBeg (MEM_TO_SHADOW(kHighMemEnd) + 1)
-#define kHighShadowBeg MEM_TO_SHADOW(kHighMemBeg)
-#define kHighShadowEnd MEM_TO_SHADOW(kHighMemEnd)
+# define kHighShadowBeg MEM_TO_SHADOW(kHighMemBeg)
+# define kHighShadowEnd MEM_TO_SHADOW(kHighMemEnd)
-# define kMidShadowBeg MEM_TO_SHADOW(kMidMemBeg)
-# define kMidShadowEnd MEM_TO_SHADOW(kMidMemEnd)
+# define kMidShadowBeg MEM_TO_SHADOW(kMidMemBeg)
+# define kMidShadowEnd MEM_TO_SHADOW(kMidMemEnd)
// With the zero shadow base we can not actually map pages starting from 0.
// This constant is somewhat arbitrary.
-#define kZeroBaseShadowStart 0
-#define kZeroBaseMaxShadowStart (1 << 18)
+# define kZeroBaseShadowStart 0
+# define kZeroBaseMaxShadowStart (1 << 18)
-#define kShadowGapBeg (kLowShadowEnd ? kLowShadowEnd + 1 \
- : kZeroBaseShadowStart)
-#define kShadowGapEnd ((kMidMemBeg ? kMidShadowBeg : kHighShadowBeg) - 1)
+# define kShadowGapBeg \
+ (kLowShadowEnd ? kLowShadowEnd + 1 : kZeroBaseShadowStart)
+# define kShadowGapEnd ((kMidMemBeg ? kMidShadowBeg : kHighShadowBeg) - 1)
-#define kShadowGap2Beg (kMidMemBeg ? kMidShadowEnd + 1 : 0)
-#define kShadowGap2End (kMidMemBeg ? kMidMemBeg - 1 : 0)
+# define kShadowGap2Beg (kMidMemBeg ? kMidShadowEnd + 1 : 0)
+# define kShadowGap2End (kMidMemBeg ? kMidMemBeg - 1 : 0)
-#define kShadowGap3Beg (kMidMemBeg ? kMidMemEnd + 1 : 0)
-#define kShadowGap3End (kMidMemBeg ? kHighShadowBeg - 1 : 0)
+# define kShadowGap3Beg (kMidMemBeg ? kMidMemEnd + 1 : 0)
+# define kShadowGap3End (kMidMemBeg ? kHighShadowBeg - 1 : 0)
namespace __asan {
PROFILE_ASAN_MAPPING();
if (kMidMemBeg) {
if (a <= kShadowGapEnd)
- return SHADOW_OFFSET == 0 || a >= kShadowGapBeg;
+ return ASAN_SHADOW_OFFSET == 0 || a >= kShadowGapBeg;
return (a >= kShadowGap2Beg && a <= kShadowGap2End) ||
(a >= kShadowGap3Beg && a <= kShadowGap3End);
}
// In zero-based shadow mode we treat addresses near zero as addresses
// in shadow gap as well.
- if (SHADOW_OFFSET == 0)
+ if (ASAN_SHADOW_OFFSET == 0)
return a <= kShadowGapEnd;
return a >= kShadowGapBeg && a <= kShadowGapEnd;
}
} // namespace __asan
-#endif
+# endif
namespace __asan {
-static inline uptr MemToShadowSize(uptr size) { return size >> SHADOW_SCALE; }
+static inline uptr MemToShadowSize(uptr size) {
+ return size >> ASAN_SHADOW_SCALE;
+}
static inline bool AddrIsInMem(uptr a) {
PROFILE_ASAN_MAPPING();
return AddrIsInLowMem(a) || AddrIsInMidMem(a) || AddrIsInHighMem(a) ||
- (flags()->protect_shadow_gap == 0 && AddrIsInShadowGap(a));
+ (flags()->protect_shadow_gap == 0 && AddrIsInShadowGap(a));
}
static inline uptr MemToShadow(uptr p) {
return AddrIsInLowShadow(a) || AddrIsInMidShadow(a) || AddrIsInHighShadow(a);
}
+static inline uptr ShadowToMem(uptr p) {
+ PROFILE_ASAN_MAPPING();
+ CHECK(AddrIsInShadow(p));
+ return SHADOW_TO_MEM(p);
+}
+
static inline bool AddrIsAlignedByGranularity(uptr a) {
PROFILE_ASAN_MAPPING();
- return (a & (SHADOW_GRANULARITY - 1)) == 0;
+ return (a & (ASAN_SHADOW_GRANULARITY - 1)) == 0;
}
static inline bool AddressIsPoisoned(uptr a) {
PROFILE_ASAN_MAPPING();
const uptr kAccessSize = 1;
- u8 *shadow_address = (u8*)MEM_TO_SHADOW(a);
+ u8 *shadow_address = (u8 *)MEM_TO_SHADOW(a);
s8 shadow_value = *shadow_address;
if (shadow_value) {
- u8 last_accessed_byte = (a & (SHADOW_GRANULARITY - 1))
- + kAccessSize - 1;
+ u8 last_accessed_byte =
+ (a & (ASAN_SHADOW_GRANULARITY - 1)) + kAccessSize - 1;
return (last_accessed_byte >= shadow_value);
}
return false;
} // namespace __asan
+#endif // __cplusplus
+
#endif // ASAN_MAPPING_H
// The idea is to chop the high bits before doing the scaling, so the two
// parts become contiguous again and the usual scheme can be applied.
-#define MEM_TO_SHADOW(mem) \
- ((((mem) << HIGH_BITS) >> (HIGH_BITS + (SHADOW_SCALE))) + (SHADOW_OFFSET))
+#define MEM_TO_SHADOW(mem) \
+ ((((mem) << HIGH_BITS) >> (HIGH_BITS + (ASAN_SHADOW_SCALE))) + \
+ (ASAN_SHADOW_OFFSET))
+#define SHADOW_TO_MEM(ptr) (__asan::ShadowToMemSparc64(ptr))
#define kLowMemBeg 0
-#define kLowMemEnd (SHADOW_OFFSET - 1)
+#define kLowMemEnd (ASAN_SHADOW_OFFSET - 1)
-#define kLowShadowBeg SHADOW_OFFSET
+#define kLowShadowBeg ASAN_SHADOW_OFFSET
#define kLowShadowEnd MEM_TO_SHADOW(kLowMemEnd)
// But of course there is the huge hole between the high shadow memory,
return a >= kShadowGapBeg && a <= kShadowGapEnd;
}
+static inline constexpr uptr ShadowToMemSparc64(uptr p) {
+ PROFILE_ASAN_MAPPING();
+ p -= ASAN_SHADOW_OFFSET;
+ p <<= ASAN_SHADOW_SCALE;
+ if (p >= 0x8000000000000) {
+ p |= (~0ULL) << VMA_BITS;
+ }
+ return p;
+}
+
+static_assert(ShadowToMemSparc64(MEM_TO_SHADOW(0x0000000000000000)) ==
+ 0x0000000000000000);
+static_assert(ShadowToMemSparc64(MEM_TO_SHADOW(0xfff8000000000000)) ==
+ 0xfff8000000000000);
+// Gets aligned down.
+static_assert(ShadowToMemSparc64(MEM_TO_SHADOW(0x0007ffffffffffff)) ==
+ 0x0007fffffffffff8);
+
} // namespace __asan
#endif // ASAN_MAPPING_SPARC64_H
// delete.
// To make sure that C++ allocation/deallocation operators are overridden on
// OS X we need to intercept them using their mangled names.
-#if !SANITIZER_MAC
+#if !SANITIZER_APPLE
CXX_OPERATOR_ATTRIBUTE
void *operator new(size_t size)
{ OPERATOR_NEW_BODY(FROM_NEW, false /*nothrow*/); }
void *operator new[](size_t size, std::align_val_t align, std::nothrow_t const&)
{ OPERATOR_NEW_BODY_ALIGN(FROM_NEW_BR, true /*nothrow*/); }
-#else // SANITIZER_MAC
+#else // SANITIZER_APPLE
INTERCEPTOR(void *, _Znwm, size_t size) {
OPERATOR_NEW_BODY(FROM_NEW, false /*nothrow*/);
}
INTERCEPTOR(void *, _ZnamRKSt9nothrow_t, size_t size, std::nothrow_t const&) {
OPERATOR_NEW_BODY(FROM_NEW_BR, true /*nothrow*/);
}
-#endif // !SANITIZER_MAC
+#endif // !SANITIZER_APPLE
#define OPERATOR_DELETE_BODY(type) \
GET_STACK_TRACE_FREE; \
GET_STACK_TRACE_FREE; \
asan_delete(ptr, size, static_cast<uptr>(align), &stack, type);
-#if !SANITIZER_MAC
+#if !SANITIZER_APPLE
CXX_OPERATOR_ATTRIBUTE
void operator delete(void *ptr) NOEXCEPT
{ OPERATOR_DELETE_BODY(FROM_NEW); }
void operator delete[](void *ptr, size_t size, std::align_val_t align) NOEXCEPT
{ OPERATOR_DELETE_BODY_SIZE_ALIGN(FROM_NEW_BR); }
-#else // SANITIZER_MAC
+#else // SANITIZER_APPLE
INTERCEPTOR(void, _ZdlPv, void *ptr)
{ OPERATOR_DELETE_BODY(FROM_NEW); }
INTERCEPTOR(void, _ZdaPv, void *ptr)
{ OPERATOR_DELETE_BODY(FROM_NEW); }
INTERCEPTOR(void, _ZdaPvRKSt9nothrow_t, void *ptr, std::nothrow_t const&)
{ OPERATOR_DELETE_BODY(FROM_NEW_BR); }
-#endif // !SANITIZER_MAC
+#endif // !SANITIZER_APPLE
//===----------------------------------------------------------------------===//
#include "asan_poisoning.h"
+
#include "asan_report.h"
#include "asan_stack.h"
#include "sanitizer_common/sanitizer_atomic.h"
-#include "sanitizer_common/sanitizer_libc.h"
#include "sanitizer_common/sanitizer_flags.h"
+#include "sanitizer_common/sanitizer_interface_internal.h"
+#include "sanitizer_common/sanitizer_libc.h"
namespace __asan {
CHECK(AddrIsAlignedByGranularity(addr));
CHECK(AddrIsInMem(addr));
CHECK(AddrIsAlignedByGranularity(addr + size));
- CHECK(AddrIsInMem(addr + size - SHADOW_GRANULARITY));
+ CHECK(AddrIsInMem(addr + size - ASAN_SHADOW_GRANULARITY));
CHECK(REAL(memset));
FastPoisonShadow(addr, size, value);
}
struct ShadowSegmentEndpoint {
u8 *chunk;
- s8 offset; // in [0, SHADOW_GRANULARITY)
+ s8 offset; // in [0, ASAN_SHADOW_GRANULARITY)
s8 value; // = *chunk;
explicit ShadowSegmentEndpoint(uptr address) {
chunk = (u8*)MemToShadow(address);
- offset = address & (SHADOW_GRANULARITY - 1);
+ offset = address & (ASAN_SHADOW_GRANULARITY - 1);
value = *chunk;
}
};
uptr end = ptr + size;
if (Verbosity()) {
Printf("__asan_%spoison_intra_object_redzone [%p,%p) %zd\n",
- poison ? "" : "un", ptr, end, size);
+ poison ? "" : "un", (void *)ptr, (void *)end, size);
if (Verbosity() >= 2)
PRINT_CURRENT_STACK();
}
CHECK(size);
CHECK_LE(size, 4096);
- CHECK(IsAligned(end, SHADOW_GRANULARITY));
- if (!IsAligned(ptr, SHADOW_GRANULARITY)) {
+ CHECK(IsAligned(end, ASAN_SHADOW_GRANULARITY));
+ if (!IsAligned(ptr, ASAN_SHADOW_GRANULARITY)) {
*(u8 *)MemToShadow(ptr) =
- poison ? static_cast<u8>(ptr % SHADOW_GRANULARITY) : 0;
- ptr |= SHADOW_GRANULARITY - 1;
+ poison ? static_cast<u8>(ptr % ASAN_SHADOW_GRANULARITY) : 0;
+ ptr |= ASAN_SHADOW_GRANULARITY - 1;
ptr++;
}
- for (; ptr < end; ptr += SHADOW_GRANULARITY)
+ for (; ptr < end; ptr += ASAN_SHADOW_GRANULARITY)
*(u8*)MemToShadow(ptr) = poison ? kAsanIntraObjectRedzone : 0;
}
if (!AddrIsInMem(end))
return end;
CHECK_LT(beg, end);
- uptr aligned_b = RoundUpTo(beg, SHADOW_GRANULARITY);
- uptr aligned_e = RoundDownTo(end, SHADOW_GRANULARITY);
+ uptr aligned_b = RoundUpTo(beg, ASAN_SHADOW_GRANULARITY);
+ uptr aligned_e = RoundDownTo(end, ASAN_SHADOW_GRANULARITY);
uptr shadow_beg = MemToShadow(aligned_b);
uptr shadow_end = MemToShadow(aligned_e);
// First check the first and the last application bytes,
- // then check the SHADOW_GRANULARITY-aligned region by calling
+ // then check the ASAN_SHADOW_GRANULARITY-aligned region by calling
// mem_is_zero on the corresponding shadow.
if (!__asan::AddressIsPoisoned(beg) && !__asan::AddressIsPoisoned(end - 1) &&
(shadow_end <= shadow_beg ||
// assumes that left border of region to be poisoned is properly aligned.
static void PoisonAlignedStackMemory(uptr addr, uptr size, bool do_poison) {
if (size == 0) return;
- uptr aligned_size = size & ~(SHADOW_GRANULARITY - 1);
+ uptr aligned_size = size & ~(ASAN_SHADOW_GRANULARITY - 1);
PoisonShadow(addr, aligned_size,
do_poison ? kAsanStackUseAfterScopeMagic : 0);
if (size == aligned_size)
REAL(memset)((void *)addr, 0, size);
}
+void __asan_set_shadow_01(uptr addr, uptr size) {
+ REAL(memset)((void *)addr, 0x01, size);
+}
+
+void __asan_set_shadow_02(uptr addr, uptr size) {
+ REAL(memset)((void *)addr, 0x02, size);
+}
+
+void __asan_set_shadow_03(uptr addr, uptr size) {
+ REAL(memset)((void *)addr, 0x03, size);
+}
+
+void __asan_set_shadow_04(uptr addr, uptr size) {
+ REAL(memset)((void *)addr, 0x04, size);
+}
+
+void __asan_set_shadow_05(uptr addr, uptr size) {
+ REAL(memset)((void *)addr, 0x05, size);
+}
+
+void __asan_set_shadow_06(uptr addr, uptr size) {
+ REAL(memset)((void *)addr, 0x06, size);
+}
+
+void __asan_set_shadow_07(uptr addr, uptr size) {
+ REAL(memset)((void *)addr, 0x07, size);
+}
+
void __asan_set_shadow_f1(uptr addr, uptr size) {
REAL(memset)((void *)addr, 0xf1, size);
}
PoisonAlignedStackMemory(addr, size, false);
}
+static void FixUnalignedStorage(uptr storage_beg, uptr storage_end,
+ uptr &old_beg, uptr &old_end, uptr &new_beg,
+ uptr &new_end) {
+ constexpr uptr granularity = ASAN_SHADOW_GRANULARITY;
+ if (UNLIKELY(!AddrIsAlignedByGranularity(storage_end))) {
+ uptr end_down = RoundDownTo(storage_end, granularity);
+ // Ignore the last unaligned granule if the storage is followed by
+ // unpoisoned byte, because we can't poison the prefix anyway. Don't call
+ // AddressIsPoisoned at all if container changes does not affect the last
+ // granule at all.
+ if ((((old_end != new_end) && Max(old_end, new_end) > end_down) ||
+ ((old_beg != new_beg) && Max(old_beg, new_beg) > end_down)) &&
+ !AddressIsPoisoned(storage_end)) {
+ old_beg = Min(end_down, old_beg);
+ old_end = Min(end_down, old_end);
+ new_beg = Min(end_down, new_beg);
+ new_end = Min(end_down, new_end);
+ }
+ }
+
+ // Handle misaligned begin and cut it off.
+ if (UNLIKELY(!AddrIsAlignedByGranularity(storage_beg))) {
+ uptr beg_up = RoundUpTo(storage_beg, granularity);
+ // The first unaligned granule needs special handling only if we had bytes
+ // there before and will have none after.
+ if ((new_beg == new_end || new_beg >= beg_up) && old_beg != old_end &&
+ old_beg < beg_up) {
+ // Keep granule prefix outside of the storage unpoisoned.
+ uptr beg_down = RoundDownTo(storage_beg, granularity);
+ *(u8 *)MemToShadow(beg_down) = storage_beg - beg_down;
+ old_beg = Max(beg_up, old_beg);
+ old_end = Max(beg_up, old_end);
+ new_beg = Max(beg_up, new_beg);
+ new_end = Max(beg_up, new_end);
+ }
+ }
+}
+
void __sanitizer_annotate_contiguous_container(const void *beg_p,
const void *end_p,
const void *old_mid_p,
const void *new_mid_p) {
- if (!flags()->detect_container_overflow) return;
+ if (!flags()->detect_container_overflow)
+ return;
VPrintf(2, "contiguous_container: %p %p %p %p\n", beg_p, end_p, old_mid_p,
new_mid_p);
- uptr beg = reinterpret_cast<uptr>(beg_p);
- uptr end = reinterpret_cast<uptr>(end_p);
- uptr old_mid = reinterpret_cast<uptr>(old_mid_p);
- uptr new_mid = reinterpret_cast<uptr>(new_mid_p);
- uptr granularity = SHADOW_GRANULARITY;
- if (!(beg <= old_mid && beg <= new_mid && old_mid <= end && new_mid <= end &&
- IsAligned(beg, granularity))) {
+ uptr storage_beg = reinterpret_cast<uptr>(beg_p);
+ uptr storage_end = reinterpret_cast<uptr>(end_p);
+ uptr old_end = reinterpret_cast<uptr>(old_mid_p);
+ uptr new_end = reinterpret_cast<uptr>(new_mid_p);
+ uptr old_beg = storage_beg;
+ uptr new_beg = storage_beg;
+ uptr granularity = ASAN_SHADOW_GRANULARITY;
+ if (!(storage_beg <= old_end && storage_beg <= new_end &&
+ old_end <= storage_end && new_end <= storage_end)) {
GET_STACK_TRACE_FATAL_HERE;
- ReportBadParamsToAnnotateContiguousContainer(beg, end, old_mid, new_mid,
- &stack);
+ ReportBadParamsToAnnotateContiguousContainer(storage_beg, storage_end,
+ old_end, new_end, &stack);
}
- CHECK_LE(end - beg,
- FIRST_32_SECOND_64(1UL << 30, 1ULL << 40)); // Sanity check.
+ CHECK_LE(storage_end - storage_beg,
+ FIRST_32_SECOND_64(1UL << 30, 1ULL << 40)); // Sanity check.
+
+ if (old_end == new_end)
+ return; // Nothing to do here.
- uptr a = RoundDownTo(Min(old_mid, new_mid), granularity);
- uptr c = RoundUpTo(Max(old_mid, new_mid), granularity);
- uptr d1 = RoundDownTo(old_mid, granularity);
+ FixUnalignedStorage(storage_beg, storage_end, old_beg, old_end, new_beg,
+ new_end);
+
+ uptr a = RoundDownTo(Min(old_end, new_end), granularity);
+ uptr c = RoundUpTo(Max(old_end, new_end), granularity);
+ uptr d1 = RoundDownTo(old_end, granularity);
// uptr d2 = RoundUpTo(old_mid, granularity);
// Currently we should be in this state:
// [a, d1) is good, [d2, c) is bad, [d1, d2) is partially good.
// if (d1 != d2)
// CHECK_EQ(*(u8*)MemToShadow(d1), old_mid - d1);
if (a + granularity <= d1)
- CHECK_EQ(*(u8*)MemToShadow(a), 0);
+ CHECK_EQ(*(u8 *)MemToShadow(a), 0);
// if (d2 + granularity <= c && c <= end)
// CHECK_EQ(*(u8 *)MemToShadow(c - granularity),
// kAsanContiguousContainerOOBMagic);
- uptr b1 = RoundDownTo(new_mid, granularity);
- uptr b2 = RoundUpTo(new_mid, granularity);
+ uptr b1 = RoundDownTo(new_end, granularity);
+ uptr b2 = RoundUpTo(new_end, granularity);
// New state:
// [a, b1) is good, [b2, c) is bad, [b1, b2) is partially good.
- PoisonShadow(a, b1 - a, 0);
- PoisonShadow(b2, c - b2, kAsanContiguousContainerOOBMagic);
+ if (b1 > a)
+ PoisonShadow(a, b1 - a, 0);
+ else if (c > b2)
+ PoisonShadow(b2, c - b2, kAsanContiguousContainerOOBMagic);
if (b1 != b2) {
CHECK_EQ(b2 - b1, granularity);
- *(u8*)MemToShadow(b1) = static_cast<u8>(new_mid - b1);
+ *(u8 *)MemToShadow(b1) = static_cast<u8>(new_end - b1);
+ }
+}
+
+// Annotates a double ended contiguous memory area like std::deque's chunk.
+// It allows detecting buggy accesses to allocated but not used begining
+// or end items of such a container.
+void __sanitizer_annotate_double_ended_contiguous_container(
+ const void *storage_beg_p, const void *storage_end_p,
+ const void *old_container_beg_p, const void *old_container_end_p,
+ const void *new_container_beg_p, const void *new_container_end_p) {
+ if (!flags()->detect_container_overflow)
+ return;
+
+ VPrintf(2, "contiguous_container: %p %p %p %p %p %p\n", storage_beg_p,
+ storage_end_p, old_container_beg_p, old_container_end_p,
+ new_container_beg_p, new_container_end_p);
+
+ uptr storage_beg = reinterpret_cast<uptr>(storage_beg_p);
+ uptr storage_end = reinterpret_cast<uptr>(storage_end_p);
+ uptr old_beg = reinterpret_cast<uptr>(old_container_beg_p);
+ uptr old_end = reinterpret_cast<uptr>(old_container_end_p);
+ uptr new_beg = reinterpret_cast<uptr>(new_container_beg_p);
+ uptr new_end = reinterpret_cast<uptr>(new_container_end_p);
+
+ constexpr uptr granularity = ASAN_SHADOW_GRANULARITY;
+
+ if (!(old_beg <= old_end && new_beg <= new_end) ||
+ !(storage_beg <= new_beg && new_end <= storage_end) ||
+ !(storage_beg <= old_beg && old_end <= storage_end)) {
+ GET_STACK_TRACE_FATAL_HERE;
+ ReportBadParamsToAnnotateDoubleEndedContiguousContainer(
+ storage_beg, storage_end, old_beg, old_end, new_beg, new_end, &stack);
}
+ CHECK_LE(storage_end - storage_beg,
+ FIRST_32_SECOND_64(1UL << 30, 1ULL << 40)); // Sanity check.
+
+ if ((old_beg == old_end && new_beg == new_end) ||
+ (old_beg == new_beg && old_end == new_end))
+ return; // Nothing to do here.
+
+ FixUnalignedStorage(storage_beg, storage_end, old_beg, old_end, new_beg,
+ new_end);
+
+ // Handle non-intersecting new/old containers separately have simpler
+ // intersecting case.
+ if (old_beg == old_end || new_beg == new_end || new_end <= old_beg ||
+ old_end <= new_beg) {
+ if (old_beg != old_end) {
+ // Poisoning the old container.
+ uptr a = RoundDownTo(old_beg, granularity);
+ uptr b = RoundUpTo(old_end, granularity);
+ PoisonShadow(a, b - a, kAsanContiguousContainerOOBMagic);
+ }
+
+ if (new_beg != new_end) {
+ // Unpoisoning the new container.
+ uptr a = RoundDownTo(new_beg, granularity);
+ uptr b = RoundDownTo(new_end, granularity);
+ PoisonShadow(a, b - a, 0);
+ if (!AddrIsAlignedByGranularity(new_end))
+ *(u8 *)MemToShadow(b) = static_cast<u8>(new_end - b);
+ }
+
+ return;
+ }
+
+ // Intersection of old and new containers is not empty.
+ CHECK_LT(new_beg, old_end);
+ CHECK_GT(new_end, old_beg);
+
+ if (new_beg < old_beg) {
+ // Round down because we can't poison prefixes.
+ uptr a = RoundDownTo(new_beg, granularity);
+ // Round down and ignore the [c, old_beg) as its state defined by unchanged
+ // [old_beg, old_end).
+ uptr c = RoundDownTo(old_beg, granularity);
+ PoisonShadow(a, c - a, 0);
+ } else if (new_beg > old_beg) {
+ // Round down and poison [a, old_beg) because it was unpoisoned only as a
+ // prefix.
+ uptr a = RoundDownTo(old_beg, granularity);
+ // Round down and ignore the [c, new_beg) as its state defined by unchanged
+ // [new_beg, old_end).
+ uptr c = RoundDownTo(new_beg, granularity);
+
+ PoisonShadow(a, c - a, kAsanContiguousContainerOOBMagic);
+ }
+
+ if (new_end > old_end) {
+ // Round down to poison the prefix.
+ uptr a = RoundDownTo(old_end, granularity);
+ // Round down and handle remainder below.
+ uptr c = RoundDownTo(new_end, granularity);
+ PoisonShadow(a, c - a, 0);
+ if (!AddrIsAlignedByGranularity(new_end))
+ *(u8 *)MemToShadow(c) = static_cast<u8>(new_end - c);
+ } else if (new_end < old_end) {
+ // Round up and handle remained below.
+ uptr a2 = RoundUpTo(new_end, granularity);
+ // Round up to poison entire granule as we had nothing in [old_end, c2).
+ uptr c2 = RoundUpTo(old_end, granularity);
+ PoisonShadow(a2, c2 - a2, kAsanContiguousContainerOOBMagic);
+
+ if (!AddrIsAlignedByGranularity(new_end)) {
+ uptr a = RoundDownTo(new_end, granularity);
+ *(u8 *)MemToShadow(a) = static_cast<u8>(new_end - a);
+ }
+ }
+}
+
+static const void *FindBadAddress(uptr begin, uptr end, bool poisoned) {
+ CHECK_LE(begin, end);
+ constexpr uptr kMaxRangeToCheck = 32;
+ if (end - begin > kMaxRangeToCheck * 2) {
+ if (auto *bad = FindBadAddress(begin, begin + kMaxRangeToCheck, poisoned))
+ return bad;
+ if (auto *bad = FindBadAddress(end - kMaxRangeToCheck, end, poisoned))
+ return bad;
+ }
+
+ for (uptr i = begin; i < end; ++i)
+ if (AddressIsPoisoned(i) != poisoned)
+ return reinterpret_cast<const void *>(i);
+ return nullptr;
}
const void *__sanitizer_contiguous_container_find_bad_address(
const void *beg_p, const void *mid_p, const void *end_p) {
if (!flags()->detect_container_overflow)
return nullptr;
+ uptr granularity = ASAN_SHADOW_GRANULARITY;
uptr beg = reinterpret_cast<uptr>(beg_p);
uptr end = reinterpret_cast<uptr>(end_p);
uptr mid = reinterpret_cast<uptr>(mid_p);
CHECK_LE(beg, mid);
CHECK_LE(mid, end);
- // Check some bytes starting from beg, some bytes around mid, and some bytes
- // ending with end.
- uptr kMaxRangeToCheck = 32;
- uptr r1_beg = beg;
- uptr r1_end = Min(beg + kMaxRangeToCheck, mid);
- uptr r2_beg = Max(beg, mid - kMaxRangeToCheck);
- uptr r2_end = Min(end, mid + kMaxRangeToCheck);
- uptr r3_beg = Max(end - kMaxRangeToCheck, mid);
- uptr r3_end = end;
- for (uptr i = r1_beg; i < r1_end; i++)
- if (AddressIsPoisoned(i))
- return reinterpret_cast<const void *>(i);
- for (uptr i = r2_beg; i < mid; i++)
- if (AddressIsPoisoned(i))
- return reinterpret_cast<const void *>(i);
- for (uptr i = mid; i < r2_end; i++)
- if (!AddressIsPoisoned(i))
- return reinterpret_cast<const void *>(i);
- for (uptr i = r3_beg; i < r3_end; i++)
- if (!AddressIsPoisoned(i))
- return reinterpret_cast<const void *>(i);
- return nullptr;
+ // If the byte after the storage is unpoisoned, everything in the granule
+ // before must stay unpoisoned.
+ uptr annotations_end =
+ (!AddrIsAlignedByGranularity(end) && !AddressIsPoisoned(end))
+ ? RoundDownTo(end, granularity)
+ : end;
+ beg = Min(beg, annotations_end);
+ mid = Min(mid, annotations_end);
+ if (auto *bad = FindBadAddress(beg, mid, false))
+ return bad;
+ if (auto *bad = FindBadAddress(mid, annotations_end, true))
+ return bad;
+ return FindBadAddress(annotations_end, end, false);
}
int __sanitizer_verify_contiguous_container(const void *beg_p,
end_p) == nullptr;
}
+const void *__sanitizer_double_ended_contiguous_container_find_bad_address(
+ const void *storage_beg_p, const void *container_beg_p,
+ const void *container_end_p, const void *storage_end_p) {
+ if (!flags()->detect_container_overflow)
+ return nullptr;
+ uptr granularity = ASAN_SHADOW_GRANULARITY;
+ uptr storage_beg = reinterpret_cast<uptr>(storage_beg_p);
+ uptr storage_end = reinterpret_cast<uptr>(storage_end_p);
+ uptr beg = reinterpret_cast<uptr>(container_beg_p);
+ uptr end = reinterpret_cast<uptr>(container_end_p);
+
+ // The prefix of the firs granule of the container is unpoisoned.
+ if (beg != end)
+ beg = Max(storage_beg, RoundDownTo(beg, granularity));
+
+ // If the byte after the storage is unpoisoned, the prefix of the last granule
+ // is unpoisoned.
+ uptr annotations_end = (!AddrIsAlignedByGranularity(storage_end) &&
+ !AddressIsPoisoned(storage_end))
+ ? RoundDownTo(storage_end, granularity)
+ : storage_end;
+ storage_beg = Min(storage_beg, annotations_end);
+ beg = Min(beg, annotations_end);
+ end = Min(end, annotations_end);
+
+ if (auto *bad = FindBadAddress(storage_beg, beg, true))
+ return bad;
+ if (auto *bad = FindBadAddress(beg, end, false))
+ return bad;
+ if (auto *bad = FindBadAddress(end, annotations_end, true))
+ return bad;
+ return FindBadAddress(annotations_end, storage_end, false);
+}
+
+int __sanitizer_verify_double_ended_contiguous_container(
+ const void *storage_beg_p, const void *container_beg_p,
+ const void *container_end_p, const void *storage_end_p) {
+ return __sanitizer_double_ended_contiguous_container_find_bad_address(
+ storage_beg_p, container_beg_p, container_end_p, storage_end_p) ==
+ nullptr;
+}
+
extern "C" SANITIZER_INTERFACE_ATTRIBUTE
void __asan_poison_intra_object_redzone(uptr ptr, uptr size) {
AsanPoisonOrUnpoisonIntraObjectRedzone(ptr, size, true);
common_flags()->clear_shadow_mmap_threshold);
#else
uptr shadow_beg = MEM_TO_SHADOW(aligned_beg);
- uptr shadow_end = MEM_TO_SHADOW(
- aligned_beg + aligned_size - SHADOW_GRANULARITY) + 1;
+ uptr shadow_end =
+ MEM_TO_SHADOW(aligned_beg + aligned_size - ASAN_SHADOW_GRANULARITY) + 1;
// FIXME: Page states are different on Windows, so using the same interface
// for mapping shadow and zeroing out pages doesn't "just work", so we should
// probably provide higher-level interface for these operations.
DCHECK(CanPoisonMemory());
bool poison_partial = flags()->poison_partial;
u8 *shadow = (u8*)MEM_TO_SHADOW(aligned_addr);
- for (uptr i = 0; i < redzone_size; i += SHADOW_GRANULARITY, shadow++) {
- if (i + SHADOW_GRANULARITY <= size) {
+ for (uptr i = 0; i < redzone_size; i += ASAN_SHADOW_GRANULARITY, shadow++) {
+ if (i + ASAN_SHADOW_GRANULARITY <= size) {
*shadow = 0; // fully addressable
} else if (i >= size) {
- *shadow = (SHADOW_GRANULARITY == 128) ? 0xff : value; // unaddressable
+ *shadow =
+ (ASAN_SHADOW_GRANULARITY == 128) ? 0xff : value; // unaddressable
} else {
// first size-i bytes are addressable
*shadow = poison_partial ? static_cast<u8>(size - i) : 0;
#include "sanitizer_common/sanitizer_platform.h"
#if SANITIZER_POSIX
-#include "asan_internal.h"
-#include "asan_interceptors.h"
-#include "asan_mapping.h"
-#include "asan_poisoning.h"
-#include "asan_report.h"
-#include "asan_stack.h"
-#include "sanitizer_common/sanitizer_libc.h"
-#include "sanitizer_common/sanitizer_posix.h"
-#include "sanitizer_common/sanitizer_procmaps.h"
-
-#include <pthread.h>
-#include <signal.h>
-#include <stdlib.h>
-#include <sys/time.h>
-#include <sys/resource.h>
-#include <unistd.h>
+# include <pthread.h>
+# include <signal.h>
+# include <stdlib.h>
+# include <sys/resource.h>
+# include <sys/time.h>
+# include <unistd.h>
+
+# include "asan_interceptors.h"
+# include "asan_internal.h"
+# include "asan_mapping.h"
+# include "asan_poisoning.h"
+# include "asan_report.h"
+# include "asan_stack.h"
+# include "lsan/lsan_common.h"
+# include "sanitizer_common/sanitizer_libc.h"
+# include "sanitizer_common/sanitizer_posix.h"
+# include "sanitizer_common/sanitizer_procmaps.h"
namespace __asan {
}
void PlatformTSDDtor(void *tsd) {
- AsanThreadContext *context = (AsanThreadContext*)tsd;
+ AsanThreadContext *context = (AsanThreadContext *)tsd;
if (context->destructor_iterations > 1) {
context->destructor_iterations--;
CHECK_EQ(0, pthread_setspecific(tsd_key, tsd));
AsanThread::TSDDtor(tsd);
}
#endif
+
+void InstallAtExitCheckLeaks() {
+ if (CAN_SANITIZE_LEAKS) {
+ if (common_flags()->detect_leaks && common_flags()->leak_check_at_exit) {
+ if (flags()->halt_on_error)
+ Atexit(__lsan::DoLeakCheck);
+ else
+ Atexit(__lsan::DoRecoverableLeakCheckVoid);
+ }
+ }
+}
+
} // namespace __asan
#endif // SANITIZER_POSIX
// Conservative upper limit.
uptr PremapShadowSize() {
uptr granularity = GetMmapGranularity();
- return RoundUpTo(GetMaxVirtualAddress() >> SHADOW_SCALE, granularity);
+ return RoundUpTo(GetMaxVirtualAddress() >> ASAN_SHADOW_SCALE, granularity);
}
// Returns an address aligned to 8 pages, such that one page on the left and
// This file contains error reporting code.
//===----------------------------------------------------------------------===//
+#include "asan_report.h"
+
+#include "asan_descriptions.h"
#include "asan_errors.h"
#include "asan_flags.h"
-#include "asan_descriptions.h"
#include "asan_internal.h"
#include "asan_mapping.h"
-#include "asan_report.h"
#include "asan_scariness_score.h"
#include "asan_stack.h"
#include "asan_thread.h"
#include "sanitizer_common/sanitizer_common.h"
#include "sanitizer_common/sanitizer_flags.h"
+#include "sanitizer_common/sanitizer_interface_internal.h"
#include "sanitizer_common/sanitizer_report_decorator.h"
#include "sanitizer_common/sanitizer_stackdepot.h"
#include "sanitizer_common/sanitizer_symbolizer.h"
static void (*error_report_callback)(const char*);
static char *error_message_buffer = nullptr;
static uptr error_message_buffer_pos = 0;
-static BlockingMutex error_message_buf_mutex(LINKER_INITIALIZED);
+static Mutex error_message_buf_mutex;
static const unsigned kAsanBuggyPcPoolSize = 25;
static __sanitizer::atomic_uintptr_t AsanBuggyPcPool[kAsanBuggyPcPoolSize];
void AppendToErrorMessageBuffer(const char *buffer) {
- BlockingMutexLock l(&error_message_buf_mutex);
+ Lock l(&error_message_buf_mutex);
if (!error_message_buffer) {
error_message_buffer =
(char*)MmapOrDieQuietly(kErrorMessageBufferSize, __func__);
const char *zone_name) {
if (zone_ptr) {
if (zone_name) {
- Printf("malloc_zone_from_ptr(%p) = %p, which is %s\n",
- ptr, zone_ptr, zone_name);
+ Printf("malloc_zone_from_ptr(%p) = %p, which is %s\n", (void *)ptr,
+ (void *)zone_ptr, zone_name);
} else {
Printf("malloc_zone_from_ptr(%p) = %p, which doesn't have a name\n",
- ptr, zone_ptr);
+ (void *)ptr, (void *)zone_ptr);
}
} else {
- Printf("malloc_zone_from_ptr(%p) = 0\n", ptr);
+ Printf("malloc_zone_from_ptr(%p) = 0\n", (void *)ptr);
}
}
DumpProcessMap();
// Copy the message buffer so that we could start logging without holding a
- // lock that gets aquired during printing.
+ // lock that gets acquired during printing.
InternalMmapVector<char> buffer_copy(kErrorMessageBufferSize);
{
- BlockingMutexLock l(&error_message_buf_mutex);
+ Lock l(&error_message_buf_mutex);
internal_memcpy(buffer_copy.data(),
error_message_buffer, kErrorMessageBufferSize);
// Clear error_message_buffer so that if we find other errors
in_report.ReportError(error);
}
+void ReportBadParamsToAnnotateDoubleEndedContiguousContainer(
+ uptr storage_beg, uptr storage_end, uptr old_container_beg,
+ uptr old_container_end, uptr new_container_beg, uptr new_container_end,
+ BufferedStackTrace *stack) {
+ ScopedInErrorReport in_report;
+ ErrorBadParamsToAnnotateDoubleEndedContiguousContainer error(
+ GetCurrentTidOrInvalid(), stack, storage_beg, storage_end,
+ old_container_beg, old_container_end, new_container_beg,
+ new_container_end);
+ in_report.ReportError(error);
+}
+
void ReportODRViolation(const __asan_global *g1, u32 stack_id1,
const __asan_global *g2, u32 stack_id2) {
ScopedInErrorReport in_report;
void ReportMacMzReallocUnknown(uptr addr, uptr zone_ptr, const char *zone_name,
BufferedStackTrace *stack) {
ScopedInErrorReport in_report;
- Printf("mz_realloc(%p) -- attempting to realloc unallocated memory.\n"
- "This is an unrecoverable problem, exiting now.\n",
- addr);
+ Printf(
+ "mz_realloc(%p) -- attempting to realloc unallocated memory.\n"
+ "This is an unrecoverable problem, exiting now.\n",
+ (void *)addr);
PrintZoneForPointer(addr, zone_ptr, zone_name);
stack->Print();
DescribeAddressIfHeap(addr);
void ReportGenericError(uptr pc, uptr bp, uptr sp, uptr addr, bool is_write,
uptr access_size, u32 exp, bool fatal) {
+ if (__asan_test_only_reported_buggy_pointer) {
+ *__asan_test_only_reported_buggy_pointer = addr;
+ return;
+ }
if (!fatal && SuppressErrorReport(pc)) return;
ENABLE_FRAME_POINTER;
}
void NOINLINE __asan_set_error_report_callback(void (*callback)(const char*)) {
- BlockingMutexLock l(&error_message_buf_mutex);
+ Lock l(&error_message_buf_mutex);
error_report_callback = callback;
}
void ReportBadParamsToAnnotateContiguousContainer(uptr beg, uptr end,
uptr old_mid, uptr new_mid,
BufferedStackTrace *stack);
+void ReportBadParamsToAnnotateDoubleEndedContiguousContainer(
+ uptr storage_beg, uptr storage_end, uptr old_container_beg,
+ uptr old_container_end, uptr new_container_beg, uptr new_container_end,
+ BufferedStackTrace *stack);
void ReportODRViolation(const __asan_global *g1, u32 stack_id1,
const __asan_global *g2, u32 stack_id2);
#include "lsan/lsan_common.h"
#include "sanitizer_common/sanitizer_atomic.h"
#include "sanitizer_common/sanitizer_flags.h"
+#include "sanitizer_common/sanitizer_interface_internal.h"
#include "sanitizer_common/sanitizer_libc.h"
#include "sanitizer_common/sanitizer_symbolizer.h"
#include "ubsan/ubsan_init.h"
static atomic_uint32_t num_calls;
if (atomic_fetch_add(&num_calls, 1, memory_order_relaxed) != 0) {
// Don't die twice - run a busy loop.
- while (1) { }
+ while (1) {
+ internal_sched_yield();
+ }
}
if (common_flags()->print_module_map >= 1)
DumpProcessMap();
- if (flags()->sleep_before_dying) {
- Report("Sleeping for %d second(s)\n", flags()->sleep_before_dying);
- SleepForSeconds(flags()->sleep_before_dying);
- }
+
+ WaitForDebugger(flags()->sleep_before_dying, "before dying");
+
if (flags()->unmap_shadow_on_exit) {
if (kMidMemBeg) {
UnmapOrDie((void*)kLowShadowBeg, kMidMemBeg - kLowShadowBeg);
// -------------------------- Globals --------------------- {{{1
int asan_inited;
bool asan_init_is_running;
+bool replace_intrin_cached;
#if !ASAN_FIXED_MAPPING
uptr kHighMemEnd, kMidMemBeg, kMidMemEnd;
NOINLINE
static void ReportGenericErrorWrapper(uptr addr, bool is_write, int size,
int exp_arg, bool fatal) {
- if (__asan_test_only_reported_buggy_pointer) {
- *__asan_test_only_reported_buggy_pointer = addr;
- } else {
- GET_CALLER_PC_BP_SP;
- ReportGenericError(pc, bp, sp, addr, is_write, size, exp_arg, fatal);
- }
+ GET_CALLER_PC_BP_SP;
+ ReportGenericError(pc, bp, sp, addr, is_write, size, exp_arg, fatal);
}
// --------------- LowLevelAllocateCallbac ---------- {{{1
#define ASAN_MEMORY_ACCESS_CALLBACK_BODY(type, is_write, size, exp_arg, fatal) \
uptr sp = MEM_TO_SHADOW(addr); \
- uptr s = size <= SHADOW_GRANULARITY ? *reinterpret_cast<u8 *>(sp) \
- : *reinterpret_cast<u16 *>(sp); \
+ uptr s = size <= ASAN_SHADOW_GRANULARITY ? *reinterpret_cast<u8 *>(sp) \
+ : *reinterpret_cast<u16 *>(sp); \
if (UNLIKELY(s)) { \
- if (UNLIKELY(size >= SHADOW_GRANULARITY || \
- ((s8)((addr & (SHADOW_GRANULARITY - 1)) + size - 1)) >= \
+ if (UNLIKELY(size >= ASAN_SHADOW_GRANULARITY || \
+ ((s8)((addr & (ASAN_SHADOW_GRANULARITY - 1)) + size - 1)) >= \
(s8)s)) { \
ReportGenericErrorWrapper(addr, is_write, size, exp_arg, fatal); \
} \
extern "C"
NOINLINE INTERFACE_ATTRIBUTE
void __asan_loadN(uptr addr, uptr size) {
- if (__asan_region_is_poisoned(addr, size)) {
+ if ((addr = __asan_region_is_poisoned(addr, size))) {
GET_CALLER_PC_BP_SP;
ReportGenericError(pc, bp, sp, addr, false, size, 0, true);
}
extern "C"
NOINLINE INTERFACE_ATTRIBUTE
void __asan_exp_loadN(uptr addr, uptr size, u32 exp) {
- if (__asan_region_is_poisoned(addr, size)) {
+ if ((addr = __asan_region_is_poisoned(addr, size))) {
GET_CALLER_PC_BP_SP;
ReportGenericError(pc, bp, sp, addr, false, size, exp, true);
}
extern "C"
NOINLINE INTERFACE_ATTRIBUTE
void __asan_loadN_noabort(uptr addr, uptr size) {
- if (__asan_region_is_poisoned(addr, size)) {
+ if ((addr = __asan_region_is_poisoned(addr, size))) {
GET_CALLER_PC_BP_SP;
ReportGenericError(pc, bp, sp, addr, false, size, 0, false);
}
extern "C"
NOINLINE INTERFACE_ATTRIBUTE
void __asan_storeN(uptr addr, uptr size) {
- if (__asan_region_is_poisoned(addr, size)) {
+ if ((addr = __asan_region_is_poisoned(addr, size))) {
GET_CALLER_PC_BP_SP;
ReportGenericError(pc, bp, sp, addr, true, size, 0, true);
}
extern "C"
NOINLINE INTERFACE_ATTRIBUTE
void __asan_exp_storeN(uptr addr, uptr size, u32 exp) {
- if (__asan_region_is_poisoned(addr, size)) {
+ if ((addr = __asan_region_is_poisoned(addr, size))) {
GET_CALLER_PC_BP_SP;
ReportGenericError(pc, bp, sp, addr, true, size, exp, true);
}
extern "C"
NOINLINE INTERFACE_ATTRIBUTE
void __asan_storeN_noabort(uptr addr, uptr size) {
- if (__asan_region_is_poisoned(addr, size)) {
+ if ((addr = __asan_region_is_poisoned(addr, size))) {
GET_CALLER_PC_BP_SP;
ReportGenericError(pc, bp, sp, addr, true, size, 0, false);
}
case 38: __asan_region_is_poisoned(0, 0); break;
case 39: __asan_describe_address(0); break;
case 40: __asan_set_shadow_00(0, 0); break;
- case 41: __asan_set_shadow_f1(0, 0); break;
- case 42: __asan_set_shadow_f2(0, 0); break;
- case 43: __asan_set_shadow_f3(0, 0); break;
- case 44: __asan_set_shadow_f5(0, 0); break;
- case 45: __asan_set_shadow_f8(0, 0); break;
+ case 41: __asan_set_shadow_01(0, 0); break;
+ case 42: __asan_set_shadow_02(0, 0); break;
+ case 43: __asan_set_shadow_03(0, 0); break;
+ case 44: __asan_set_shadow_04(0, 0); break;
+ case 45: __asan_set_shadow_05(0, 0); break;
+ case 46: __asan_set_shadow_06(0, 0); break;
+ case 47: __asan_set_shadow_07(0, 0); break;
+ case 48: __asan_set_shadow_f1(0, 0); break;
+ case 49: __asan_set_shadow_f2(0, 0); break;
+ case 50: __asan_set_shadow_f3(0, 0); break;
+ case 51: __asan_set_shadow_f5(0, 0); break;
+ case 52: __asan_set_shadow_f8(0, 0); break;
}
// clang-format on
}
kHighMemEnd = GetMaxUserVirtualAddress();
// Increase kHighMemEnd to make sure it's properly
// aligned together with kHighMemBeg:
- kHighMemEnd |= (GetMmapGranularity() << SHADOW_SCALE) - 1;
+ kHighMemEnd |= (GetMmapGranularity() << ASAN_SHADOW_SCALE) - 1;
#endif // !ASAN_FIXED_MAPPING
CHECK_EQ((kHighMemBeg % GetMmapGranularity()), 0);
}
Printf("malloc_context_size=%zu\n",
(uptr)common_flags()->malloc_context_size);
- Printf("SHADOW_SCALE: %d\n", (int)SHADOW_SCALE);
- Printf("SHADOW_GRANULARITY: %d\n", (int)SHADOW_GRANULARITY);
- Printf("SHADOW_OFFSET: 0x%zx\n", (uptr)SHADOW_OFFSET);
- CHECK(SHADOW_SCALE >= 3 && SHADOW_SCALE <= 7);
+ Printf("SHADOW_SCALE: %d\n", (int)ASAN_SHADOW_SCALE);
+ Printf("SHADOW_GRANULARITY: %d\n", (int)ASAN_SHADOW_GRANULARITY);
+ Printf("SHADOW_OFFSET: 0x%zx\n", (uptr)ASAN_SHADOW_OFFSET);
+ CHECK(ASAN_SHADOW_SCALE >= 3 && ASAN_SHADOW_SCALE <= 7);
if (kMidMemBeg)
CHECK(kMidShadowBeg > kLowShadowEnd &&
kMidMemBeg > kMidShadowEnd &&
kHighShadowBeg > kMidMemEnd);
}
-#if defined(__thumb__) && defined(__linux__)
-#define START_BACKGROUND_THREAD_IN_ASAN_INTERNAL
-#endif
-
-#ifndef START_BACKGROUND_THREAD_IN_ASAN_INTERNAL
-static bool UNUSED __local_asan_dyninit = [] {
- MaybeStartBackgroudThread();
- SetSoftRssLimitExceededCallback(AsanSoftRssLimitExceededCallback);
-
- return false;
-}();
-#endif
-
static void AsanInitInternal() {
if (LIKELY(asan_inited)) return;
SanitizerToolName = "AddressSanitizer";
// initialization steps look at flags().
InitializeFlags();
+ WaitForDebugger(flags()->sleep_before_init, "before init");
+
// Stop performing init at this point if we are being loaded via
// dlopen() and the platform supports it.
if (SANITIZER_SUPPORTS_INIT_FOR_DLOPEN && UNLIKELY(HandleDlopenInit())) {
__sanitizer::InitializePlatformEarly();
- // Re-exec ourselves if we need to set additional env or command line args.
- MaybeReexec();
-
// Setup internal allocator callback.
- SetLowLevelAllocateMinAlignment(SHADOW_GRANULARITY);
+ SetLowLevelAllocateMinAlignment(ASAN_SHADOW_GRANULARITY);
SetLowLevelAllocateCallback(OnLowLevelAllocate);
InitializeAsanInterceptors();
allocator_options.SetFrom(flags(), common_flags());
InitializeAllocator(allocator_options);
-#ifdef START_BACKGROUND_THREAD_IN_ASAN_INTERNAL
- MaybeStartBackgroudThread();
- SetSoftRssLimitExceededCallback(AsanSoftRssLimitExceededCallback);
-#endif
+ if (SANITIZER_START_BACKGROUND_THREAD_IN_ASAN_INTERNAL)
+ MaybeStartBackgroudThread();
// On Linux AsanThread::ThreadStart() calls malloc() that's why asan_inited
// should be set to 1 prior to initializing the threads.
+ replace_intrin_cached = flags()->replace_intrin;
asan_inited = 1;
asan_init_is_running = false;
if (CAN_SANITIZE_LEAKS) {
__lsan::InitCommonLsan();
- if (common_flags()->detect_leaks && common_flags()->leak_check_at_exit) {
- if (flags()->halt_on_error)
- Atexit(__lsan::DoLeakCheck);
- else
- Atexit(__lsan::DoRecoverableLeakCheckVoid);
- }
+ InstallAtExitCheckLeaks();
}
#if CAN_SANITIZE_UB
VReport(1, "AddressSanitizer Init done\n");
- if (flags()->sleep_after_init) {
- Report("Sleeping for %d second(s)\n", flags()->sleep_after_init);
- SleepForSeconds(flags()->sleep_after_init);
- }
+ WaitForDebugger(flags()->sleep_after_init, "after init");
}
// Initialize as requested from some part of ASan runtime library (interceptors,
"False positive error reports may follow\n"
"For details see "
"https://github.com/google/sanitizers/issues/189\n",
- type, top, bottom, top - bottom, top - bottom);
+ type, (void *)top, (void *)bottom, (void *)(top - bottom),
+ top - bottom);
return;
}
- PoisonShadow(bottom, RoundUpTo(top - bottom, SHADOW_GRANULARITY), 0);
+ PoisonShadow(bottom, RoundUpTo(top - bottom, ASAN_SHADOW_GRANULARITY), 0);
}
static void UnpoisonDefaultStack() {
--- /dev/null
+//===-- asan_static_rtl.cpp -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of AddressSanitizer, an address sanity checker.
+//
+// Main file of the ASan run-time library.
+//===----------------------------------------------------------------------===//
+
+// This file is empty for now. Main reason to have it is workaround for Windows
+// build, which complains because no files are part of the asan_static lib.
+
+#include "sanitizer_common/sanitizer_common.h"
+
+#define REPORT_FUNCTION(Name) \
+ extern "C" SANITIZER_WEAK_ATTRIBUTE void Name(__asan::uptr addr); \
+ extern "C" void Name##_asm(uptr addr) { Name(addr); }
+
+namespace __asan {
+
+REPORT_FUNCTION(__asan_report_load1)
+REPORT_FUNCTION(__asan_report_load2)
+REPORT_FUNCTION(__asan_report_load4)
+REPORT_FUNCTION(__asan_report_load8)
+REPORT_FUNCTION(__asan_report_load16)
+REPORT_FUNCTION(__asan_report_store1)
+REPORT_FUNCTION(__asan_report_store2)
+REPORT_FUNCTION(__asan_report_store4)
+REPORT_FUNCTION(__asan_report_store8)
+REPORT_FUNCTION(__asan_report_store16)
+
+} // namespace __asan
--- /dev/null
+#include "asan_mapping.h"
+#include "sanitizer_common/sanitizer_asm.h"
+
+#if defined(__x86_64__)
+#include "sanitizer_common/sanitizer_platform.h"
+
+.file "asan_rtl_x86_64.S"
+
+#define NAME(n, reg, op, s, i) n##_##op##_##i##_##s##_##reg
+
+#define FNAME(reg, op, s, i) NAME(__asan_check, reg, op, s, i)
+#define RLABEL(reg, op, s, i) NAME(.return, reg, op, s, i)
+#define CLABEL(reg, op, s, i) NAME(.check, reg, op, s, i)
+#define FLABEL(reg, op, s, i) NAME(.fail, reg, op, s, i)
+
+#define BEGINF(reg, op, s, i) \
+.section .text.FNAME(reg, op, s, i),"ax",@progbits ;\
+.globl FNAME(reg, op, s, i) ;\
+.hidden FNAME(reg, op, s, i) ;\
+ASM_TYPE_FUNCTION(FNAME(reg, op, s, i)) ;\
+.cfi_startproc ;\
+FNAME(reg, op, s, i): ;\
+
+#define ENDF .cfi_endproc ;\
+
+// Access check functions for 1,2 and 4 byte types, which require extra checks.
+#define ASAN_MEMORY_ACCESS_INITIAL_CHECK_ADD(reg, op, s) \
+ mov %##reg,%r10 ;\
+ shr $0x3,%r10 ;\
+ movsbl ASAN_SHADOW_OFFSET_CONST(%r10),%r10d ;\
+ test %r10d,%r10d ;\
+ jne CLABEL(reg, op, s, add) ;\
+RLABEL(reg, op, s, add): ;\
+ retq ;\
+
+#define ASAN_MEMORY_ACCESS_EXTRA_CHECK_1(reg, op, i) \
+CLABEL(reg, op, 1, i): ;\
+ push %rcx ;\
+ mov %##reg,%rcx ;\
+ and $0x7,%ecx ;\
+ cmp %r10d,%ecx ;\
+ pop %rcx ;\
+ jl RLABEL(reg, op, 1, i);\
+ mov %##reg,%rdi ;\
+ jmp __asan_report_##op##1_asm ;\
+
+#define ASAN_MEMORY_ACCESS_EXTRA_CHECK_2(reg, op, i) \
+CLABEL(reg, op, 2, i): ;\
+ push %rcx ;\
+ mov %##reg,%rcx ;\
+ and $0x7,%ecx ;\
+ add $0x1,%ecx ;\
+ cmp %r10d,%ecx ;\
+ pop %rcx ;\
+ jl RLABEL(reg, op, 2, i);\
+ mov %##reg,%rdi ;\
+ jmp __asan_report_##op##2_asm ;\
+
+#define ASAN_MEMORY_ACCESS_EXTRA_CHECK_4(reg, op, i) \
+CLABEL(reg, op, 4, i): ;\
+ push %rcx ;\
+ mov %##reg,%rcx ;\
+ and $0x7,%ecx ;\
+ add $0x3,%ecx ;\
+ cmp %r10d,%ecx ;\
+ pop %rcx ;\
+ jl RLABEL(reg, op, 4, i);\
+ mov %##reg,%rdi ;\
+ jmp __asan_report_##op##4_asm ;\
+
+#define ASAN_MEMORY_ACCESS_CALLBACK_ADD_1(reg, op) \
+BEGINF(reg, op, 1, add) ;\
+ ASAN_MEMORY_ACCESS_INITIAL_CHECK_ADD(reg, op, 1) ;\
+ ASAN_MEMORY_ACCESS_EXTRA_CHECK_1(reg, op, add) ;\
+ENDF
+
+#define ASAN_MEMORY_ACCESS_CALLBACK_ADD_2(reg, op) \
+BEGINF(reg, op, 2, add) ;\
+ ASAN_MEMORY_ACCESS_INITIAL_CHECK_ADD(reg, op, 2) ;\
+ ASAN_MEMORY_ACCESS_EXTRA_CHECK_2(reg, op, add) ;\
+ENDF
+
+#define ASAN_MEMORY_ACCESS_CALLBACK_ADD_4(reg, op) \
+BEGINF(reg, op, 4, add) ;\
+ ASAN_MEMORY_ACCESS_INITIAL_CHECK_ADD(reg, op, 4) ;\
+ ASAN_MEMORY_ACCESS_EXTRA_CHECK_4(reg, op, add) ;\
+ENDF
+
+// Access check functions for 8 and 16 byte types: no extra checks required.
+#define ASAN_MEMORY_ACCESS_CHECK_ADD(reg, op, s, c) \
+ mov %##reg,%r10 ;\
+ shr $0x3,%r10 ;\
+ ##c $0x0,ASAN_SHADOW_OFFSET_CONST(%r10) ;\
+ jne FLABEL(reg, op, s, add) ;\
+ retq ;\
+
+#define ASAN_MEMORY_ACCESS_FAIL(reg, op, s, i) \
+FLABEL(reg, op, s, i): ;\
+ mov %##reg,%rdi ;\
+ jmp __asan_report_##op##s##_asm;\
+
+#define ASAN_MEMORY_ACCESS_CALLBACK_ADD_8(reg, op) \
+BEGINF(reg, op, 8, add) ;\
+ ASAN_MEMORY_ACCESS_CHECK_ADD(reg, op, 8, cmpb) ;\
+ ASAN_MEMORY_ACCESS_FAIL(reg, op, 8, add) ;\
+ENDF
+
+#define ASAN_MEMORY_ACCESS_CALLBACK_ADD_16(reg, op) \
+BEGINF(reg, op, 16, add) ;\
+ ASAN_MEMORY_ACCESS_CHECK_ADD(reg, op, 16, cmpw) ;\
+ ASAN_MEMORY_ACCESS_FAIL(reg, op, 16, add) ;\
+ENDF
+
+#define ASAN_MEMORY_ACCESS_CALLBACKS_ADD(reg) \
+ASAN_MEMORY_ACCESS_CALLBACK_ADD_1(reg, load) \
+ASAN_MEMORY_ACCESS_CALLBACK_ADD_1(reg, store) \
+ASAN_MEMORY_ACCESS_CALLBACK_ADD_2(reg, load) \
+ASAN_MEMORY_ACCESS_CALLBACK_ADD_2(reg, store) \
+ASAN_MEMORY_ACCESS_CALLBACK_ADD_4(reg, load) \
+ASAN_MEMORY_ACCESS_CALLBACK_ADD_4(reg, store) \
+ASAN_MEMORY_ACCESS_CALLBACK_ADD_8(reg, load) \
+ASAN_MEMORY_ACCESS_CALLBACK_ADD_8(reg, store) \
+ASAN_MEMORY_ACCESS_CALLBACK_ADD_16(reg, load) \
+ASAN_MEMORY_ACCESS_CALLBACK_ADD_16(reg, store) \
+
+
+// Instantiate all but R10 and R11 callbacks. We are using PLTSafe class with
+// the intrinsic, which guarantees that the code generation will never emit
+// R10 or R11 callback.
+ASAN_MEMORY_ACCESS_CALLBACKS_ADD(RAX)
+ASAN_MEMORY_ACCESS_CALLBACKS_ADD(RBX)
+ASAN_MEMORY_ACCESS_CALLBACKS_ADD(RCX)
+ASAN_MEMORY_ACCESS_CALLBACKS_ADD(RDX)
+ASAN_MEMORY_ACCESS_CALLBACKS_ADD(RSI)
+ASAN_MEMORY_ACCESS_CALLBACKS_ADD(RDI)
+ASAN_MEMORY_ACCESS_CALLBACKS_ADD(RBP)
+ASAN_MEMORY_ACCESS_CALLBACKS_ADD(R8)
+ASAN_MEMORY_ACCESS_CALLBACKS_ADD(R9)
+ASAN_MEMORY_ACCESS_CALLBACKS_ADD(R12)
+ASAN_MEMORY_ACCESS_CALLBACKS_ADD(R13)
+ASAN_MEMORY_ACCESS_CALLBACKS_ADD(R14)
+ASAN_MEMORY_ACCESS_CALLBACKS_ADD(R15)
+
+#endif
+
+NO_EXEC_STACK_DIRECTIVE
"protect_shadow_gap=0:"
" not protecting shadow gap, allocating gap's shadow\n"
"|| `[%p, %p]` || ShadowGap's shadow ||\n",
- GapShadowBeg, GapShadowEnd);
+ (void*)GapShadowBeg, (void*)GapShadowEnd);
ReserveShadowMemoryRange(GapShadowBeg, GapShadowEnd,
"unprotected gap shadow");
return;
"Shadow memory range interleaves with an existing memory mapping. "
"ASan cannot proceed correctly. ABORTING.\n");
Report("ASan shadow was supposed to be located in the [%p-%p] range.\n",
- shadow_start, kHighShadowEnd);
+ (void*)shadow_start, (void*)kHighShadowEnd);
MaybeReportLinuxPIEBug();
DumpProcessMap();
Die();
dst_ptr[i] += src_ptr[i];
}
-static BlockingMutex print_lock(LINKER_INITIALIZED);
+static Mutex print_lock;
static AsanStats unknown_thread_stats(LINKER_INITIALIZED);
static AsanStats dead_threads_stats(LINKER_INITIALIZED);
-static BlockingMutex dead_threads_stats_lock(LINKER_INITIALIZED);
+static Mutex dead_threads_stats_lock;
// Required for malloc_zone_statistics() on OS X. This can't be stored in
// per-thread AsanStats.
static uptr max_malloced_memory;
}
stats->MergeFrom(&unknown_thread_stats);
{
- BlockingMutexLock lock(&dead_threads_stats_lock);
+ Lock lock(&dead_threads_stats_lock);
stats->MergeFrom(&dead_threads_stats);
}
// This is not very accurate: we may miss allocation peaks that happen
}
void FlushToDeadThreadStats(AsanStats *stats) {
- BlockingMutexLock lock(&dead_threads_stats_lock);
+ Lock lock(&dead_threads_stats_lock);
dead_threads_stats.MergeFrom(stats);
stats->Clear();
}
AsanStats stats;
GetAccumulatedStats(&stats);
// Use lock to keep reports from mixing up.
- BlockingMutexLock lock(&print_lock);
+ Lock lock(&print_lock);
stats.Print();
- StackDepotStats *stack_depot_stats = StackDepotGetStats();
+ StackDepotStats stack_depot_stats = StackDepotGetStats();
Printf("Stats: StackDepot: %zd ids; %zdM allocated\n",
- stack_depot_stats->n_uniq_ids, stack_depot_stats->allocated >> 20);
+ stack_depot_stats.n_uniq_ids, stack_depot_stats.allocated >> 20);
PrintInternalAllocatorStats();
}
static ALIGNED(16) char thread_registry_placeholder[sizeof(ThreadRegistry)];
static ThreadRegistry *asan_thread_registry;
-static BlockingMutex mu_for_thread_context(LINKER_INITIALIZED);
+static Mutex mu_for_thread_context;
static LowLevelAllocator allocator_for_thread_context;
static ThreadContextBase *GetAsanThreadContext(u32 tid) {
- BlockingMutexLock lock(&mu_for_thread_context);
+ Lock lock(&mu_for_thread_context);
return new(allocator_for_thread_context) AsanThreadContext(tid);
}
thread->start_routine_ = start_routine;
thread->arg_ = arg;
AsanThreadContext::CreateThreadContextArgs args = {thread, stack};
- asanThreadRegistry().CreateThread(*reinterpret_cast<uptr *>(thread), detached,
- parent_tid, &args);
+ asanThreadRegistry().CreateThread(0, detached, parent_tid, &args);
return thread;
}
int local = 0;
VReport(1, "T%d: stack [%p,%p) size 0x%zx; local=%p\n", tid(),
(void *)stack_bottom_, (void *)stack_top_, stack_top_ - stack_bottom_,
- &local);
+ (void *)&local);
}
// Fuchsia doesn't use ThreadStart.
uptr stack_size = 0;
GetThreadStackAndTls(tid() == kMainTid, &stack_bottom_, &stack_size,
&tls_begin_, &tls_size);
- stack_top_ = RoundDownTo(stack_bottom_ + stack_size, SHADOW_GRANULARITY);
+ stack_top_ = RoundDownTo(stack_bottom_ + stack_size, ASAN_SHADOW_GRANULARITY);
tls_end_ = tls_begin_ + tls_size;
dtls_ = DTLS_Get();
if (stack_top_ != stack_bottom_)
PoisonShadow(stack_bottom_, stack_top_ - stack_bottom_, 0);
if (tls_begin_ != tls_end_) {
- uptr tls_begin_aligned = RoundDownTo(tls_begin_, SHADOW_GRANULARITY);
- uptr tls_end_aligned = RoundUpTo(tls_end_, SHADOW_GRANULARITY);
- FastPoisonShadowPartialRightRedzone(tls_begin_aligned,
- tls_end_ - tls_begin_aligned,
- tls_end_aligned - tls_end_, 0);
+ uptr tls_begin_aligned = RoundDownTo(tls_begin_, ASAN_SHADOW_GRANULARITY);
+ uptr tls_end_aligned = RoundUpTo(tls_end_, ASAN_SHADOW_GRANULARITY);
+ FastPoisonShadow(tls_begin_aligned, tls_end_aligned - tls_begin_aligned, 0);
}
}
return true;
}
uptr aligned_addr = RoundDownTo(addr, SANITIZER_WORDSIZE / 8); // align addr.
- uptr mem_ptr = RoundDownTo(aligned_addr, SHADOW_GRANULARITY);
+ uptr mem_ptr = RoundDownTo(aligned_addr, ASAN_SHADOW_GRANULARITY);
u8 *shadow_ptr = (u8*)MemToShadow(aligned_addr);
u8 *shadow_bottom = (u8*)MemToShadow(bottom);
while (shadow_ptr >= shadow_bottom &&
*shadow_ptr != kAsanStackLeftRedzoneMagic) {
shadow_ptr--;
- mem_ptr -= SHADOW_GRANULARITY;
+ mem_ptr -= ASAN_SHADOW_GRANULARITY;
}
while (shadow_ptr >= shadow_bottom &&
*shadow_ptr == kAsanStackLeftRedzoneMagic) {
shadow_ptr--;
- mem_ptr -= SHADOW_GRANULARITY;
+ mem_ptr -= ASAN_SHADOW_GRANULARITY;
}
if (shadow_ptr < shadow_bottom) {
return false;
}
- uptr* ptr = (uptr*)(mem_ptr + SHADOW_GRANULARITY);
+ uptr *ptr = (uptr *)(mem_ptr + ASAN_SHADOW_GRANULARITY);
CHECK(ptr[0] == kCurrentStackFrameMagic);
access->offset = addr - (uptr)ptr;
access->frame_pc = ptr[2];
void SetCurrentThread(AsanThread *t) {
CHECK(t->context());
- VReport(2, "SetCurrentThread: %p for thread %p\n", t->context(),
+ VReport(2, "SetCurrentThread: %p for thread %p\n", (void *)t->context(),
(void *)GetThreadSelf());
// Make sure we do not reset the current AsanThread.
CHECK_EQ(0, AsanTSDGet());
// --- Implementation of LSan-specific functions --- {{{1
namespace __lsan {
+void LockThreadRegistry() { __asan::asanThreadRegistry().Lock(); }
+
+void UnlockThreadRegistry() { __asan::asanThreadRegistry().Unlock(); }
+
+static ThreadRegistry *GetAsanThreadRegistryLocked() {
+ __asan::asanThreadRegistry().CheckLocked();
+ return &__asan::asanThreadRegistry();
+}
+
+void EnsureMainThreadIDIsCorrect() { __asan::EnsureMainThreadIDIsCorrect(); }
+
bool GetThreadRangesLocked(tid_t os_id, uptr *stack_begin, uptr *stack_end,
uptr *tls_begin, uptr *tls_end, uptr *cache_begin,
uptr *cache_end, DTLS **dtls) {
void GetAllThreadAllocatorCachesLocked(InternalMmapVector<uptr> *caches) {}
-void ForEachExtraStackRange(tid_t os_id, RangeIteratorCallback callback,
- void *arg) {
+void GetThreadExtraStackRangesLocked(tid_t os_id,
+ InternalMmapVector<Range> *ranges) {
__asan::AsanThread *t = __asan::GetAsanThreadByOsIDLocked(os_id);
if (!t)
return;
__asan::FakeStack *fake_stack = t->get_fake_stack();
if (!fake_stack)
return;
- fake_stack->ForEachFakeFrame(callback, arg);
+
+ fake_stack->ForEachFakeFrame(
+ [](uptr begin, uptr end, void *arg) {
+ reinterpret_cast<InternalMmapVector<Range> *>(arg)->push_back(
+ {begin, end});
+ },
+ ranges);
}
-void LockThreadRegistry() {
- __asan::asanThreadRegistry().Lock();
+void GetThreadExtraStackRangesLocked(InternalMmapVector<Range> *ranges) {
+ GetAsanThreadRegistryLocked()->RunCallbackForEachThreadLocked(
+ [](ThreadContextBase *tctx, void *arg) {
+ GetThreadExtraStackRangesLocked(
+ tctx->os_id, reinterpret_cast<InternalMmapVector<Range> *>(arg));
+ },
+ ranges);
}
-void UnlockThreadRegistry() {
- __asan::asanThreadRegistry().Unlock();
+void GetAdditionalThreadContextPtrsLocked(InternalMmapVector<uptr> *ptrs) {
+ GetAsanThreadRegistryLocked()->RunCallbackForEachThreadLocked(
+ [](ThreadContextBase *tctx, void *ptrs) {
+ // Look for the arg pointer of threads that have been created or are
+ // running. This is necessary to prevent false positive leaks due to the
+ // AsanThread holding the only live reference to a heap object. This
+ // can happen because the `pthread_create()` interceptor doesn't wait
+ // for the child thread to start before returning and thus loosing the
+ // the only live reference to the heap object on the stack.
+
+ __asan::AsanThreadContext *atctx =
+ static_cast<__asan::AsanThreadContext *>(tctx);
+
+ // Note ThreadStatusRunning is required because there is a small window
+ // where the thread status switches to `ThreadStatusRunning` but the
+ // `arg` pointer still isn't on the stack yet.
+ if (atctx->status != ThreadStatusCreated &&
+ atctx->status != ThreadStatusRunning)
+ return;
+
+ uptr thread_arg = reinterpret_cast<uptr>(atctx->thread->get_arg());
+ if (!thread_arg)
+ return;
+
+ auto ptrsVec = reinterpret_cast<InternalMmapVector<uptr> *>(ptrs);
+ ptrsVec->push_back(thread_arg);
+ },
+ ptrs);
}
-ThreadRegistry *GetThreadRegistryLocked() {
- __asan::asanThreadRegistry().CheckLocked();
- return &__asan::asanThreadRegistry();
+void GetRunningThreadsLocked(InternalMmapVector<tid_t> *threads) {
+ GetAsanThreadRegistryLocked()->RunCallbackForEachThreadLocked(
+ [](ThreadContextBase *tctx, void *threads) {
+ if (tctx->status == ThreadStatusRunning)
+ reinterpret_cast<InternalMmapVector<tid_t> *>(threads)->push_back(
+ tctx->os_id);
+ },
+ threads);
}
-void EnsureMainThreadIDIsCorrect() {
- __asan::EnsureMainThreadIDIsCorrect();
+void FinishThreadLocked(u32 tid) {
+ GetAsanThreadRegistryLocked()->FinishThread(tid);
}
+
} // namespace __lsan
// ---------------------- Interface ---------------- {{{1
-//===-- asan_win.cpp ------------------------------------------------------===//
+//===-- asan_win.cpp
+//------------------------------------------------------===//>
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
#include "sanitizer_common/sanitizer_platform.h"
#if SANITIZER_WINDOWS
-#define WIN32_LEAN_AND_MEAN
-#include <windows.h>
-
-#include <stdlib.h>
-
-#include "asan_interceptors.h"
-#include "asan_internal.h"
-#include "asan_mapping.h"
-#include "asan_report.h"
-#include "asan_stack.h"
-#include "asan_thread.h"
-#include "sanitizer_common/sanitizer_libc.h"
-#include "sanitizer_common/sanitizer_mutex.h"
-#include "sanitizer_common/sanitizer_win.h"
-#include "sanitizer_common/sanitizer_win_defs.h"
+# define WIN32_LEAN_AND_MEAN
+# include <stdlib.h>
+# include <windows.h>
+
+# include "asan_interceptors.h"
+# include "asan_internal.h"
+# include "asan_mapping.h"
+# include "asan_report.h"
+# include "asan_stack.h"
+# include "asan_thread.h"
+# include "sanitizer_common/sanitizer_libc.h"
+# include "sanitizer_common/sanitizer_mutex.h"
+# include "sanitizer_common/sanitizer_win.h"
+# include "sanitizer_common/sanitizer_win_defs.h"
using namespace __asan;
static LPTOP_LEVEL_EXCEPTION_FILTER default_seh_handler;
static LPTOP_LEVEL_EXCEPTION_FILTER user_seh_handler;
-extern "C" SANITIZER_INTERFACE_ATTRIBUTE
-long __asan_unhandled_exception_filter(EXCEPTION_POINTERS *info) {
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE long __asan_unhandled_exception_filter(
+ EXCEPTION_POINTERS *info) {
EXCEPTION_RECORD *exception_record = info->ExceptionRecord;
CONTEXT *context = info->ContextRecord;
}
}
+void InstallAtExitCheckLeaks() {}
+
void AsanApplyToGlobals(globals_op_fptr op, const void *needle) {
UNIMPLEMENTED();
}
}
uptr FindDynamicShadowStart() {
- return MapDynamicShadow(MemToShadowSize(kHighMemEnd), SHADOW_SCALE,
+ return MapDynamicShadow(MemToShadowSize(kHighMemEnd), ASAN_SHADOW_SCALE,
/*min_shadow_base_alignment*/ 0, kHighMemEnd);
}
void AsanCheckIncompatibleRT() {}
-void ReadContextStack(void *context, uptr *stack, uptr *ssize) {
- UNIMPLEMENTED();
-}
-
void AsanOnDeadlySignal(int, void *siginfo, void *context) { UNIMPLEMENTED(); }
bool PlatformUnpoisonStacks() { return false; }
// TODO(timurrrr): Do we need to add _Crt* stuff here? (see asan_malloc_win.cpp)
+# if defined(_MSC_VER) && !defined(__clang__)
+// Disable warnings such as: 'void memchr(void)': incorrect number of arguments
+// for intrinsic function, expected '3' arguments.
+# pragma warning(push)
+# pragma warning(disable : 4392)
+# endif
+
INTERCEPT_LIBRARY_FUNCTION(atoi);
INTERCEPT_LIBRARY_FUNCTION(atol);
INTERCEPT_LIBRARY_FUNCTION(frexp);
INTERCEPT_LIBRARY_FUNCTION(wcslen);
INTERCEPT_LIBRARY_FUNCTION(wcsnlen);
+# if defined(_MSC_VER) && !defined(__clang__)
+# pragma warning(pop)
+# endif
+
#ifdef _WIN64
INTERCEPT_LIBRARY_FUNCTION(__C_specific_handler);
#else
-#!/bin/bash
+#!/usr/bin/env bash
#===- lib/asan/scripts/asan_device_setup -----------------------------------===#
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
def is_valid_arch(s):
return s in ["i386", "x86_64", "x86_64h", "arm", "armv6", "armv7", "armv7s",
"armv7k", "arm64", "powerpc64", "powerpc64le", "s390x", "s390",
- "riscv64"]
+ "riscv64", "loongarch64"]
def guess_arch(addr):
# Guess which arch we're running. 10 = len('0x') + 8 hex digits.
set(ASAN_UNITTEST_COMMON_CFLAGS
${COMPILER_RT_UNITTEST_CFLAGS}
${COMPILER_RT_GTEST_CFLAGS}
- ${COMPILER_RT_ASAN_SHADOW_SCALE_LLVM_FLAG}
+ ${SANITIZER_TEST_CXX_CFLAGS}
-I${COMPILER_RT_SOURCE_DIR}/include
-I${COMPILER_RT_SOURCE_DIR}/lib
-I${COMPILER_RT_SOURCE_DIR}/lib/asan
# This will ensure the target linker is used
# during cross compilation
set(ASAN_UNITTEST_COMMON_LINK_FLAGS
- ${COMPILER_RT_UNITTEST_LINK_FLAGS})
+ ${COMPILER_RT_UNITTEST_LINK_FLAGS}
+ ${COMPILER_RT_UNWINDER_LINK_LIBS}
+ ${SANITIZER_TEST_CXX_LIBRARIES})
# -gline-tables-only must be enough for ASan, so use it if possible.
if(COMPILER_RT_TEST_COMPILER_ID MATCHES "Clang")
# Use -D instead of definitions to please custom compile command.
list(APPEND ASAN_UNITTEST_COMMON_CFLAGS
- ${COMPILER_RT_ASAN_SHADOW_SCALE_FLAG}
-DASAN_HAS_IGNORELIST=1
-DASAN_HAS_EXCEPTIONS=1
-DASAN_UAR=0
function(add_asan_tests arch test_runtime)
cmake_parse_arguments(TEST "" "KIND" "CFLAGS" ${ARGN})
+ # The Lit files are configured once per architecture and static/dynamic
+ # selection. Each configuration expects the test binaries in a corresponding
+ # subdirectory. Generate subdirectory names based on the architecture name.
+ string(TOUPPER ${arch} ARCH_UPPER_CASE)
+ set(CONFIG_NAME ${ARCH_UPPER_CASE}${OS_NAME}Config)
+ set(CONFIG_NAME_DYNAMIC ${ARCH_UPPER_CASE}${OS_NAME}DynamicConfig)
+
# Closure to keep the values.
function(generate_asan_tests test_objects test_suite testname)
generate_compiler_rt_tests(${test_objects} ${test_suite} ${testname} ${arch}
COMPILE_DEPS ${ASAN_UNITTEST_HEADERS} ${ASAN_IGNORELIST_FILE}
- DEPS gtest asan
+ DEPS llvm_gtest asan
KIND ${TEST_KIND}
${ARGN}
)
set("${test_objects}" "${${test_objects}}" PARENT_SCOPE)
endfunction()
+ set(TARGET_LINK_FLAGS)
+ get_target_link_flags_for_arch(${arch} TARGET_LINK_FLAGS)
+
set(ASAN_INST_TEST_OBJECTS)
generate_asan_tests(ASAN_INST_TEST_OBJECTS AsanUnitTests
"Asan-${arch}${TEST_KIND}-Test"
- SUBDIR "default"
- LINK_FLAGS ${ASAN_UNITTEST_INSTRUMENTED_LINK_FLAGS}
+ SUBDIR "${CONFIG_NAME}"
+ LINK_FLAGS ${ASAN_UNITTEST_INSTRUMENTED_LINK_FLAGS} ${TARGET_LINK_FLAGS}
SOURCES ${ASAN_INST_TEST_SOURCES}
CFLAGS ${ASAN_UNITTEST_INSTRUMENTED_CFLAGS} ${TEST_CFLAGS})
set(ASAN_DYNAMIC_TEST_OBJECTS)
generate_asan_tests(ASAN_DYNAMIC_TEST_OBJECTS
AsanDynamicUnitTests "${dynamic_test_name}"
- SUBDIR "dynamic"
+ SUBDIR "${CONFIG_NAME_DYNAMIC}"
CFLAGS ${ASAN_UNITTEST_INSTRUMENTED_CFLAGS} -D_MT -D_DLL
SOURCES ${ASAN_INST_TEST_SOURCES}
LINK_FLAGS ${ASAN_DYNAMIC_UNITTEST_INSTRUMENTED_LINK_FLAGS}
# Otherwise, reuse ASAN_INST_TEST_OBJECTS.
add_compiler_rt_test(AsanDynamicUnitTests "${dynamic_test_name}" "${arch}"
- SUBDIR "dynamic"
+ SUBDIR "${CONFIG_NAME_DYNAMIC}"
OBJECTS ${ASAN_INST_TEST_OBJECTS}
DEPS asan ${ASAN_INST_TEST_OBJECTS}
- LINK_FLAGS ${ASAN_DYNAMIC_UNITTEST_INSTRUMENTED_LINK_FLAGS}
+ LINK_FLAGS ${ASAN_DYNAMIC_UNITTEST_INSTRUMENTED_LINK_FLAGS} ${TARGET_LINK_FLAGS}
)
endif()
endif()
set(ASAN_NOINST_TEST_OBJECTS)
generate_asan_tests(ASAN_NOINST_TEST_OBJECTS
AsanUnitTests "Asan-${arch}${TEST_KIND}-Noinst-Test"
- SUBDIR "default"
+ SUBDIR "${CONFIG_NAME}"
CFLAGS ${ASAN_UNITTEST_COMMON_CFLAGS}
- LINK_FLAGS ${ASAN_UNITTEST_NOINST_LINK_FLAGS}
+ LINK_FLAGS ${ASAN_UNITTEST_NOINST_LINK_FLAGS} ${TARGET_LINK_FLAGS}
SOURCES ${ASAN_NOINST_TEST_SOURCES}
RUNTIME ${test_runtime})
set(ASAN_BENCHMARK_OBJECTS)
generate_asan_tests(ASAN_BENCHMARK_OBJECTS
AsanBenchmarks "Asan-${arch}${TEST_KIND}-Benchmark"
- SUBDIR "default"
+ SUBDIR "${CONFIG_NAME}"
CFLAGS ${ASAN_UNITTEST_INSTRUMENTED_CFLAGS}
SOURCES ${ASAN_BENCHMARKS_SOURCES}
- LINK_FLAGS ${ASAN_UNITTEST_INSTRUMENTED_LINK_FLAGS})
+ LINK_FLAGS ${ASAN_UNITTEST_INSTRUMENTED_LINK_FLAGS} ${TARGET_LINK_FLAGS})
endfunction()
if(COMPILER_RT_CAN_EXECUTE_TESTS AND NOT ANDROID)
set(ASAN_TEST_RUNTIME_OBJECTS
$<TARGET_OBJECTS:RTAsan.${arch}>
$<TARGET_OBJECTS:RTAsan_cxx.${arch}>
+ $<TARGET_OBJECTS:RTAsan_static.${arch}>
$<TARGET_OBJECTS:RTInterception.${arch}>
$<TARGET_OBJECTS:RTSanitizerCommon.${arch}>
$<TARGET_OBJECTS:RTSanitizerCommonLibc.${arch}>
# Test w/o ASan instrumentation. Link it with ASan statically.
add_executable(AsanNoinstTest # FIXME: .arch?
$<TARGET_OBJECTS:RTAsan.${arch}>
+ $<TARGET_OBJECTS:RTAsan_static.${arch}>
$<TARGET_OBJECTS:RTInterception.${arch}>
$<TARGET_OBJECTS:RTSanitizerCommon.${arch}>
$<TARGET_OBJECTS:RTSanitizerCommonLibc.${arch}>
TEST(AddressSanitizerInterface, GetHeapSizeTest) {
// ASan allocator does not keep huge chunks in free list, but unmaps them.
// The chunk should be greater than the quarantine size,
- // otherwise it will be stuck in quarantine instead of being unmaped.
+ // otherwise it will be stuck in quarantine instead of being unmapped.
static const size_t kLargeMallocSize = (1 << 28) + 1; // 256M
free(Ident(malloc(kLargeMallocSize))); // Drain quarantine.
size_t old_heap_size = __sanitizer_get_heap_size();
#define BAD_ACCESS(ptr, offset) \
EXPECT_TRUE(__asan_address_is_poisoned(ptr + offset))
-#if !defined(ASAN_SHADOW_SCALE) || ASAN_SHADOW_SCALE == 3
static const char* kUseAfterPoisonErrorMessage = "use-after-poison";
TEST(AddressSanitizerInterface, SimplePoisonMemoryRegionTest) {
BAD_ACCESS(array, 96);
free(array);
}
-#endif // !defined(ASAN_SHADOW_SCALE) || ASAN_SHADOW_SCALE == 3
TEST(AddressSanitizerInterface, PushAndPopWithPoisoningTest) {
// Vector of capacity 20
__asan_poison_memory_region(array, sizeof(array));
BAD_ACCESS(array, 20);
__asan_handle_no_return();
+ // Fake stack does not need to be unpoisoned.
+ if (__asan_get_current_fake_stack())
+ return;
// It unpoisons the whole thread stack.
GOOD_ACCESS(array, 20);
}
__asan_set_shadow_00((uptr)buffer.data(), buffer.size());
EXPECT_EQ(std::vector<char>(buffer.size(), 0x00), buffer);
+ __asan_set_shadow_01((uptr)buffer.data(), buffer.size());
+ EXPECT_EQ(std::vector<char>(buffer.size(), 0x01), buffer);
+
+ __asan_set_shadow_02((uptr)buffer.data(), buffer.size());
+ EXPECT_EQ(std::vector<char>(buffer.size(), 0x02), buffer);
+
+ __asan_set_shadow_03((uptr)buffer.data(), buffer.size());
+ EXPECT_EQ(std::vector<char>(buffer.size(), 0x03), buffer);
+
+ __asan_set_shadow_04((uptr)buffer.data(), buffer.size());
+ EXPECT_EQ(std::vector<char>(buffer.size(), 0x04), buffer);
+
+ __asan_set_shadow_05((uptr)buffer.data(), buffer.size());
+ EXPECT_EQ(std::vector<char>(buffer.size(), 0x05), buffer);
+
+ __asan_set_shadow_06((uptr)buffer.data(), buffer.size());
+ EXPECT_EQ(std::vector<char>(buffer.size(), 0x06), buffer);
+
+ __asan_set_shadow_07((uptr)buffer.data(), buffer.size());
+ EXPECT_EQ(std::vector<char>(buffer.size(), 0x07), buffer);
+
__asan_set_shadow_f1((uptr)buffer.data(), buffer.size());
EXPECT_EQ(std::vector<char>(buffer.size(), 0xf1), buffer);
MEMSET(array + length, 0, zero);
MEMSET(array + length + 1, 0, zero);
- // try to memset bytes to the right of array
+ // try to memset bytes after array
EXPECT_DEATH(MEMSET(array, 0, size + 1),
RightOOBWriteMessage(0));
EXPECT_DEATH(MEMSET((char*)(array + length) - 1, element, 6),
RightOOBWriteMessage(0));
EXPECT_DEATH(MEMSET(array + 1, element, size + sizeof(T)),
RightOOBWriteMessage(0));
- // whole interval is to the right
+ // whole interval is after
EXPECT_DEATH(MEMSET(array + length + 1, 0, 10),
RightOOBWriteMessage(sizeof(T)));
- // try to memset bytes to the left of array
+ // try to memset bytes before array
EXPECT_DEATH(MEMSET((char*)array - 1, element, size),
LeftOOBWriteMessage(1));
EXPECT_DEATH(MEMSET((char*)array - 5, 0, 6),
EXPECT_DEATH(memset(array - 5, element, size + 5 * sizeof(T)),
LeftOOBWriteMessage(5 * sizeof(T)));
}
- // whole interval is to the left
+ // whole interval is before
EXPECT_DEATH(MEMSET(array - 2, 0, sizeof(T)),
LeftOOBWriteMessage(2 * sizeof(T)));
- // try to memset bytes both to the left & to the right
+ // try to memset bytes both before & after
EXPECT_DEATH(MEMSET((char*)array - 2, element, size + 4),
LeftOOBWriteMessage(2));
// fprintf(stderr, " large oob memset: %p %p %zd\n", x1, x2, size);
// Do a memset on x1 with huge out-of-bound access that will end up in x2.
EXPECT_DEATH(Ident(memset)(x1, 0, size * 2),
- "is located 0 bytes to the right");
+ "is located 0 bytes after");
delete [] x1;
delete [] x2;
return;
M::transfer(dest, src - 1, zero);
M::transfer(dest, src, zero);
- // try to change mem to the right of dest
+ // try to change mem after dest
EXPECT_DEATH(M::transfer(dest + 1, src, size),
RightOOBWriteMessage(0));
EXPECT_DEATH(M::transfer((char*)(dest + length) - 1, src, 5),
RightOOBWriteMessage(0));
- // try to change mem to the left of dest
+ // try to change mem before dest
EXPECT_DEATH(M::transfer(dest - 2, src, size),
LeftOOBWriteMessage(2 * sizeof(T)));
EXPECT_DEATH(M::transfer((char*)dest - 3, src, 4),
LeftOOBWriteMessage(3));
- // try to access mem to the right of src
+ // try to access mem after src
EXPECT_DEATH(M::transfer(dest, src + 2, size),
RightOOBReadMessage(0));
EXPECT_DEATH(M::transfer(dest, (char*)(src + length) - 3, 6),
RightOOBReadMessage(0));
- // try to access mem to the left of src
+ // try to access mem before src
EXPECT_DEATH(M::transfer(dest, src - 1, size),
LeftOOBReadMessage(sizeof(T)));
EXPECT_DEATH(M::transfer(dest, (char*)src - 6, 7),
// This test file should be compiled w/o asan instrumentation.
//===----------------------------------------------------------------------===//
-#include "asan_allocator.h"
-#include "asan_internal.h"
-#include "asan_mapping.h"
-#include "asan_test_utils.h"
-#include <sanitizer/allocator_interface.h>
-
#include <assert.h>
+#include <sanitizer/allocator_interface.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h> // for memset()
+
#include <algorithm>
-#include <vector>
#include <limits>
+#include <vector>
+
+#include "asan_allocator.h"
+#include "asan_internal.h"
+#include "asan_mapping.h"
+#include "asan_test_utils.h"
using namespace __sanitizer;
}
// Test __asan_load1 & friends.
-TEST(AddressSanitizer, LoadStoreCallbacks) {
- typedef void (*CB)(uptr p);
- CB cb[2][5] = {
- {
- __asan_load1, __asan_load2, __asan_load4, __asan_load8, __asan_load16,
- }, {
- __asan_store1, __asan_store2, __asan_store4, __asan_store8,
- __asan_store16,
- }
- };
-
+typedef void (*CB)(uptr p);
+static void TestLoadStoreCallbacks(CB cb[2][5]) {
uptr buggy_ptr;
__asan_test_only_reported_buggy_pointer = &buggy_ptr;
}
__asan_test_only_reported_buggy_pointer = 0;
}
+
+TEST(AddressSanitizer, LoadStoreCallbacks) {
+ CB cb[2][5] = {{
+ __asan_load1,
+ __asan_load2,
+ __asan_load4,
+ __asan_load8,
+ __asan_load16,
+ },
+ {
+ __asan_store1,
+ __asan_store2,
+ __asan_store4,
+ __asan_store8,
+ __asan_store16,
+ }};
+ TestLoadStoreCallbacks(cb);
+}
+
+#if defined(__x86_64__) && \
+ !(defined(SANITIZER_APPLE) || defined(SANITIZER_WINDOWS))
+// clang-format off
+
+#define CALL_ASAN_MEMORY_ACCESS_CALLBACK_ADD(s, reg, op) \
+ void CallAsanMemoryAccessAdd##reg##op##s(uptr address) { \
+ asm("push %%" #reg " \n" \
+ "mov %[x], %%" #reg " \n" \
+ "call __asan_check_" #op "_add_" #s "_" #reg "\n" \
+ "pop %%" #reg " \n" \
+ : \
+ : [x] "r"(address) \
+ : "r8", "rdi"); \
+ }
+
+#define TEST_ASAN_MEMORY_ACCESS_CALLBACKS_ADD(reg) \
+ CALL_ASAN_MEMORY_ACCESS_CALLBACK_ADD(1, reg, load) \
+ CALL_ASAN_MEMORY_ACCESS_CALLBACK_ADD(1, reg, store) \
+ CALL_ASAN_MEMORY_ACCESS_CALLBACK_ADD(2, reg, load) \
+ CALL_ASAN_MEMORY_ACCESS_CALLBACK_ADD(2, reg, store) \
+ CALL_ASAN_MEMORY_ACCESS_CALLBACK_ADD(4, reg, load) \
+ CALL_ASAN_MEMORY_ACCESS_CALLBACK_ADD(4, reg, store) \
+ CALL_ASAN_MEMORY_ACCESS_CALLBACK_ADD(8, reg, load) \
+ CALL_ASAN_MEMORY_ACCESS_CALLBACK_ADD(8, reg, store) \
+ CALL_ASAN_MEMORY_ACCESS_CALLBACK_ADD(16, reg, load) \
+ CALL_ASAN_MEMORY_ACCESS_CALLBACK_ADD(16, reg, store) \
+ \
+ TEST(AddressSanitizer, LoadStoreCallbacksAddX86##reg) { \
+ CB cb[2][5] = {{ \
+ CallAsanMemoryAccessAdd##reg##load1, \
+ CallAsanMemoryAccessAdd##reg##load2, \
+ CallAsanMemoryAccessAdd##reg##load4, \
+ CallAsanMemoryAccessAdd##reg##load8, \
+ CallAsanMemoryAccessAdd##reg##load16, \
+ }, \
+ { \
+ CallAsanMemoryAccessAdd##reg##store1, \
+ CallAsanMemoryAccessAdd##reg##store2, \
+ CallAsanMemoryAccessAdd##reg##store4, \
+ CallAsanMemoryAccessAdd##reg##store8, \
+ CallAsanMemoryAccessAdd##reg##store16, \
+ }}; \
+ TestLoadStoreCallbacks(cb); \
+ }
+
+// Instantiate all but R10 and R11 callbacks. We are using PLTSafe class with
+// the intrinsic, which guarantees that the code generation will never emit
+// R10 or R11 callbacks.
+TEST_ASAN_MEMORY_ACCESS_CALLBACKS_ADD(RAX)
+TEST_ASAN_MEMORY_ACCESS_CALLBACKS_ADD(RBX)
+TEST_ASAN_MEMORY_ACCESS_CALLBACKS_ADD(RCX)
+TEST_ASAN_MEMORY_ACCESS_CALLBACKS_ADD(RDX)
+TEST_ASAN_MEMORY_ACCESS_CALLBACKS_ADD(RSI)
+TEST_ASAN_MEMORY_ACCESS_CALLBACKS_ADD(RDI)
+TEST_ASAN_MEMORY_ACCESS_CALLBACKS_ADD(RBP)
+TEST_ASAN_MEMORY_ACCESS_CALLBACKS_ADD(R8)
+TEST_ASAN_MEMORY_ACCESS_CALLBACKS_ADD(R9)
+TEST_ASAN_MEMORY_ACCESS_CALLBACKS_ADD(R12)
+TEST_ASAN_MEMORY_ACCESS_CALLBACKS_ADD(R13)
+TEST_ASAN_MEMORY_ACCESS_CALLBACKS_ADD(R14)
+TEST_ASAN_MEMORY_ACCESS_CALLBACKS_ADD(R15)
+
+// clang-format on
+#endif
static std::string GetLeftOOBMessage(int off) {
char str[100];
- sprintf(str, "is located.*%d byte.*to the left", off);
+ sprintf(str, "is located.*%d byte.*before", off);
return str;
}
char str[100];
#if !defined(_WIN32)
// FIXME: Fix PR42868 and remove SEGV match.
- sprintf(str, "is located.*%d byte.*to the right|SEGV", off);
+ sprintf(str, "is located.*%d byte.*after|SEGV", off);
#else
// `|` doesn't work in googletest's regexes on Windows,
// see googletest/docs/advanced.md#regular-expression-syntax
// But it's not needed on Windows anyways.
- sprintf(str, "is located.*%d byte.*to the right", off);
+ sprintf(str, "is located.*%d byte.*after", off);
#endif
return str;
}
} // namespace
// Input to a test is a zero-terminated string str with given length
-// Accesses to the bytes to the left and to the right of str
+// Accesses to the bytes before and after str
// are presumed to produce OOB errors
void StrLenOOBTestTemplate(char *str, size_t length, OOBKind oob_kind) {
// Normal strlen calls
}
// Arg of strlen is not malloced, OOB access
if (oob_kind != OOBKind::Global) {
- // We don't insert RedZones to the left of global variables
+ // We don't insert RedZones before global variables
EXPECT_DEATH(Ident(strlen(str - 1)), LeftOOBReadMessage(oob_kind, 1));
EXPECT_DEATH(Ident(strlen(str - 5)), LeftOOBReadMessage(oob_kind, 5));
}
static void TestLargeMalloc(size_t size) {
char buff[1024];
- sprintf(buff, "is located 1 bytes to the left of %lu-byte", (long)size);
+ sprintf(buff, "is located 1 bytes before %lu-byte", (long)size);
EXPECT_DEATH(Ident((char*)malloc(size))[-1] = 0, buff);
}
if (SANITIZER_WORDSIZE != 64 || ASAN_AVOID_EXPENSIVE_TESTS) return;
size_t n_megs = 4100;
EXPECT_DEATH(Ident((char*)malloc(n_megs << 20))[-1] = 0,
- "is located 1 bytes to the left|"
+ "is located 1 bytes before|"
"AddressSanitizer failed to allocate");
}
#endif
for (int align = 16; align <= (1 << 23); align *= 2) {
size_t size = align * 5;
EXPECT_DEATH(MemalignRun(align, size, -1),
- "is located 1 bytes to the left");
+ "is located 1 bytes before");
EXPECT_DEATH(MemalignRun(align, size, size + 1),
- "is located 1 bytes to the right");
+ "is located 1 bytes after");
}
}
#endif // SANITIZER_TEST_HAS_MEMALIGN
#if !defined(__ANDROID__) && !defined(__arm__) && !defined(__aarch64__) && \
!defined(__mips__) && !defined(__mips64) && !defined(__s390__) && \
- !defined(__riscv)
+ !defined(__riscv) && !defined(__loongarch__)
NOINLINE void BuiltinLongJmpFunc1(jmp_buf buf) {
// create three red zones for these two stack objects.
int a;
}
}
#endif // !defined(__ANDROID__) && !defined(__arm__) &&
- // !defined(__aarch64__) && !defined(__mips__)
- // !defined(__mips64) && !defined(__s390__)
- // !defined(__riscv)
+ // !defined(__aarch64__) && !defined(__mips__) &&
+ // !defined(__mips64) && !defined(__s390__) &&
+ // !defined(__riscv) && !defined(__loongarch__)
TEST(AddressSanitizer, UnderscopeLongJmpTest) {
static jmp_buf buf;
EXPECT_DEATH(_mm_store_si128((__m128i*)p, value_wide),
"WRITE of size 16");
EXPECT_DEATH(_mm_store_si128((__m128i*)p, value_wide),
- "located 0 bytes to the right of 12-byte");
+ "located 0 bytes after 12-byte");
free(a);
}
#endif
#if !GTEST_USES_SIMPLE_RE
"buffer-overflow.*%s.*"
#endif
- "located %d bytes to the right",
+ "located %d bytes after",
#if !GTEST_USES_SIMPLE_RE
is_write ? "WRITE" : "READ",
#endif
#if !GTEST_USES_SIMPLE_RE
ASAN_PCRE_DOTALL "%s.*"
#endif
- "located %d bytes to the left",
+ "located %d bytes before",
#if !GTEST_USES_SIMPLE_RE
is_write ? "WRITE" : "READ",
#endif
std::string LeftOOBAccessMessage(int oob_distance) {
assert(oob_distance > 0);
char expected_str[100];
- sprintf(expected_str, "located %d bytes to the left", oob_distance);
+ sprintf(expected_str, "located %d bytes before", oob_distance);
return std::string(expected_str);
}
EXPECT_DEATH(READ_N_BYTES, \
ASAN_PCRE_DOTALL \
"AddressSanitizer: heap-buffer-overflow" \
- ".* is located 0 bytes to the right of 10-byte region"); \
+ ".* is located 0 bytes after 10-byte region"); \
close(fd); \
delete [] x; \
glob5[Ident(4)] = 0;
EXPECT_DEATH(glob5[Ident(5)] = 0,
- "0 bytes to the right of global variable.*glob5.* size 5");
+ "0 bytes after global variable.*glob5.* size 5");
EXPECT_DEATH(glob5[Ident(5+6)] = 0,
- "6 bytes to the right of global variable.*glob5.* size 5");
+ "6 bytes after global variable.*glob5.* size 5");
Ident(static110); // avoid optimizations
static110[Ident(0)] = 0;
static110[Ident(109)] = 0;
EXPECT_DEATH(static110[Ident(110)] = 0,
- "0 bytes to the right of global variable");
+ "0 bytes after global variable");
EXPECT_DEATH(static110[Ident(110+7)] = 0,
- "7 bytes to the right of global variable");
+ "7 bytes after global variable");
Ident(func_static15); // avoid optimizations
func_static15[Ident(0)] = 0;
EXPECT_DEATH(func_static15[Ident(15)] = 0,
- "0 bytes to the right of global variable");
+ "0 bytes after global variable");
EXPECT_DEATH(func_static15[Ident(15 + 9)] = 0,
- "9 bytes to the right of global variable");
+ "9 bytes after global variable");
Ident(fs1);
Ident(fs2);
// We don't create left redzones, so this is not 100% guaranteed to fail.
// But most likely will.
- EXPECT_DEATH(fs2[Ident(-1)] = 0, "is located.*of global variable");
+ EXPECT_DEATH(fs2[Ident(-1)] = 0, "is located.* global variable");
EXPECT_DEATH(Ident(Ident(ConstGlob)[8]),
- "is located 1 bytes to the right of .*ConstGlob");
+ "is located 1 bytes after .*ConstGlob");
EXPECT_DEATH(Ident(Ident(StaticConstGlob)[5]),
- "is located 2 bytes to the right of .*StaticConstGlob");
+ "is located 2 bytes after .*StaticConstGlob");
// call stuff from another file.
GlobalsTest(0);
// Default ASAN_OPTIONS for the unit tests.
extern "C" const char* __asan_default_options() {
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
// On Darwin, we default to `abort_on_error=1`, which would make tests run
// much slower. Let's override this and run lit tests with 'abort_on_error=0'
// and make sure we do not overwhelm the syslog while testing. Also, let's
#endif
}
-namespace __sanitizer {
-bool ReexecDisabled() {
-#if __has_feature(address_sanitizer) && SANITIZER_MAC
- // Allow re-exec in instrumented unit tests on Darwin. Technically, we only
- // need this for 10.10 and below, where re-exec is required for the
- // interceptors to work, but to avoid duplicating the version detection logic,
- // let's just allow re-exec for all Darwin versions. On newer OS versions,
- // returning 'false' doesn't do anything anyway, because we don't re-exec.
- return false;
-#else
- return true;
-#endif
-}
-} // namespace __sanitizer
-
int main(int argc, char **argv) {
testing::GTEST_FLAG(death_test_style) = "threadsafe";
testing::InitGoogleTest(&argc, argv);
___asan_default_suppressions
___asan_on_error
___asan_set_shadow_00
+___asan_set_shadow_01
+___asan_set_shadow_02
+___asan_set_shadow_03
+___asan_set_shadow_04
+___asan_set_shadow_05
+___asan_set_shadow_06
+___asan_set_shadow_07
___asan_set_shadow_f1
___asan_set_shadow_f2
___asan_set_shadow_f3
if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
cmake_minimum_required(VERSION 3.13.4)
+ if ("${CMAKE_VERSION}" VERSION_LESS "3.20.0")
+ message(WARNING
+ "Your CMake version is ${CMAKE_VERSION}. Starting with LLVM 17.0.0, the "
+ "minimum version of CMake required to build LLVM will become 3.20.0, and "
+ "using an older CMake will become an error. Please upgrade your CMake to "
+ "at least 3.20.0 now to avoid issues in the future!")
+ endif()
set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
project(CompilerRTBuiltins C ASM)
set(COMPILER_RT_STANDALONE_BUILD TRUE)
set(COMPILER_RT_BUILTINS_STANDALONE_BUILD TRUE)
+
+ set(COMPILER_RT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../..")
+
+ set(LLVM_COMMON_CMAKE_UTILS "${COMPILER_RT_SOURCE_DIR}/../cmake")
+
+ # Add path for custom modules
list(INSERT CMAKE_MODULE_PATH 0
- "${CMAKE_SOURCE_DIR}/../../cmake"
- "${CMAKE_SOURCE_DIR}/../../cmake/Modules")
+ "${COMPILER_RT_SOURCE_DIR}/cmake"
+ "${COMPILER_RT_SOURCE_DIR}/cmake/Modules"
+ "${LLVM_COMMON_CMAKE_UTILS}"
+ "${LLVM_COMMON_CMAKE_UTILS}/Modules"
+ )
+
include(base-config-ix)
include(CompilerRTUtils)
- load_llvm_config()
+ if (NOT LLVM_RUNTIMES_BUILD)
+ load_llvm_config()
+ endif()
construct_compiler_rt_default_triple()
+ include(SetPlatformToolchainTools)
if(APPLE)
include(CompilerRTDarwinUtils)
endif()
include(UseLibtool)
endif()
include(AddCompilerRT)
-
- if(${CMAKE_SYSTEM_NAME} MATCHES "AIX")
- set(CMAKE_C_ARCHIVE_CREATE "<CMAKE_AR> -X32_64 qc <TARGET> <LINK_FLAGS> <OBJECTS>")
- set(CMAKE_CXX_ARCHIVE_CREATE "<CMAKE_AR> -X32_64 qc <TARGET> <LINK_FLAGS> <OBJECTS>")
- set(CMAKE_C_ARCHIVE_APPEND "<CMAKE_AR> -X32_64 q <TARGET> <LINK_FLAGS> <OBJECTS>")
- set(CMAKE_CXX_ARCHIVE_APPEND "<CMAKE_AR> -X32_64 q <TARGET> <LINK_FLAGS> <OBJECTS>")
- set(CMAKE_C_ARCHIVE_FINISH "<CMAKE_RANLIB> -X32_64 <TARGET>")
- set(CMAKE_CXX_ARCHIVE_FINISH "<CMAKE_RANLIB> -X32_64 <TARGET>")
- endif()
endif()
if (COMPILER_RT_STANDALONE_BUILD)
endif()
include(builtin-config-ix)
+include(CMakePushCheckState)
if(${CMAKE_SYSTEM_NAME} MATCHES "AIX")
include(CompilerRTAIXUtils)
umodti3.c
)
+# We only build BF16 files when "__bf16" is available.
+set(BF16_SOURCES
+ truncdfbf2.c
+ truncsfbf2.c
+)
+
# TODO: Several "tf" files (and divtc3.c, but not multc3.c) are in
# GENERIC_SOURCES instead of here.
set(GENERIC_TF_SOURCES
set(i386_SOURCES ${GENERIC_SOURCES} ${x86_ARCH_SOURCES})
endif () # if (NOT MSVC)
-set(arm_SOURCES
+
+# builtin support for Targets that have Arm state or have Thumb2
+set(arm_or_thumb2_base_SOURCES
arm/fp_mode.c
arm/bswapdi2.S
arm/bswapsi2.S
arm/divmodsi4.S
arm/divsi3.S
arm/modsi3.S
+ arm/udivmodsi4.S
+ arm/udivsi3.S
+ arm/umodsi3.S
+ ${GENERIC_SOURCES}
+)
+
+set(arm_sync_SOURCES
arm/sync_fetch_and_add_4.S
arm/sync_fetch_and_add_8.S
arm/sync_fetch_and_and_4.S
arm/sync_fetch_and_umin_8.S
arm/sync_fetch_and_xor_4.S
arm/sync_fetch_and_xor_8.S
- arm/udivmodsi4.S
- arm/udivsi3.S
- arm/umodsi3.S
- ${GENERIC_SOURCES}
)
-set(thumb1_SOURCES
+# builtin support for Thumb-only targets with very limited Thumb2 technology,
+# such as v6-m and v8-m.baseline
+set(thumb1_base_SOURCES
arm/divsi3.S
arm/udivsi3.S
arm/comparesf2.S
set(arm_Thumb1_icache_SOURCES
arm/sync_synchronize.S
)
+
+# thumb1 calling into Arm to cover support
set(arm_Thumb1_SOURCES
${arm_Thumb1_JT_SOURCES}
${arm_Thumb1_SjLj_EH_SOURCES}
${arm_Thumb1_icache_SOURCES}
)
+# base functionality for Arm Targets prior to Arm v7-a and Armv6-m such as v6,
+# v5t, v4t
+set(arm_min_SOURCES
+ ${arm_or_thumb2_base_SOURCES}
+ ${arm_EABI_SOURCES}
+)
+
if(MINGW)
set(arm_SOURCES
arm/aeabi_idivmod.S
arm/aeabi_uidivmod.S
arm/aeabi_uldivmod.S
arm/chkstk.S
- mingw_fixfloat.c
- ${arm_SOURCES}
+ ${arm_or_thumb2_base_SOURCES}
+ ${arm_sync_SOURCES}
+ )
+
+ set(thumb1_SOURCES
+ ${thumb1_base_SOURCES}
)
elseif(NOT WIN32)
# TODO the EABI sources should only be added to EABI targets
set(arm_SOURCES
- ${arm_SOURCES}
+ ${arm_or_thumb2_base_SOURCES}
+ ${arm_sync_SOURCES}
${arm_EABI_SOURCES}
${arm_Thumb1_SOURCES}
)
set(thumb1_SOURCES
- ${thumb1_SOURCES}
+ ${thumb1_base_SOURCES}
${arm_EABI_SOURCES}
)
endif()
)
endif()
+set(armv4t_SOURCES ${arm_min_SOURCES})
+set(armv5te_SOURCES ${arm_min_SOURCES})
+set(armv6_SOURCES ${arm_min_SOURCES})
set(armhf_SOURCES ${arm_SOURCES})
set(armv7_SOURCES ${arm_SOURCES})
set(armv7s_SOURCES ${arm_SOURCES})
set(armv8m.main_SOURCES ${arm_SOURCES})
set(armv8.1m.main_SOURCES ${arm_SOURCES})
+# 8-bit AVR MCU
+set(avr_SOURCES
+ avr/mulqi3.S
+ avr/mulhi3.S
+ avr/exit.S
+ avr/divmodhi4.S
+ avr/udivmodhi4.S
+ avr/divmodqi4.S
+ avr/udivmodqi4.S
+ ${GENERIC_SOURCES}
+)
+
# hexagon arch
set(hexagon_SOURCES
hexagon/common_entry_exit_abi1.S
${GENERIC_TF_SOURCES}
)
+set(loongarch_SOURCES
+ loongarch/fp_mode.c
+ ${GENERIC_SOURCES}
+ ${GENERIC_TF_SOURCES}
+)
+set(loongarch64_SOURCES
+ ${loongarch_SOURCES}
+)
set(mips_SOURCES ${GENERIC_SOURCES})
set(mipsel_SOURCES ${mips_SOURCES})
set(powerpc_SOURCES ${GENERIC_SOURCES})
+set(powerpcspe_SOURCES ${GENERIC_SOURCES})
+
set(powerpc64_SOURCES
ppc/divtc3.c
ppc/fixtfdi.c
set(powerpc64le_SOURCES ${powerpc64_SOURCES})
set(riscv_SOURCES
+ riscv/fp_mode.c
riscv/save.S
riscv/restore.S
${GENERIC_SOURCES}
darwin_add_builtin_libraries(${BUILTIN_SUPPORTED_OS})
else ()
set(BUILTIN_CFLAGS "")
+ add_security_warnings(BUILTIN_CFLAGS 0)
- append_list_if(COMPILER_RT_HAS_FLOAT16 -DCOMPILER_RT_HAS_FLOAT16 BUILTIN_CFLAGS)
+ if (COMPILER_RT_HAS_FCF_PROTECTION_FLAG)
+ append_list_if(COMPILER_RT_ENABLE_CET -fcf-protection=full BUILTIN_CFLAGS)
+ endif()
append_list_if(COMPILER_RT_HAS_STD_C11_FLAG -std=c11 BUILTIN_CFLAGS)
append_list_if(COMPILER_RT_HAS_VISIBILITY_HIDDEN_FLAG VISIBILITY_HIDDEN BUILTIN_DEFS)
endif()
+ if(COMPILER_RT_DISABLE_AARCH64_FMV)
+ list(APPEND BUILTIN_DEFS DISABLE_AARCH64_FMV)
+ endif()
+
append_list_if(COMPILER_RT_HAS_ASM_LSE HAS_ASM_LSE BUILTIN_DEFS)
foreach (arch ${BUILTIN_SUPPORTED_ARCH})
if (CAN_TARGET_${arch})
+ cmake_push_check_state()
+ # TODO: we should probably make most of the checks in builtin-config depend on the target flags.
+ message(STATUS "Performing additional configure checks with target flags: ${TARGET_${arch}_CFLAGS}")
set(BUILTIN_CFLAGS_${arch} ${BUILTIN_CFLAGS})
+ list(APPEND CMAKE_REQUIRED_FLAGS ${TARGET_${arch}_CFLAGS} ${BUILTIN_CFLAGS_${arch}})
# For ARM archs, exclude any VFP builtins if VFP is not supported
if (${arch} MATCHES "^(arm|armhf|armv7|armv7s|armv7k|armv7m|armv7em|armv8m.main|armv8.1m.main)$")
string(REPLACE ";" " " _TARGET_${arch}_CFLAGS "${TARGET_${arch}_CFLAGS}")
SOURCE "#if !(__ARM_FP & 0x8)
#error No double-precision support!
#endif
- int main() { return 0; }")
+ int main(void) { return 0; }")
if(NOT COMPILER_RT_HAS_${arch}_VFP_DP)
list(REMOVE_ITEM ${arch}_SOURCES ${arm_Thumb1_VFPv2_DP_SOURCES})
endif()
endif()
endif()
+ check_c_source_compiles("_Float16 foo(_Float16 x) { return x; }"
+ COMPILER_RT_HAS_${arch}_FLOAT16)
+ append_list_if(COMPILER_RT_HAS_${arch}_FLOAT16 -DCOMPILER_RT_HAS_FLOAT16 BUILTIN_CFLAGS_${arch})
+ check_c_source_compiles("__bf16 foo(__bf16 x) { return x; }"
+ COMPILER_RT_HAS_${arch}_BFLOAT16)
+ # Build BF16 files only when "__bf16" is available.
+ if(COMPILER_RT_HAS_${arch}_BFLOAT16)
+ list(APPEND ${arch}_SOURCES ${BF16_SOURCES})
+ endif()
# Remove a generic C builtin when an arch-specific builtin is specified.
filter_builtin_sources(${arch}_SOURCES ${arch})
DEFS ${BUILTIN_DEFS}
CFLAGS ${BUILTIN_CFLAGS_${arch}}
PARENT_TARGET builtins)
+ cmake_pop_check_state()
endif ()
endforeach ()
endif ()
# archive, i.e., libatomic.a. Once cmake adds support of such usage for AIX,
# this ad-hoc part can be removed.
if(${CMAKE_SYSTEM_NAME} MATCHES "AIX")
- archive_aix_libatomic(clang_rt.atomic
+ archive_aix_libatomic(clang_rt.atomic libatomic
ARCHS ${BUILTIN_SUPPORTED_ARCH}
PARENT_TARGET builtins-standalone-atomic)
endif()
// There is no C interface to the *_vfp_d8_d15_regs functions. There are
// called in the prolog and epilog of Thumb1 functions. When the C++ ABI use
-// SJLJ for exceptions, each function with a catch clause or destuctors needs
-// to save and restore all registers in it prolog and epliog. But there is
+// SJLJ for exceptions, each function with a catch clause or destructors needs
+// to save and restore all registers in it prolog and epilog. But there is
// no way to access vector and high float registers from thumb1 code, so the
// compiler must add call outs to these helper functions in the prolog and
// epilog.
float __floatsisfvfp(int a); // Appears to convert from
// int to float.
double __floatunssidfvfp(unsigned int a); // Appears to convert from
- // unisgned int to double.
+ // unsigned int to double.
float __floatunssisfvfp(unsigned int a); // Appears to convert from
- // unisgned int to float.
+ // unsigned int to float.
int __gedf2vfp(double a, double b); // Appears to return __gedf2
// (a >= b)
int __gesf2vfp(float a, float b); // Appears to return __gesf2
CRT_FE_TONEAREST;
#endif
-CRT_FE_ROUND_MODE __fe_getround() {
+CRT_FE_ROUND_MODE __fe_getround(void) {
#ifdef __ARM_FP
uint64_t fpcr;
__asm__ __volatile__("mrs %0, fpcr" : "=r" (fpcr));
#endif
}
-int __fe_raise_inexact() {
+int __fe_raise_inexact(void) {
#ifdef __ARM_FP
uint64_t fpsr;
__asm__ __volatile__("mrs %0, fpsr" : "=r" (fpsr));
NOT_HERE_BEFORE_10_6(__umoddi3)
NOT_HERE_BEFORE_10_6(__umodti3)
-#if __ppc__
+#if __powerpc__
NOT_HERE_BEFORE_10_6(__gcc_qadd)
NOT_HERE_BEFORE_10_6(__gcc_qdiv)
NOT_HERE_BEFORE_10_6(__gcc_qmul)
NOT_HERE_BEFORE_10_6(__gcc_qsub)
NOT_HERE_BEFORE_10_6(__trampoline_setup)
-#endif // __ppc__
+#endif // __powerpc__
NOT_HERE_IN_10_8_AND_EARLIER(__atomic_compare_exchange)
NOT_HERE_IN_10_8_AND_EARLIER(__atomic_compare_exchange_1)
CRT_FE_TONEAREST;
#endif
-CRT_FE_ROUND_MODE __fe_getround() {
+CRT_FE_ROUND_MODE __fe_getround(void) {
#ifdef __ARM_FP
uint32_t fpscr;
__asm__ __volatile__("vmrs %0, fpscr" : "=r" (fpscr));
#endif
}
-int __fe_raise_inexact() {
+int __fe_raise_inexact(void) {
#ifdef __ARM_FP
uint32_t fpscr;
__asm__ __volatile__("vmrs %0, fpscr" : "=r" (fpscr));
#include "../assembly.h"
+#if __ARM_ARCH >= 7
+#define DMB dmb
+#elif __ARM_ARCH >= 6
+#define DMB mcr p15, #0, r0, c7, c10, #5
+#else
+#error DMB is only supported on ARMv6+
+#endif
+
#define SYNC_OP_4(op) \
.p2align 2; \
- .thumb; \
.syntax unified; \
- DEFINE_COMPILERRT_THUMB_FUNCTION(__sync_fetch_and_##op) \
- dmb; \
+ DEFINE_COMPILERRT_FUNCTION(__sync_fetch_and_##op) \
+ DMB; \
mov r12, r0; \
LOCAL_LABEL(tryatomic_##op) : ldrex r0, [r12]; \
op(r2, r0, r1); \
strex r3, r2, [r12]; \
cmp r3, #0; \
bne LOCAL_LABEL(tryatomic_##op); \
- dmb; \
+ DMB; \
bx lr
#define SYNC_OP_8(op) \
.p2align 2; \
- .thumb; \
.syntax unified; \
- DEFINE_COMPILERRT_THUMB_FUNCTION(__sync_fetch_and_##op) \
+ DEFINE_COMPILERRT_FUNCTION(__sync_fetch_and_##op) \
push {r4, r5, r6, lr}; \
- dmb; \
+ DMB; \
mov r12, r0; \
LOCAL_LABEL(tryatomic_##op) : ldrexd r0, r1, [r12]; \
op(r4, r5, r0, r1, r2, r3); \
strexd r6, r4, r5, [r12]; \
cmp r6, #0; \
bne LOCAL_LABEL(tryatomic_##op); \
- dmb; \
+ DMB; \
pop { r4, r5, r6, pc }
#define MINMAX_4(rD, rN, rM, cmp_kind) \
//
// extern float __truncdfsf2vfp(double a);
//
-// Converts double precision float to signle precision result.
+// Converts double precision float to single precision result.
// Uses Darwin calling convention where a double precision parameter is
-// passed in a R0/R1 pair and a signle precision result is returned in R0.
+// passed in a R0/R1 pair and a single precision result is returned in R0.
//
.syntax unified
.p2align 2
#ifndef COMPILERRT_ASSEMBLY_H
#define COMPILERRT_ASSEMBLY_H
+#if defined(__linux__) && defined(__CET__)
+#if __has_include(<cet.h>)
+#include <cet.h>
+#endif
+#endif
+
#if defined(__APPLE__) && defined(__aarch64__)
#define SEPARATOR %%
#else
static Lock locks[SPINLOCK_COUNT]; // initialized to OS_SPINLOCK_INIT which is 0
#else
+_Static_assert(__atomic_always_lock_free(sizeof(uintptr_t), 0),
+ "Implementation assumes lock-free pointer-size cmpxchg");
typedef _Atomic(uintptr_t) Lock;
/// Unlock a lock. This is a release operation.
__inline static void unlock(Lock *l) {
return tmp; \
}
+#define ATOMIC_RMW_NAND(n, lockfree, type) \
+ type __atomic_fetch_nand_##n(type *ptr, type val, int model) { \
+ if (lockfree(ptr)) \
+ return __c11_atomic_fetch_nand((_Atomic(type) *)ptr, val, model); \
+ Lock *l = lock_for_pointer(ptr); \
+ lock(l); \
+ type tmp = *ptr; \
+ *ptr = ~(tmp & val); \
+ unlock(l); \
+ return tmp; \
+ }
+
#define OPTIMISED_CASE(n, lockfree, type) ATOMIC_RMW(n, lockfree, type, add, +)
OPTIMISED_CASES
#undef OPTIMISED_CASE
#define OPTIMISED_CASE(n, lockfree, type) ATOMIC_RMW(n, lockfree, type, xor, ^)
OPTIMISED_CASES
#undef OPTIMISED_CASE
+// Allow build with clang without __c11_atomic_fetch_nand builtin (pre-14)
+#if __has_builtin(__c11_atomic_fetch_nand)
+#define OPTIMISED_CASE(n, lockfree, type) ATOMIC_RMW_NAND(n, lockfree, type)
+OPTIMISED_CASES
+#undef OPTIMISED_CASE
+#endif
--- /dev/null
+//===------------- divmodhi4.S - sint16 div & mod -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// As described at
+// https://gcc.gnu.org/wiki/avr-gcc#Exceptions_to_the_Calling_Convention, the
+// prototype is `struct {sint16, sint16} __divmodhi4(sint16, sint16)`.
+// The sint16 quotient is returned via R23:R22, and the sint16 remainder is
+// returned via R25:R24, while registers R21/R26/27/Rtmp and bit T in SREG
+// are clobbered.
+//
+//===----------------------------------------------------------------------===//
+
+ .text
+ .align 2
+
+#ifdef __AVR_TINY__
+ .set __tmp_reg__, 16
+#else
+ .set __tmp_reg__, 0
+#endif
+
+ .globl __divmodhi4
+ .type __divmodhi4, @function
+
+__divmodhi4:
+ bst r25, 7
+ mov __tmp_reg__, r23
+ brtc __divmodhi4_a
+ com __tmp_reg__
+ rcall __divmodhi4_b
+
+__divmodhi4_a:
+ sbrc r23, 7
+ rcall __divmodhi4_c
+ rcall __udivmodhi4 ; Call __udivmodhi4 to do real calculation.
+ sbrc __tmp_reg__, 7
+ rcall __divmodhi4_c
+ brtc __divmodhi4_exit
+
+__divmodhi4_b:
+ com r25
+ neg r24
+ sbci r25, 255
+ ret ; Return quotient via R23:R22 and remainder via R25:R24.
+
+__divmodhi4_c:
+ com r23
+ neg r22
+ sbci r23, 255
+
+__divmodhi4_exit:
+ ret ; Return quotient via R23:R22 and remainder via R25:r24.
--- /dev/null
+//===------------- divmodqi4.S - sint8 div & mod --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// As described at
+// https://gcc.gnu.org/wiki/avr-gcc#Exceptions_to_the_Calling_Convention, the
+// prototype is `struct {sint8, sint8} __divmodqi4(sint8, sint8)`.
+// The sint8 quotient is returned via R24, and the sint8 remainder is returned
+// via R25, while registers R23/Rtmp and bit T in SREG are clobbered.
+//
+//===----------------------------------------------------------------------===//
+
+ .text
+ .align 2
+
+#ifdef __AVR_TINY__
+ .set __tmp_reg__, 16
+#else
+ .set __tmp_reg__, 0
+#endif
+
+ .globl __divmodqi4
+ .type __divmodqi4, @function
+
+__divmodqi4:
+ bst r24, 7
+ mov __tmp_reg__, r24
+ eor __tmp_reg__, r22
+ sbrc r24, 7
+ neg r24
+ sbrc r22, 7
+ neg r22
+ rcall __udivmodqi4 ; Call __udivmodqi4 to do real calculation.
+ brtc __divmodqi4_1
+ neg r25
+
+__divmodqi4_1:
+ sbrc __tmp_reg__, 7
+ neg r24
+ ret ; Return quotient via R24 and remainder via R25.
--- /dev/null
+//===------------ exit.S - global terminator for AVR ----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+ .text
+ .align 2
+
+ .globl _exit
+ .type _exit, @function
+
+_exit:
+ cli ; Disable all interrupts.
+__stop_program:
+ rjmp __stop_program ; Fall into an infinite loop.
--- /dev/null
+//===------------ mulhi3.S - int16 multiplication -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// The corresponding C code is something like:
+//
+// int __mulhi3(int A, int B) {
+// int S = 0;
+// while (A != 0) {
+// if (A & 1)
+// S += B;
+// A = ((unsigned int) A) >> 1;
+// B <<= 1;
+// }
+// return S;
+// }
+//
+// __mulhi3 has special ABI, as the implementation of libgcc, R25:R24 is used
+// to return result, while Rtmp/R21/R22/R23 are clobbered.
+//
+//===----------------------------------------------------------------------===//
+
+ .text
+ .align 2
+
+#ifdef __AVR_TINY__
+ .set __tmp_reg__, 16
+ .set __zero_reg__, 17
+#else
+ .set __tmp_reg__, 0
+ .set __zero_reg__, 1
+#endif
+
+ .globl __mulhi3
+ .type __mulhi3, @function
+
+__mulhi3:
+ ; Use Rzero:Rtmp to store the result.
+ clr __tmp_reg__
+ clr __zero_reg__ ; S = 0;
+
+__mulhi3_loop:
+ clr r21
+ cp r24, r21
+ cpc r25, r21
+ breq __mulhi3_end ; while (A != 0) {
+
+ mov r21, r24
+ andi r21, 1
+ breq __mulhi3_loop_a ; if (A & 1)
+ add __tmp_reg__, r22
+ adc __zero_reg__, r23 ; S += B;
+
+__mulhi3_loop_a:
+ lsr r25
+ ror r24 ; A = ((unsigned int) A) >> 1;
+ lsl r22
+ rol r23 ; B <<= 1;
+ rjmp __mulhi3_loop ; }
+
+__mulhi3_end:
+ ; Return the result via R25:R24.
+ mov r24, __tmp_reg__
+ mov r25, __zero_reg__
+ ; Restore __zero_reg__ to 0.
+ clr __zero_reg__
+ ret ; return S;
--- /dev/null
+//===------------ mulhi3.S - int8 multiplication --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// The corresponding C code is something like:
+//
+// char __mulqi3(char A, char B) {
+// int S = 0;
+// while (A != 0) {
+// if (A & 1)
+// S += B;
+// B <<= 1;
+// A = ((unsigned char) A) >> 1;
+// }
+// return S;
+// }
+//
+// __mulqi3 has special ABI, as the implementation of libgcc, the result is
+// returned via R24, while Rtmp and R22 are clobbered.
+//
+//===----------------------------------------------------------------------===//
+
+ .text
+ .align 2
+
+#ifdef __AVR_TINY__
+ .set __tmp_reg__, 16
+#else
+ .set __tmp_reg__, 0
+#endif
+
+ .globl __mulqi3
+ .type __mulqi3, @function
+
+__mulqi3:
+ clr __tmp_reg__ ; S = 0;
+
+__mulqi3_loop:
+ cpi r24, 0
+ breq __mulqi3_end ; while (A != 0) {
+ sbrc r24, 0 ; if (A & 1)
+ add __tmp_reg__, r22 ; S += B;
+ add r22, r22 ; B <<= 1;
+ lsr r24 ; A = ((unsigned char) A) >> 1;
+ rjmp __mulqi3_loop ; }
+
+__mulqi3_end:
+ mov r24, __tmp_reg__
+ ret ; return S;
--- /dev/null
+//===------------ udivmodhi4.S - uint16 div & mod -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// As described at
+// https://gcc.gnu.org/wiki/avr-gcc#Exceptions_to_the_Calling_Convention, the
+// prototype is `struct {uint16, uint16} __udivmodhi4(uint16, uint16)`.
+// The uint16 quotient is returned via R23:R22, and the uint16 remainder is
+// returned via R25:R24, while R21/R26/R27 are clobbered.
+//
+//===----------------------------------------------------------------------===//
+
+ .text
+ .align 2
+
+ .globl __udivmodhi4
+ .type __udivmodhi4, @function
+
+__udivmodhi4:
+ sub r26, r26
+ sub r27, r27 ; Initialize the remainder to zero.
+ ldi r21, 17 ; Only loop 16 rounds for uint16.
+
+__udivmodhi4_loop:
+ adc r24, r24
+ adc r25, r25
+ dec r21
+ breq __udivmodhi4_end
+ adc r26, r26
+ adc r27, r27
+ cp r26, r22
+ cpc r27, r23 ; Compare with the divisor.
+ brcs __udivmodhi4_loop
+ sub r26, r22
+ sbc r27, r23 ; Subtract the divisor.
+ rjmp __udivmodhi4_loop
+
+__udivmodhi4_end:
+ com r24
+ com r25
+ mov r22, r24
+ mov r23, r25 ; The quotient is returned in R23:R22.
+ mov r24, r26
+ mov r25, r27 ; The remainder is returned in in R25:R24.
+ ret
--- /dev/null
+//===------------ udivmodqi4.S - uint8 div & mod --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// As described at
+// https://gcc.gnu.org/wiki/avr-gcc#Exceptions_to_the_Calling_Convention, the
+// prototype is `struct {uint8, uint8} __udivmodqi4(uint8, uint8)`.
+// The uint8 quotient is returned via R24, and the uint8 remainder is returned
+// via R25, while R23 is clobbered.
+//
+//===----------------------------------------------------------------------===//
+
+ .text
+ .align 2
+
+ .globl __udivmodqi4
+ .type __udivmodqi4, @function
+
+__udivmodqi4:
+ sub r25, r25 ; Initialize the remainder to zero.
+ ldi r23, 9 ; Only loop 8 rounds for uint8.
+
+__udivmodqi4_loop:
+ adc r24, r24
+ dec r23
+ breq __udivmodqi4_end
+ adc r25, r25
+ cp r25, r22 ; Compare with the divisor.
+ brcs __udivmodqi4_loop
+ sub r25, r22 ; Subtract the divisor.
+ rjmp __udivmodqi4_loop
+
+__udivmodqi4_end:
+ com r24 ; The uint8 quotient is returned via R24.
+ ret ; The uint8 remainder is returned via R25.
// This file is based on LLVM's lib/Support/Host.cpp.
// It implements the operating system Host concept and builtin
// __cpu_model for the compiler_rt library for x86 and
-// __aarch64_have_lse_atomics for AArch64.
+// __aarch64_have_lse_atomics, __aarch64_cpu_features for AArch64.
//
//===----------------------------------------------------------------------===//
-#if defined(HAVE_INIT_PRIORITY)
-#define CONSTRUCTOR_ATTRIBUTE __attribute__((__constructor__ 101))
-#elif __has_attribute(__constructor__)
-#define CONSTRUCTOR_ATTRIBUTE __attribute__((__constructor__))
+#ifndef __has_attribute
+#define __has_attribute(attr) 0
+#endif
+
+#if __has_attribute(constructor)
+#if __GNUC__ >= 9
+// Ordinarily init priorities below 101 are disallowed as they are reserved for the
+// implementation. However, we are the implementation, so silence the diagnostic,
+// since it doesn't apply to us.
+#pragma GCC diagnostic ignored "-Wprio-ctor-dtor"
+#endif
+// We're choosing init priority 90 to force our constructors to run before any
+// constructors in the end user application (starting at priority 101). This value
+// matches the libgcc choice for the same functions.
+#define CONSTRUCTOR_ATTRIBUTE __attribute__((constructor(90)))
#else
// FIXME: For MSVC, we should make a function pointer global in .CRT$X?? so that
// this runs during initialization.
#include <intrin.h>
#endif
-#ifndef __has_attribute
-#define __has_attribute(attr) 0
-#endif
-
enum VendorSignatures {
SIG_INTEL = 0x756e6547, // Genu
SIG_AMD = 0x68747541, // Auth
INTEL_GOLDMONT_PLUS,
INTEL_TREMONT,
AMDFAM19H,
+ ZHAOXIN_FAM7H,
+ INTEL_SIERRAFOREST,
+ INTEL_GRANDRIDGE,
CPU_TYPE_MAX
};
INTEL_COREI7_ALDERLAKE,
AMDFAM19H_ZNVER3,
INTEL_COREI7_ROCKETLAKE,
+ ZHAOXIN_FAM7H_LUJIAZUI,
+ AMDFAM19H_ZNVER4,
+ INTEL_COREI7_GRANITERAPIDS,
CPU_SUBTYPE_MAX
};
// Check motivated by bug reports for OpenSSL crashing on CPUs without CPUID
// support. Consequently, for i386, the presence of CPUID is checked first
// via the corresponding eflags bit.
-static bool isCpuIdSupported() {
+static bool isCpuIdSupported(void) {
#if defined(__GNUC__) || defined(__clang__)
#if defined(__i386__)
int __cpuid_supported;
*Subtype = INTEL_COREI7_ICELAKE_CLIENT;
break;
+ // Tigerlake:
+ case 0x8c:
+ case 0x8d:
+ CPU = "tigerlake";
+ *Type = INTEL_COREI7;
+ *Subtype = INTEL_COREI7_TIGERLAKE;
+ break;
+
+ // Alderlake:
+ case 0x97:
+ case 0x9a:
+ // Raptorlake:
+ case 0xb7:
+ // Meteorlake:
+ case 0xaa:
+ case 0xac:
+ CPU = "alderlake";
+ *Type = INTEL_COREI7;
+ *Subtype = INTEL_COREI7_ALDERLAKE;
+ break;
+
// Icelake Xeon:
case 0x6a:
case 0x6c:
*Subtype = INTEL_COREI7_ICELAKE_SERVER;
break;
+ // Emerald Rapids:
+ case 0xcf:
// Sapphire Rapids:
case 0x8f:
CPU = "sapphirerapids";
*Subtype = INTEL_COREI7_SAPPHIRERAPIDS;
break;
+ // Granite Rapids:
+ case 0xae:
+ case 0xad:
+ CPU = "graniterapids";
+ *Type = INTEL_COREI7;
+ *Subtype = INTEL_COREI7_GRANITERAPIDS;
+ break;
+
case 0x1c: // Most 45 nm Intel Atom processors
case 0x26: // 45 nm Atom Lincroft
case 0x27: // 32 nm Atom Medfield
*Type = INTEL_TREMONT;
break;
+ // Sierraforest:
+ case 0xaf:
+ CPU = "sierraforest";
+ *Type = INTEL_SIERRAFOREST;
+ break;
+
+ // Grandridge:
+ case 0xb6:
+ CPU = "grandridge";
+ *Type = INTEL_GRANDRIDGE;
+ break;
+
case 0x57:
CPU = "knl";
*Type = INTEL_KNL;
case 25:
CPU = "znver3";
*Type = AMDFAM19H;
- if (Model <= 0x0f) {
+ if (Model <= 0x0f || (Model >= 0x20 && Model <= 0x5f)) {
+ // Family 19h Models 00h-0Fh - Zen3
+ // Family 19h Models 20h-2Fh - Zen3
+ // Family 19h Models 30h-3Fh - Zen3
+ // Family 19h Models 40h-4Fh - Zen3+
+ // Family 19h Models 50h-5Fh - Zen3+
*Subtype = AMDFAM19H_ZNVER3;
- break; // 00h-0Fh: Zen3
+ break;
+ }
+ if ((Model >= 0x10 && Model <= 0x1f) ||
+ (Model >= 0x60 && Model <= 0x74) ||
+ (Model >= 0x78 && Model <= 0x7b) ||
+ (Model >= 0xA0 && Model <= 0xAf)) {
+ CPU = "znver4";
+ *Subtype = AMDFAM19H_ZNVER4;
+ break; // "znver4"
}
break;
default:
return 0;
}
#elif defined(__aarch64__)
+
+#ifndef AT_HWCAP
+#define AT_HWCAP 16
+#endif
+#ifndef HWCAP_CPUID
+#define HWCAP_CPUID (1 << 11)
+#endif
+#ifndef HWCAP_FP
+#define HWCAP_FP (1 << 0)
+#endif
+#ifndef HWCAP_ASIMD
+#define HWCAP_ASIMD (1 << 1)
+#endif
+#ifndef HWCAP_AES
+#define HWCAP_AES (1 << 3)
+#endif
+#ifndef HWCAP_PMULL
+#define HWCAP_PMULL (1 << 4)
+#endif
+#ifndef HWCAP_SHA1
+#define HWCAP_SHA1 (1 << 5)
+#endif
+#ifndef HWCAP_SHA2
+#define HWCAP_SHA2 (1 << 6)
+#endif
+#ifndef HWCAP_ATOMICS
+#define HWCAP_ATOMICS (1 << 8)
+#endif
+#ifndef HWCAP_FPHP
+#define HWCAP_FPHP (1 << 9)
+#endif
+#ifndef HWCAP_ASIMDHP
+#define HWCAP_ASIMDHP (1 << 10)
+#endif
+#ifndef HWCAP_ASIMDRDM
+#define HWCAP_ASIMDRDM (1 << 12)
+#endif
+#ifndef HWCAP_JSCVT
+#define HWCAP_JSCVT (1 << 13)
+#endif
+#ifndef HWCAP_FCMA
+#define HWCAP_FCMA (1 << 14)
+#endif
+#ifndef HWCAP_LRCPC
+#define HWCAP_LRCPC (1 << 15)
+#endif
+#ifndef HWCAP_DCPOP
+#define HWCAP_DCPOP (1 << 16)
+#endif
+#ifndef HWCAP_SHA3
+#define HWCAP_SHA3 (1 << 17)
+#endif
+#ifndef HWCAP_SM3
+#define HWCAP_SM3 (1 << 18)
+#endif
+#ifndef HWCAP_SM4
+#define HWCAP_SM4 (1 << 19)
+#endif
+#ifndef HWCAP_ASIMDDP
+#define HWCAP_ASIMDDP (1 << 20)
+#endif
+#ifndef HWCAP_SHA512
+#define HWCAP_SHA512 (1 << 21)
+#endif
+#ifndef HWCAP_SVE
+#define HWCAP_SVE (1 << 22)
+#endif
+#ifndef HWCAP_ASIMDFHM
+#define HWCAP_ASIMDFHM (1 << 23)
+#endif
+#ifndef HWCAP_DIT
+#define HWCAP_DIT (1 << 24)
+#endif
+#ifndef HWCAP_ILRCPC
+#define HWCAP_ILRCPC (1 << 26)
+#endif
+#ifndef HWCAP_FLAGM
+#define HWCAP_FLAGM (1 << 27)
+#endif
+#ifndef HWCAP_SSBS
+#define HWCAP_SSBS (1 << 28)
+#endif
+#ifndef HWCAP_SB
+#define HWCAP_SB (1 << 29)
+#endif
+
+#ifndef AT_HWCAP2
+#define AT_HWCAP2 26
+#endif
+#ifndef HWCAP2_DCPODP
+#define HWCAP2_DCPODP (1 << 0)
+#endif
+#ifndef HWCAP2_SVE2
+#define HWCAP2_SVE2 (1 << 1)
+#endif
+#ifndef HWCAP2_SVEAES
+#define HWCAP2_SVEAES (1 << 2)
+#endif
+#ifndef HWCAP2_SVEPMULL
+#define HWCAP2_SVEPMULL (1 << 3)
+#endif
+#ifndef HWCAP2_SVEBITPERM
+#define HWCAP2_SVEBITPERM (1 << 4)
+#endif
+#ifndef HWCAP2_SVESHA3
+#define HWCAP2_SVESHA3 (1 << 5)
+#endif
+#ifndef HWCAP2_SVESM4
+#define HWCAP2_SVESM4 (1 << 6)
+#endif
+#ifndef HWCAP2_FLAGM2
+#define HWCAP2_FLAGM2 (1 << 7)
+#endif
+#ifndef HWCAP2_FRINT
+#define HWCAP2_FRINT (1 << 8)
+#endif
+#ifndef HWCAP2_SVEI8MM
+#define HWCAP2_SVEI8MM (1 << 9)
+#endif
+#ifndef HWCAP2_SVEF32MM
+#define HWCAP2_SVEF32MM (1 << 10)
+#endif
+#ifndef HWCAP2_SVEF64MM
+#define HWCAP2_SVEF64MM (1 << 11)
+#endif
+#ifndef HWCAP2_SVEBF16
+#define HWCAP2_SVEBF16 (1 << 12)
+#endif
+#ifndef HWCAP2_I8MM
+#define HWCAP2_I8MM (1 << 13)
+#endif
+#ifndef HWCAP2_BF16
+#define HWCAP2_BF16 (1 << 14)
+#endif
+#ifndef HWCAP2_DGH
+#define HWCAP2_DGH (1 << 15)
+#endif
+#ifndef HWCAP2_RNG
+#define HWCAP2_RNG (1 << 16)
+#endif
+#ifndef HWCAP2_BTI
+#define HWCAP2_BTI (1 << 17)
+#endif
+#ifndef HWCAP2_MTE
+#define HWCAP2_MTE (1 << 18)
+#endif
+#ifndef HWCAP2_RPRES
+#define HWCAP2_RPRES (1 << 21)
+#endif
+#ifndef HWCAP2_MTE3
+#define HWCAP2_MTE3 (1 << 22)
+#endif
+#ifndef HWCAP2_SME
+#define HWCAP2_SME (1 << 23)
+#endif
+#ifndef HWCAP2_SME_I16I64
+#define HWCAP2_SME_I16I64 (1 << 24)
+#endif
+#ifndef HWCAP2_SME_F64F64
+#define HWCAP2_SME_F64F64 (1 << 25)
+#endif
+#ifndef HWCAP2_WFXT
+#define HWCAP2_WFXT (1UL << 31)
+#endif
+#ifndef HWCAP2_EBF16
+#define HWCAP2_EBF16 (1UL << 32)
+#endif
+#ifndef HWCAP2_SVE_EBF16
+#define HWCAP2_SVE_EBF16 (1UL << 33)
+#endif
+
// LSE support detection for out-of-line atomics
// using HWCAP and Auxiliary vector
_Bool __aarch64_have_lse_atomics
__attribute__((visibility("hidden"), nocommon));
+
#if defined(__has_include)
#if __has_include(<sys/auxv.h>)
#include <sys/auxv.h>
-#ifndef AT_HWCAP
-#define AT_HWCAP 16
-#endif
-#ifndef HWCAP_ATOMICS
-#define HWCAP_ATOMICS (1 << 8)
+#if __has_include(<asm/hwcap.h>)
+#include <asm/hwcap.h>
+
+#if defined(__ANDROID__)
+#include <string.h>
+#include <sys/system_properties.h>
+#elif defined(__Fuchsia__)
+#include <zircon/features.h>
+#include <zircon/syscalls.h>
#endif
+
+// Detect Exynos 9810 CPU
+#define IF_EXYNOS9810 \
+ char arch[PROP_VALUE_MAX]; \
+ if (__system_property_get("ro.arch", arch) > 0 && \
+ strncmp(arch, "exynos9810", sizeof("exynos9810") - 1) == 0)
+
static void CONSTRUCTOR_ATTRIBUTE init_have_lse_atomics(void) {
+#if defined(__FreeBSD__)
+ unsigned long hwcap;
+ int result = elf_aux_info(AT_HWCAP, &hwcap, sizeof hwcap);
+ __aarch64_have_lse_atomics = result == 0 && (hwcap & HWCAP_ATOMICS) != 0;
+#elif defined(__Fuchsia__)
+ // This ensures the vDSO is a direct link-time dependency of anything that
+ // needs this initializer code.
+#pragma comment(lib, "zircon")
+ uint32_t features;
+ zx_status_t status = _zx_system_get_features(ZX_FEATURE_KIND_CPU, &features);
+ __aarch64_have_lse_atomics =
+ status == ZX_OK && (features & ZX_ARM64_FEATURE_ISA_ATOMICS) != 0;
+#else
unsigned long hwcap = getauxval(AT_HWCAP);
- __aarch64_have_lse_atomics = (hwcap & HWCAP_ATOMICS) != 0;
+ _Bool result = (hwcap & HWCAP_ATOMICS) != 0;
+#if defined(__ANDROID__)
+ if (result) {
+ // Some cores in the Exynos 9810 CPU are ARMv8.2 and others are ARMv8.0;
+ // only the former support LSE atomics. However, the kernel in the
+ // initial Android 8.0 release of Galaxy S9/S9+ devices incorrectly
+ // reported the feature as being supported.
+ //
+ // The kernel appears to have been corrected to mark it unsupported as of
+ // the Android 9.0 release on those devices, and this issue has not been
+ // observed anywhere else. Thus, this workaround may be removed if
+ // compiler-rt ever drops support for Android 8.0.
+ IF_EXYNOS9810 result = false;
+ }
+#endif // defined(__ANDROID__)
+ __aarch64_have_lse_atomics = result;
+#endif // defined(__FreeBSD__)
+}
+
+#if !defined(DISABLE_AARCH64_FMV)
+// CPUFeatures must correspond to the same AArch64 features in
+// AArch64TargetParser.h
+enum CPUFeatures {
+ FEAT_RNG,
+ FEAT_FLAGM,
+ FEAT_FLAGM2,
+ FEAT_FP16FML,
+ FEAT_DOTPROD,
+ FEAT_SM4,
+ FEAT_RDM,
+ FEAT_LSE,
+ FEAT_FP,
+ FEAT_SIMD,
+ FEAT_CRC,
+ FEAT_SHA1,
+ FEAT_SHA2,
+ FEAT_SHA3,
+ FEAT_AES,
+ FEAT_PMULL,
+ FEAT_FP16,
+ FEAT_DIT,
+ FEAT_DPB,
+ FEAT_DPB2,
+ FEAT_JSCVT,
+ FEAT_FCMA,
+ FEAT_RCPC,
+ FEAT_RCPC2,
+ FEAT_FRINTTS,
+ FEAT_DGH,
+ FEAT_I8MM,
+ FEAT_BF16,
+ FEAT_EBF16,
+ FEAT_RPRES,
+ FEAT_SVE,
+ FEAT_SVE_BF16,
+ FEAT_SVE_EBF16,
+ FEAT_SVE_I8MM,
+ FEAT_SVE_F32MM,
+ FEAT_SVE_F64MM,
+ FEAT_SVE2,
+ FEAT_SVE_AES,
+ FEAT_SVE_PMULL128,
+ FEAT_SVE_BITPERM,
+ FEAT_SVE_SHA3,
+ FEAT_SVE_SM4,
+ FEAT_SME,
+ FEAT_MEMTAG,
+ FEAT_MEMTAG2,
+ FEAT_MEMTAG3,
+ FEAT_SB,
+ FEAT_PREDRES,
+ FEAT_SSBS,
+ FEAT_SSBS2,
+ FEAT_BTI,
+ FEAT_LS64,
+ FEAT_LS64_V,
+ FEAT_LS64_ACCDATA,
+ FEAT_WFXT,
+ FEAT_SME_F64,
+ FEAT_SME_I64,
+ FEAT_SME2,
+ FEAT_MAX
+};
+
+// Architecture features used
+// in Function Multi Versioning
+struct {
+ unsigned long long features;
+ // As features grows new fields could be added
+} __aarch64_cpu_features __attribute__((visibility("hidden"), nocommon));
+
+void init_cpu_features_resolver(unsigned long hwcap, unsigned long hwcap2) {
+#define setCPUFeature(F) __aarch64_cpu_features.features |= 1ULL << F
+#define getCPUFeature(id, ftr) __asm__("mrs %0, " #id : "=r"(ftr))
+#define extractBits(val, start, number) \
+ (val & ((1ULL << number) - 1ULL) << start) >> start
+ if (hwcap & HWCAP_CRC32)
+ setCPUFeature(FEAT_CRC);
+ if (hwcap & HWCAP_PMULL)
+ setCPUFeature(FEAT_PMULL);
+ if (hwcap & HWCAP_FLAGM)
+ setCPUFeature(FEAT_FLAGM);
+ if (hwcap2 & HWCAP2_FLAGM2) {
+ setCPUFeature(FEAT_FLAGM);
+ setCPUFeature(FEAT_FLAGM2);
+ }
+ if (hwcap & HWCAP_SM3 && hwcap & HWCAP_SM4)
+ setCPUFeature(FEAT_SM4);
+ if (hwcap & HWCAP_ASIMDDP)
+ setCPUFeature(FEAT_DOTPROD);
+ if (hwcap & HWCAP_ASIMDFHM)
+ setCPUFeature(FEAT_FP16FML);
+ if (hwcap & HWCAP_FPHP) {
+ setCPUFeature(FEAT_FP16);
+ setCPUFeature(FEAT_FP);
+ }
+ if (hwcap & HWCAP_DIT)
+ setCPUFeature(FEAT_DIT);
+ if (hwcap & HWCAP_ASIMDRDM)
+ setCPUFeature(FEAT_RDM);
+ if (hwcap & HWCAP_ILRCPC)
+ setCPUFeature(FEAT_RCPC2);
+ if (hwcap & HWCAP_AES)
+ setCPUFeature(FEAT_AES);
+ if (hwcap & HWCAP_SHA1)
+ setCPUFeature(FEAT_SHA1);
+ if (hwcap & HWCAP_SHA2)
+ setCPUFeature(FEAT_SHA2);
+ if (hwcap & HWCAP_JSCVT)
+ setCPUFeature(FEAT_JSCVT);
+ if (hwcap & HWCAP_FCMA)
+ setCPUFeature(FEAT_FCMA);
+ if (hwcap & HWCAP_SB)
+ setCPUFeature(FEAT_SB);
+ if (hwcap & HWCAP_SSBS)
+ setCPUFeature(FEAT_SSBS2);
+ if (hwcap2 & HWCAP2_MTE) {
+ setCPUFeature(FEAT_MEMTAG);
+ setCPUFeature(FEAT_MEMTAG2);
+ }
+ if (hwcap2 & HWCAP2_MTE3) {
+ setCPUFeature(FEAT_MEMTAG);
+ setCPUFeature(FEAT_MEMTAG2);
+ setCPUFeature(FEAT_MEMTAG3);
+ }
+ if (hwcap2 & HWCAP2_SVEAES)
+ setCPUFeature(FEAT_SVE_AES);
+ if (hwcap2 & HWCAP2_SVEPMULL) {
+ setCPUFeature(FEAT_SVE_AES);
+ setCPUFeature(FEAT_SVE_PMULL128);
+ }
+ if (hwcap2 & HWCAP2_SVEBITPERM)
+ setCPUFeature(FEAT_SVE_BITPERM);
+ if (hwcap2 & HWCAP2_SVESHA3)
+ setCPUFeature(FEAT_SVE_SHA3);
+ if (hwcap2 & HWCAP2_SVESM4)
+ setCPUFeature(FEAT_SVE_SM4);
+ if (hwcap2 & HWCAP2_DCPODP)
+ setCPUFeature(FEAT_DPB2);
+ if (hwcap & HWCAP_ATOMICS)
+ setCPUFeature(FEAT_LSE);
+ if (hwcap2 & HWCAP2_RNG)
+ setCPUFeature(FEAT_RNG);
+ if (hwcap2 & HWCAP2_I8MM)
+ setCPUFeature(FEAT_I8MM);
+ if (hwcap2 & HWCAP2_EBF16)
+ setCPUFeature(FEAT_EBF16);
+ if (hwcap2 & HWCAP2_SVE_EBF16)
+ setCPUFeature(FEAT_SVE_EBF16);
+ if (hwcap2 & HWCAP2_DGH)
+ setCPUFeature(FEAT_DGH);
+ if (hwcap2 & HWCAP2_FRINT)
+ setCPUFeature(FEAT_FRINTTS);
+ if (hwcap2 & HWCAP2_SVEI8MM)
+ setCPUFeature(FEAT_SVE_I8MM);
+ if (hwcap2 & HWCAP2_SVEF32MM)
+ setCPUFeature(FEAT_SVE_F32MM);
+ if (hwcap2 & HWCAP2_SVEF64MM)
+ setCPUFeature(FEAT_SVE_F64MM);
+ if (hwcap2 & HWCAP2_BTI)
+ setCPUFeature(FEAT_BTI);
+ if (hwcap2 & HWCAP2_RPRES)
+ setCPUFeature(FEAT_RPRES);
+ if (hwcap2 & HWCAP2_WFXT)
+ setCPUFeature(FEAT_WFXT);
+ if (hwcap2 & HWCAP2_SME)
+ setCPUFeature(FEAT_SME);
+ if (hwcap2 & HWCAP2_SME_I16I64)
+ setCPUFeature(FEAT_SME_I64);
+ if (hwcap2 & HWCAP2_SME_F64F64)
+ setCPUFeature(FEAT_SME_F64);
+ if (hwcap & HWCAP_CPUID) {
+ unsigned long ftr;
+ getCPUFeature(ID_AA64PFR1_EL1, ftr);
+ // ID_AA64PFR1_EL1.MTE >= 0b0001
+ if (extractBits(ftr, 8, 4) >= 0x1)
+ setCPUFeature(FEAT_MEMTAG);
+ // ID_AA64PFR1_EL1.SSBS == 0b0001
+ if (extractBits(ftr, 4, 4) == 0x1)
+ setCPUFeature(FEAT_SSBS);
+ // ID_AA64PFR1_EL1.SME == 0b0010
+ if (extractBits(ftr, 24, 4) == 0x2)
+ setCPUFeature(FEAT_SME2);
+ getCPUFeature(ID_AA64PFR0_EL1, ftr);
+ // ID_AA64PFR0_EL1.FP != 0b1111
+ if (extractBits(ftr, 16, 4) != 0xF) {
+ setCPUFeature(FEAT_FP);
+ // ID_AA64PFR0_EL1.AdvSIMD has the same value as ID_AA64PFR0_EL1.FP
+ setCPUFeature(FEAT_SIMD);
+ }
+ // ID_AA64PFR0_EL1.SVE != 0b0000
+ if (extractBits(ftr, 32, 4) != 0x0) {
+ // get ID_AA64ZFR0_EL1, that name supported
+ // if sve enabled only
+ getCPUFeature(S3_0_C0_C4_4, ftr);
+ // ID_AA64ZFR0_EL1.SVEver == 0b0000
+ if (extractBits(ftr, 0, 4) == 0x0)
+ setCPUFeature(FEAT_SVE);
+ // ID_AA64ZFR0_EL1.SVEver == 0b0001
+ if (extractBits(ftr, 0, 4) == 0x1)
+ setCPUFeature(FEAT_SVE2);
+ // ID_AA64ZFR0_EL1.BF16 != 0b0000
+ if (extractBits(ftr, 20, 4) != 0x0)
+ setCPUFeature(FEAT_SVE_BF16);
+ }
+ getCPUFeature(ID_AA64ISAR0_EL1, ftr);
+ // ID_AA64ISAR0_EL1.SHA3 != 0b0000
+ if (extractBits(ftr, 32, 4) != 0x0)
+ setCPUFeature(FEAT_SHA3);
+ getCPUFeature(ID_AA64ISAR1_EL1, ftr);
+ // ID_AA64ISAR1_EL1.DPB >= 0b0001
+ if (extractBits(ftr, 0, 4) >= 0x1)
+ setCPUFeature(FEAT_DPB);
+ // ID_AA64ISAR1_EL1.LRCPC != 0b0000
+ if (extractBits(ftr, 20, 4) != 0x0)
+ setCPUFeature(FEAT_RCPC);
+ // ID_AA64ISAR1_EL1.SPECRES == 0b0001
+ if (extractBits(ftr, 40, 4) == 0x2)
+ setCPUFeature(FEAT_PREDRES);
+ // ID_AA64ISAR1_EL1.BF16 != 0b0000
+ if (extractBits(ftr, 44, 4) != 0x0)
+ setCPUFeature(FEAT_BF16);
+ // ID_AA64ISAR1_EL1.LS64 >= 0b0001
+ if (extractBits(ftr, 60, 4) >= 0x1)
+ setCPUFeature(FEAT_LS64);
+ // ID_AA64ISAR1_EL1.LS64 >= 0b0010
+ if (extractBits(ftr, 60, 4) >= 0x2)
+ setCPUFeature(FEAT_LS64_V);
+ // ID_AA64ISAR1_EL1.LS64 >= 0b0011
+ if (extractBits(ftr, 60, 4) >= 0x3)
+ setCPUFeature(FEAT_LS64_ACCDATA);
+ } else {
+ // Set some features in case of no CPUID support
+ if (hwcap & (HWCAP_FP | HWCAP_FPHP)) {
+ setCPUFeature(FEAT_FP);
+ // FP and AdvSIMD fields have the same value
+ setCPUFeature(FEAT_SIMD);
+ }
+ if (hwcap & HWCAP_DCPOP || hwcap2 & HWCAP2_DCPODP)
+ setCPUFeature(FEAT_DPB);
+ if (hwcap & HWCAP_LRCPC || hwcap & HWCAP_ILRCPC)
+ setCPUFeature(FEAT_RCPC);
+ if (hwcap2 & HWCAP2_BF16 || hwcap2 & HWCAP2_EBF16)
+ setCPUFeature(FEAT_BF16);
+ if (hwcap2 & HWCAP2_SVEBF16)
+ setCPUFeature(FEAT_SVE_BF16);
+ if (hwcap2 & HWCAP2_SVE2 && hwcap & HWCAP_SVE)
+ setCPUFeature(FEAT_SVE2);
+ if (hwcap & HWCAP_SHA3)
+ setCPUFeature(FEAT_SHA3);
+ }
+}
+
+void CONSTRUCTOR_ATTRIBUTE init_cpu_features(void) {
+ unsigned long hwcap;
+ unsigned long hwcap2;
+ // CPU features already initialized.
+ if (__aarch64_cpu_features.features)
+ return;
+ setCPUFeature(FEAT_MAX);
+#if defined(__FreeBSD__)
+ int res = 0;
+ res = elf_aux_info(AT_HWCAP, &hwcap, sizeof hwcap);
+ res |= elf_aux_info(AT_HWCAP2, &hwcap2, sizeof hwcap2);
+ if (res)
+ return;
+#else
+#if defined(__ANDROID__)
+ // Don't set any CPU features,
+ // detection could be wrong on Exynos 9810.
+ IF_EXYNOS9810 return;
+#endif // defined(__ANDROID__)
+ hwcap = getauxval(AT_HWCAP);
+ hwcap2 = getauxval(AT_HWCAP2);
+#endif // defined(__FreeBSD__)
+ init_cpu_features_resolver(hwcap, hwcap2);
+#undef extractBits
+#undef getCPUFeature
+#undef setCPUFeature
+#undef IF_EXYNOS9810
}
+#endif // !defined(DISABLE_AARCH64_FMV)
#endif // defined(__has_include)
#endif // __has_include(<sys/auxv.h>)
+#endif // __has_include(<asm/hwcap.h>)
#endif // defined(__aarch64__)
//
// It should never be exported from a dylib, so it is marked
// visibility hidden.
+#ifndef DONT_DEFINE_EPRINTF
#ifndef _WIN32
__attribute__((visibility("hidden")))
#endif
fflush(stderr);
compilerrt_abort();
}
+#endif
COMPILER_RT_ALIAS(__fixdfdi, __aeabi_d2lz)
#endif
#endif
+
+#if defined(__MINGW32__) && defined(__arm__)
+COMPILER_RT_ALIAS(__fixdfdi, __dtoi64)
+#endif
COMPILER_RT_ALIAS(__fixsfdi, __aeabi_f2lz)
#endif
#endif
+
+#if defined(__MINGW32__) && defined(__arm__)
+COMPILER_RT_ALIAS(__fixsfdi, __stoi64)
+#endif
COMPILER_RT_ALIAS(__fixunsdfdi, __aeabi_d2ulz)
#endif
#endif
+
+#if defined(__MINGW32__) && defined(__arm__)
+COMPILER_RT_ALIAS(__fixunsdfdi, __dtou64)
+#endif
COMPILER_RT_ALIAS(__fixunssfdi, __aeabi_f2ulz)
#endif
#endif
+
+#if defined(__MINGW32__) && defined(__arm__)
+COMPILER_RT_ALIAS(__fixunssfdi, __stou64)
+#endif
// mmmm mmmm mmmm
#if defined(_MSC_VER) && !defined(__clang__)
-// MSVC throws a warning about 'unitialized variable use' here,
+// MSVC throws a warning about 'uninitialized variable use' here,
// disable it for builds that warn-as-error
#pragma warning(push)
#pragma warning(disable : 4700)
// mmmm mmmm mmmm
#if defined(_MSC_VER) && !defined(__clang__)
-// MSVC throws a warning about 'unitialized variable use' here,
+// MSVC throws a warning about 'uninitialized variable use' here,
// disable it for builds that warn-as-error
#pragma warning(push)
#pragma warning(disable : 4700)
// mmmm mmmm mmmm
#if defined(_MSC_VER) && !defined(__clang__)
-// MSVC throws a warning about 'unitialized variable use' here,
+// MSVC throws a warning about 'uninitialized variable use' here,
// disable it for builds that warn-as-error
#pragma warning(push)
#pragma warning(disable : 4700)
COMPILER_RT_ALIAS(__floatdidf, __aeabi_l2d)
#endif
#endif
+
+#if defined(__MINGW32__) && defined(__arm__)
+COMPILER_RT_ALIAS(__floatdidf, __i64tod)
+#endif
COMPILER_RT_ALIAS(__floatdisf, __aeabi_l2f)
#endif
#endif
+
+#if defined(__MINGW32__) && defined(__arm__)
+COMPILER_RT_ALIAS(__floatdisf, __i64tos)
+#endif
#include "int_lib.h"
-COMPILER_RT_ABI fp_t __floatsisf(int a) {
+COMPILER_RT_ABI fp_t __floatsisf(si_int a) {
const int aWidth = sizeof a * CHAR_BIT;
}
// Exponent of (fp_t)a is the width of abs(a).
- const int exponent = (aWidth - 1) - __builtin_clz(a);
+ const int exponent = (aWidth - 1) - clzsi(a);
rep_t result;
// Shift a into the significand field, rounding if it is a right-shift
#include "fp_lib.h"
#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT)
-COMPILER_RT_ABI fp_t __floatsitf(int a) {
+COMPILER_RT_ABI fp_t __floatsitf(si_int a) {
const int aWidth = sizeof a * CHAR_BIT;
// All other cases begin by extracting the sign and absolute value of a
rep_t sign = 0;
- unsigned aAbs = (unsigned)a;
+ su_int aAbs = (su_int)a;
if (a < 0) {
sign = signBit;
- aAbs = ~(unsigned)a + 1U;
+ aAbs = ~(su_int)a + (su_int)1U;
}
// Exponent of (fp_t)a is the width of abs(a).
- const int exponent = (aWidth - 1) - __builtin_clz(aAbs);
+ const int exponent = (aWidth - 1) - clzsi(aAbs);
rep_t result;
// Shift a into the significand field and clear the implicit bit.
COMPILER_RT_ALIAS(__floatundidf, __aeabi_ul2d)
#endif
#endif
+
+#if defined(__MINGW32__) && defined(__arm__)
+COMPILER_RT_ALIAS(__floatundidf, __u64tod)
+#endif
COMPILER_RT_ALIAS(__floatundisf, __aeabi_ul2f)
#endif
#endif
+
+#if defined(__MINGW32__) && defined(__arm__)
+COMPILER_RT_ALIAS(__floatundisf, __u64tos)
+#endif
#include "int_lib.h"
-COMPILER_RT_ABI fp_t __floatunsisf(unsigned int a) {
+COMPILER_RT_ABI fp_t __floatunsisf(su_int a) {
const int aWidth = sizeof a * CHAR_BIT;
return fromRep(0);
// Exponent of (fp_t)a is the width of abs(a).
- const int exponent = (aWidth - 1) - __builtin_clz(a);
+ const int exponent = (aWidth - 1) - clzsi(a);
rep_t result;
// Shift a into the significand field, rounding if it is a right-shift
#include "fp_lib.h"
#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT)
-COMPILER_RT_ABI fp_t __floatunsitf(unsigned int a) {
+COMPILER_RT_ABI fp_t __floatunsitf(su_int a) {
const int aWidth = sizeof a * CHAR_BIT;
return fromRep(0);
// Exponent of (fp_t)a is the width of abs(a).
- const int exponent = (aWidth - 1) - __builtin_clz(a);
+ const int exponent = (aWidth - 1) - clzsi(a);
rep_t result;
// Shift a into the significand field and clear the implicit bit.
#elif __SIZEOF_POINTER__ == 8 && __SIZEOF_LONG__ == 4
// LLP64 ABIs use long long instead of long.
typedef long long CMP_RESULT;
+#elif __AVR__
+// AVR uses a single byte for the return value.
+typedef char CMP_RESULT;
#else
// Otherwise the comparison functions return long.
typedef long CMP_RESULT;
return __builtin_clzl(a);
#else
if (a & REP_C(0xffffffff00000000))
- return __builtin_clz(a >> 32);
+ return clzsi(a >> 32);
else
- return 32 + __builtin_clz(a & REP_C(0xffffffff));
+ return 32 + clzsi(a & REP_C(0xffffffff));
#endif
}
#include "fp_mode.h"
// IEEE-754 default rounding (to nearest, ties to even).
-CRT_FE_ROUND_MODE __fe_getround() { return CRT_FE_TONEAREST; }
+CRT_FE_ROUND_MODE __fe_getround(void) { return CRT_FE_TONEAREST; }
-int __fe_raise_inexact() {
+int __fe_raise_inexact(void) {
return 0;
}
//
//===----------------------------------------------------------------------===//
-#ifndef FP_MODE
-#define FP_MODE
+#ifndef FP_MODE_H
+#define FP_MODE_H
typedef enum {
CRT_FE_TONEAREST,
#define DST_REP_C UINT16_C
static const int dstSigBits = 10;
+#elif defined DST_BFLOAT
+typedef __bf16 dst_t;
+typedef uint16_t dst_rep_t;
+#define DST_REP_C UINT16_C
+static const int dstSigBits = 7;
+
#else
#error Destination should be single precision or double precision!
#endif // end destination precision
}
#if defined(__arm__) && !defined(__USING_SJLJ_EXCEPTIONS__) && \
- !defined(__ARM_DWARF_EH__)
+ !defined(__ARM_DWARF_EH__) && !defined(__SEH__)
#define USING_ARM_EHABI 1
_Unwind_Reason_Code __gnu_unwind_frame(struct _Unwind_Exception *,
struct _Unwind_Context *);
#define X87_TOWARDZERO 0x0c00
#define X87_RMODE_MASK (X87_TONEAREST | X87_UPWARD | X87_DOWNWARD | X87_TOWARDZERO)
-CRT_FE_ROUND_MODE __fe_getround() {
+CRT_FE_ROUND_MODE __fe_getround(void) {
// Assume that the rounding mode state for the fpu agrees with the SSE unit.
unsigned short cw;
__asm__ __volatile__ ("fnstcw %0" : "=m" (cw));
return CRT_FE_TONEAREST;
}
-int __fe_raise_inexact() {
+int __fe_raise_inexact(void) {
float f = 1.0f, g = 3.0f;
__asm__ __volatile__ ("fdivs %1" : "+t" (f) : "m" (g));
return 0;
#error "unknown endianness"
#endif // !_LITTLE_ENDIAN
-#endif // Solaris and AuroraUX.
+#endif // Solaris
// ..
} udwords;
#if defined(__LP64__) || defined(__wasm__) || defined(__mips64) || \
- defined(__riscv) || defined(_WIN64)
+ defined(__SIZEOF_INT128__) || defined(_WIN64)
#define CRT_HAS_128BIT
#endif
--- /dev/null
+//=== lib/builtins/loongarch/fp_mode.c - Floaing-point mode utilities -*- C -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include "../fp_mode.h"
+
+#define LOONGARCH_TONEAREST 0x0000
+#define LOONGARCH_TOWARDZERO 0x0100
+#define LOONGARCH_UPWARD 0x0200
+#define LOONGARCH_DOWNWARD 0x0300
+
+#define LOONGARCH_RMODE_MASK (LOONGARCH_TONEAREST | LOONGARCH_TOWARDZERO | \
+ LOONGARCH_UPWARD | LOONGARCH_DOWNWARD)
+
+#define LOONGARCH_INEXACT 0x10000
+
+CRT_FE_ROUND_MODE __fe_getround(void) {
+#if __loongarch_frlen != 0
+ int fcsr;
+# ifdef __clang__
+ __asm__ __volatile__("movfcsr2gr %0, $fcsr0" : "=r" (fcsr));
+# else
+ __asm__ __volatile__("movfcsr2gr %0, $r0" : "=r" (fcsr));
+# endif
+ fcsr &= LOONGARCH_RMODE_MASK;
+ switch (fcsr) {
+ case LOONGARCH_TOWARDZERO:
+ return CRT_FE_TOWARDZERO;
+ case LOONGARCH_DOWNWARD:
+ return CRT_FE_DOWNWARD;
+ case LOONGARCH_UPWARD:
+ return CRT_FE_UPWARD;
+ case LOONGARCH_TONEAREST:
+ default:
+ return CRT_FE_TONEAREST;
+ }
+#else
+ return CRT_FE_TONEAREST;
+#endif
+}
+
+int __fe_raise_inexact(void) {
+#if __loongarch_frlen != 0
+ int fcsr;
+# ifdef __clang__
+ __asm__ __volatile__("movfcsr2gr %0, $fcsr0" : "=r" (fcsr));
+ __asm__ __volatile__(
+ "movgr2fcsr $fcsr0, %0" :: "r" (fcsr | LOONGARCH_INEXACT));
+# else
+ __asm__ __volatile__("movfcsr2gr %0, $r0" : "=r" (fcsr));
+ __asm__ __volatile__(
+ "movgr2fcsr $r0, %0" :: "r" (fcsr | LOONGARCH_INEXACT));
+# endif
+#endif
+ return 0;
+}
atomic_signal_fence
atomic_thread_fence
int_util
+fp_mode
}
int32_t __isOSVersionAtLeast(int32_t Major, int32_t Minor, int32_t Subminor) {
- (int32_t) Minor;
- (int32_t) Subminor;
+ (void) Minor;
+ (void) Subminor;
static pthread_once_t once = PTHREAD_ONCE_INIT;
pthread_once(&once, readSystemProperties);
--- /dev/null
+//=== lib/builtins/riscv/fp_mode.c - Floaing-point mode utilities -*- C -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include "../fp_mode.h"
+
+#define RISCV_TONEAREST 0x0
+#define RISCV_TOWARDZERO 0x1
+#define RISCV_DOWNWARD 0x2
+#define RISCV_UPWARD 0x3
+
+#define RISCV_INEXACT 0x1
+
+CRT_FE_ROUND_MODE __fe_getround(void) {
+#if defined(__riscv_f)
+ int frm;
+ __asm__ __volatile__("frrm %0" : "=r" (frm));
+ switch (frm) {
+ case RISCV_TOWARDZERO:
+ return CRT_FE_TOWARDZERO;
+ case RISCV_DOWNWARD:
+ return CRT_FE_DOWNWARD;
+ case RISCV_UPWARD:
+ return CRT_FE_UPWARD;
+ case RISCV_TONEAREST:
+ default:
+ return CRT_FE_TONEAREST;
+ }
+#else
+ return CRT_FE_TONEAREST;
+#endif
+}
+
+int __fe_raise_inexact(void) {
+#if defined(__riscv_f)
+ __asm__ __volatile__("csrsi fflags, %0" :: "i" (RISCV_INEXACT));
+#endif
+ return 0;
+}
__riscv_restore_12:
ld s11, 8(sp)
addi sp, sp, 16
- // fallthrough into __riscv_restore_11/10/9/8
+ // fallthrough into __riscv_restore_11/10
.globl __riscv_restore_11
.type __riscv_restore_11,@function
.type __riscv_restore_3,@function
.globl __riscv_restore_2
.type __riscv_restore_2,@function
- .globl __riscv_restore_1
- .type __riscv_restore_1,@function
- .globl __riscv_restore_0
- .type __riscv_restore_0,@function
__riscv_restore_3:
__riscv_restore_2:
ld s2, 0(sp)
addi sp, sp, 16
// fallthrough into __riscv_restore_1/0
+ .globl __riscv_restore_1
+ .type __riscv_restore_1,@function
+ .globl __riscv_restore_0
+ .type __riscv_restore_0,@function
__riscv_restore_1:
__riscv_restore_0:
ld s0, 0(sp)
.type __riscv_save_1,@function
.globl __riscv_save_0
.type __riscv_save_0,@function
+__riscv_save_1:
+__riscv_save_0:
addi sp, sp, -16
sd s0, 0(sp)
sd ra, 8(sp)
// which loads r11 with a pointer to the outer function's locals
// and then jumps to the target nested function.
-#if __ppc__ && !defined(__powerpc64__)
+#if __powerpc__ && !defined(__powerpc64__)
COMPILER_RT_ABI void __trampoline_setup(uint32_t *trampOnStack,
int trampSizeAllocated,
const void *realFunc, void *localsPtr) {
// clear instruction cache
__clear_cache(trampOnStack, &trampOnStack[10]);
}
-#endif // __ppc__ && !defined(__powerpc64__)
+#endif // __powerpc__ && !defined(__powerpc64__)
--- /dev/null
+//===-- lib/truncdfbf2.c - double -> bfloat conversion ------------*- C -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#define SRC_DOUBLE
+#define DST_BFLOAT
+#include "fp_trunc_impl.inc"
+
+COMPILER_RT_ABI dst_t __truncdfbf2(double a) { return __truncXfYf2__(a); }
--- /dev/null
+//===-- lib/truncsfbf2.c - single -> bfloat conversion ------------*- C -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#define SRC_SINGLE
+#define DST_BFLOAT
+#include "fp_trunc_impl.inc"
+
+COMPILER_RT_ABI dst_t __truncsfbf2(float a) { return __truncXfYf2__(a); }
// MSVC throws a warning about mod 0 here, disable it for builds that
// warn-as-error
#pragma warning(push)
-#pragma warning(disable : 4724)
+#pragma warning(disable : 4723 4724)
#endif
COMPILER_RT_ABI du_int __udivmoddi4(du_int a, du_int b, du_int *rem) {
r.s.high = n.s.high & (d.s.high - 1);
*rem = r.all;
}
- return n.s.high >> __builtin_ctz(d.s.high);
+ return n.s.high >> ctzsi(d.s.high);
}
// K K
// ---
*rem = n.s.low & (d.s.low - 1);
if (d.s.low == 1)
return n.all;
- sr = __builtin_ctz(d.s.low);
+ sr = ctzsi(d.s.low);
q.s.high = n.s.high >> sr;
q.s.low = (n.s.high << (n_uword_bits - sr)) | (n.s.low >> sr);
return q.all;
${SANITIZER_COMMON_CFLAGS}
)
+ # Too many existing bugs, needs cleanup.
+ append_list_if(COMPILER_RT_HAS_WNO_FORMAT -Wno-format CFI_CFLAGS)
+
set(CFI_DIAG_CFLAGS
-DCFI_ENABLE_DIAG=1
)
}
if (symtab > strtab) {
- VReport(1, "Can not handle: symtab > strtab (%p > %zx)\n", symtab, strtab);
+ VReport(1, "Can not handle: symtab > strtab (%zx > %zx)\n", symtab, strtab);
return 0;
}
if (phdr_idx == info->dlpi_phnum) {
// Nope, either different segments or just bogus pointers.
// Can not handle this.
- VReport(1, "Can not handle: symtab %p, strtab %zx\n", symtab, strtab);
+ VReport(1, "Can not handle: symtab %zx, strtab %zx\n", symtab, strtab);
return 0;
}
}
THREADLOCAL int in_loader;
-BlockingMutex shadow_update_lock(LINKER_INITIALIZED);
+Mutex shadow_update_lock;
-void EnterLoader() NO_THREAD_SAFETY_ANALYSIS {
+void EnterLoader() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
if (in_loader == 0) {
shadow_update_lock.Lock();
}
++in_loader;
}
-void ExitLoader() NO_THREAD_SAFETY_ANALYSIS {
+void ExitLoader() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
CHECK(in_loader > 0);
--in_loader;
UpdateShadow();
return;
}
CFICheckFn cfi_check = sv.get_cfi_check();
- VReport(2, "__cfi_check at %p\n", cfi_check);
+ VReport(2, "__cfi_check at %p\n", (void *)cfi_check);
cfi_check(CallSiteTypeId, Ptr, DiagData);
}
return res;
}
-static BlockingMutex interceptor_init_lock(LINKER_INITIALIZED);
+static Mutex interceptor_init_lock;
static bool interceptors_inited = false;
static void EnsureInterceptorsInitialized() {
- BlockingMutexLock lock(&interceptor_init_lock);
+ Lock lock(&interceptor_init_lock);
if (interceptors_inited)
return;
-add_compiler_rt_component(crt)
-
-function(check_cxx_section_exists section output)
- cmake_parse_arguments(ARG "" "" "SOURCE;FLAGS" ${ARGN})
- if(NOT ARG_SOURCE)
- set(ARG_SOURCE "int main() { return 0; }\n")
+if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
+ cmake_minimum_required(VERSION 3.13.4)
+ if ("${CMAKE_VERSION}" VERSION_LESS "3.20.0")
+ message(WARNING
+ "Your CMake version is ${CMAKE_VERSION}. Starting with LLVM 17.0.0, the "
+ "minimum version of CMake required to build LLVM will become 3.20.0, and "
+ "using an older CMake will become an error. Please upgrade your CMake to "
+ "at least 3.20.0 now to avoid issues in the future!")
endif()
- string(RANDOM TARGET_NAME)
- set(TARGET_NAME "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/cmTC_${TARGET_NAME}.dir")
- file(MAKE_DIRECTORY ${TARGET_NAME})
+ set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
+ project(CompilerRTCRT C)
+ set(COMPILER_RT_STANDALONE_BUILD TRUE)
+ set(COMPILER_RT_CRT_STANDALONE_BUILD TRUE)
- file(WRITE "${TARGET_NAME}/CheckSectionExists.c" "${ARG_SOURCE}\n")
+ set(COMPILER_RT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../..")
- string(REGEX MATCHALL "<[A-Za-z0-9_]*>" substitutions
- ${CMAKE_C_COMPILE_OBJECT})
+ set(LLVM_COMMON_CMAKE_UTILS "${COMPILER_RT_SOURCE_DIR}/../cmake")
- set(try_compile_flags "${ARG_FLAGS}")
- if(CMAKE_C_COMPILER_ID MATCHES Clang AND CMAKE_C_COMPILER_TARGET)
- list(APPEND try_compile_flags "-target ${CMAKE_C_COMPILER_TARGET}")
- endif()
- append_list_if(COMPILER_RT_HAS_FNO_LTO_FLAG -fno-lto try_compile_flags)
- if(NOT COMPILER_RT_ENABLE_PGO)
- if(LLVM_PROFDATA_FILE AND COMPILER_RT_HAS_FNO_PROFILE_INSTR_USE_FLAG)
- list(APPEND try_compile_flags "-fno-profile-instr-use")
- endif()
- if(LLVM_BUILD_INSTRUMENTED MATCHES IR AND COMPILER_RT_HAS_FNO_PROFILE_GENERATE_FLAG)
- list(APPEND try_compile_flags "-fno-profile-generate")
- elseif(LLVM_BUILD_INSTRUMENTED AND COMPILER_RT_HAS_FNO_PROFILE_INSTR_GENERATE_FLAG)
- list(APPEND try_compile_flags "-fno-profile-instr-generate")
- endif()
- endif()
+ # Add path for custom modules
+ list(INSERT CMAKE_MODULE_PATH 0
+ "${COMPILER_RT_SOURCE_DIR}/cmake"
+ "${COMPILER_RT_SOURCE_DIR}/cmake/Modules"
+ "${LLVM_COMMON_CMAKE_UTILS}"
+ "${LLVM_COMMON_CMAKE_UTILS}/Modules"
+ )
- string(REPLACE ";" " " extra_flags "${try_compile_flags}")
-
- set(test_compile_command "${CMAKE_C_COMPILE_OBJECT}")
- foreach(substitution ${substitutions})
- if(substitution STREQUAL "<CMAKE_C_COMPILER>")
- string(REPLACE "<CMAKE_C_COMPILER>" "${CMAKE_C_COMPILER} ${CMAKE_C_COMPILER_ARG1}"
- test_compile_command ${test_compile_command})
- elseif(substitution STREQUAL "<OBJECT>")
- string(REPLACE "<OBJECT>" "${TARGET_NAME}/CheckSectionExists.o"
- test_compile_command ${test_compile_command})
- elseif(substitution STREQUAL "<SOURCE>")
- string(REPLACE "<SOURCE>" "${TARGET_NAME}/CheckSectionExists.c"
- test_compile_command ${test_compile_command})
- elseif(substitution STREQUAL "<FLAGS>")
- string(REPLACE "<FLAGS>" "${CMAKE_C_FLAGS} ${extra_flags}"
- test_compile_command ${test_compile_command})
- else()
- string(REPLACE "${substitution}" "" test_compile_command
- ${test_compile_command})
- endif()
- endforeach()
+ include(base-config-ix)
+ include(CompilerRTUtils)
- # Strip quotes from the compile command, as the compiler is not expecting
- # quoted arguments (potential quotes added from D62063).
- string(REPLACE "\"" "" test_compile_command "${test_compile_command}")
+ load_llvm_config()
+ construct_compiler_rt_default_triple()
- string(REPLACE " " ";" test_compile_command "${test_compile_command}")
+ include(SetPlatformToolchainTools)
+ include(AddCompilerRT)
+endif()
- execute_process(
- COMMAND ${test_compile_command}
- RESULT_VARIABLE TEST_RESULT
- OUTPUT_VARIABLE TEST_OUTPUT
- ERROR_VARIABLE TEST_ERROR
- )
+include(crt-config-ix)
- # Explicitly throw a fatal error message if test_compile_command fails.
- if(TEST_RESULT)
- message(FATAL_ERROR "${TEST_ERROR}")
- return()
- endif()
+if(COMPILER_RT_HAS_CRT)
+ add_compiler_rt_component(crt)
- execute_process(
- COMMAND ${CMAKE_OBJDUMP} -h "${TARGET_NAME}/CheckSectionExists.o"
- RESULT_VARIABLE CHECK_RESULT
- OUTPUT_VARIABLE CHECK_OUTPUT
- ERROR_VARIABLE CHECK_ERROR
- )
- string(FIND "${CHECK_OUTPUT}" "${section}" SECTION_FOUND)
+ include(CheckSectionExists)
+ check_section_exists(".init_array" COMPILER_RT_HAS_INITFINI_ARRAY
+ SOURCE "volatile int x;\n__attribute__((constructor)) void f(void) {x = 0;}\nint main(void) { return 0; }\n")
- if(NOT SECTION_FOUND EQUAL -1)
- set(${output} TRUE PARENT_SCOPE)
- else()
- set(${output} FALSE PARENT_SCOPE)
+ append_list_if(COMPILER_RT_HAS_STD_C11_FLAG -std=c11 CRT_CFLAGS)
+ append_list_if(COMPILER_RT_HAS_INITFINI_ARRAY -DCRT_HAS_INITFINI_ARRAY CRT_CFLAGS)
+ append_list_if(COMPILER_RT_CRT_USE_EH_FRAME_REGISTRY -DEH_USE_FRAME_REGISTRY CRT_CFLAGS)
+ append_list_if(COMPILER_RT_HAS_FPIC_FLAG -fPIC CRT_CFLAGS)
+ append_list_if(COMPILER_RT_HAS_WNO_PEDANTIC -Wno-pedantic CRT_CFLAGS)
+ if (COMPILER_RT_HAS_FCF_PROTECTION_FLAG)
+ append_list_if(COMPILER_RT_ENABLE_CET -fcf-protection=full CRT_CFLAGS)
endif()
- file(REMOVE_RECURSE ${TARGET_NAME})
-endfunction()
-
-check_cxx_section_exists(".init_array" COMPILER_RT_HAS_INITFINI_ARRAY
- SOURCE "volatile int x;\n__attribute__((constructor)) void f() {x = 0;}\nint main() { return 0; }\n")
-
-append_list_if(COMPILER_RT_HAS_STD_C11_FLAG -std=c11 CRT_CFLAGS)
-append_list_if(COMPILER_RT_HAS_INITFINI_ARRAY -DCRT_HAS_INITFINI_ARRAY CRT_CFLAGS)
-append_list_if(COMPILER_RT_CRT_USE_EH_FRAME_REGISTRY -DEH_USE_FRAME_REGISTRY CRT_CFLAGS)
-append_list_if(COMPILER_RT_HAS_FPIC_FLAG -fPIC CRT_CFLAGS)
-append_list_if(COMPILER_RT_HAS_WNO_PEDANTIC -Wno-pedantic CRT_CFLAGS)
-
-foreach(arch ${CRT_SUPPORTED_ARCH})
- add_compiler_rt_runtime(clang_rt.crtbegin
- OBJECT
- ARCHS ${arch}
- SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/crtbegin.c
- CFLAGS ${CRT_CFLAGS}
- PARENT_TARGET crt)
- add_compiler_rt_runtime(clang_rt.crtend
- OBJECT
- ARCHS ${arch}
- SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/crtend.c
- CFLAGS ${CRT_CFLAGS}
- PARENT_TARGET crt)
-endforeach()
+ foreach(arch ${CRT_SUPPORTED_ARCH})
+ add_compiler_rt_runtime(clang_rt.crtbegin
+ OBJECT
+ ARCHS ${arch}
+ SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/crtbegin.c
+ CFLAGS ${CRT_CFLAGS}
+ PARENT_TARGET crt)
+ add_compiler_rt_runtime(clang_rt.crtend
+ OBJECT
+ ARCHS ${arch}
+ SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/crtend.c
+ CFLAGS ${CRT_CFLAGS}
+ PARENT_TARGET crt)
+ endforeach()
+endif()
extern void __cxa_finalize(void *) __attribute__((weak));
-static void __attribute__((used)) __do_init() {
+static void __attribute__((used)) __do_init(void) {
static _Bool __initialized;
if (__builtin_expect(__initialized, 0))
return;
extern fp __DTOR_LIST_END__[];
#endif
-static void __attribute__((used)) __do_fini() {
+static void __attribute__((used)) __do_fini(void) {
static _Bool __finalized;
if (__builtin_expect(__finalized, 0))
return;
# Prevent clang from generating libc calls.
append_list_if(COMPILER_RT_HAS_FFREESTANDING_FLAG -ffreestanding DFSAN_COMMON_CFLAGS)
+# Too many existing bugs, needs cleanup.
+append_list_if(COMPILER_RT_HAS_WNO_FORMAT -Wno-format DFSAN_COMMON_CFLAGS)
+
# Static runtime library.
add_compiler_rt_component(dfsan)
fname);
}
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __dfsan_wrapper_extern_weak_null(
+ const void *addr, char *fname) {
+ if (!addr)
+ Report(
+ "ERROR: DataFlowSanitizer: dfsan generated wrapper calling null "
+ "extern_weak function %s\nIf this only happens with dfsan, the "
+ "dfsan instrumentation pass may be accidentally optimizing out a "
+ "null check\n",
+ fname);
+}
+
// Use '-mllvm -dfsan-debug-nonzero-labels' and break on this function
// to try to figure out where labels are being introduced in a nominally
// label-free program.
// origins by copying origins in a reverse order; otherwise, copy origins in
// a normal order. The orders of origin transfer are consistent with the
// orders of how memcpy and memmove transfer user data.
- uptr src_aligned_beg = reinterpret_cast<uptr>(src) & ~3UL;
- uptr src_aligned_end = (reinterpret_cast<uptr>(src) + size) & ~3UL;
- uptr dst_aligned_beg = reinterpret_cast<uptr>(dst) & ~3UL;
+ uptr src_aligned_beg = OriginAlignDown((uptr)src);
+ uptr src_aligned_end = OriginAlignDown((uptr)src + size);
+ uptr dst_aligned_beg = OriginAlignDown((uptr)dst);
if (dst_aligned_beg < src_aligned_end && dst_aligned_beg >= src_aligned_beg)
return ReverseCopyOrigin(dst, src, size, stack);
return CopyOrigin(dst, src, size, stack);
*(u32 *)(end - kOriginAlign) = origin;
}
-static void WriteShadowInRange(dfsan_label label, uptr beg_shadow_addr,
- uptr end_shadow_addr) {
- // TODO: After changing dfsan_label to 8bit, use internal_memset when label
- // is not 0.
- dfsan_label *labelp = (dfsan_label *)beg_shadow_addr;
- if (label) {
- for (; (uptr)labelp < end_shadow_addr; ++labelp) *labelp = label;
- return;
- }
-
- for (; (uptr)labelp < end_shadow_addr; ++labelp) {
- // Don't write the label if it is already the value we need it to be.
- // In a program where most addresses are not labeled, it is common that
- // a page of shadow memory is entirely zeroed. The Linux copy-on-write
- // implementation will share all of the zeroed pages, making a copy of a
- // page when any value is written. The un-sharing will happen even if
- // the value written does not change the value in memory. Avoiding the
- // write when both |label| and |*labelp| are zero dramatically reduces
- // the amount of real memory used by large programs.
- if (!*labelp)
- continue;
-
- *labelp = 0;
- }
-}
-
-static void WriteShadowWithSize(dfsan_label label, uptr shadow_addr,
- uptr size) {
- WriteShadowInRange(label, shadow_addr, shadow_addr + size * sizeof(label));
-}
-
#define RET_CHAIN_ORIGIN(id) \
GET_CALLER_PC_BP_SP; \
(void)sp; \
MoveOrigin(dst, src, len, &stack);
}
-SANITIZER_INTERFACE_ATTRIBUTE void dfsan_mem_origin_transfer(const void *dst,
- const void *src,
- uptr len) {
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void dfsan_mem_origin_transfer(
+ const void *dst, const void *src, uptr len) {
__dfsan_mem_origin_transfer(dst, src, len);
}
+static void CopyShadow(void *dst, const void *src, uptr len) {
+ internal_memcpy((void *)__dfsan::shadow_for(dst),
+ (const void *)__dfsan::shadow_for(src),
+ len * sizeof(dfsan_label));
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void dfsan_mem_shadow_transfer(
+ void *dst, const void *src, uptr len) {
+ CopyShadow(dst, src, len);
+}
+
+// Copy shadow and origins of the len bytes from src to dst.
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
+__dfsan_mem_shadow_origin_transfer(void *dst, const void *src, uptr size) {
+ if (src == dst)
+ return;
+ CopyShadow(dst, src, size);
+ if (dfsan_get_track_origins()) {
+ // Duplicating code instead of calling __dfsan_mem_origin_transfer
+ // so that the getting the caller stack frame works correctly.
+ GET_CALLER_PC_BP;
+ GET_STORE_STACK_TRACE_PC_BP(pc, bp);
+ MoveOrigin(dst, src, size, &stack);
+ }
+}
+
+// Copy shadow and origins as per __atomic_compare_exchange.
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
+__dfsan_mem_shadow_origin_conditional_exchange(u8 condition, void *target,
+ void *expected,
+ const void *desired, uptr size) {
+ void *dst;
+ const void *src;
+ // condition is result of native call to __atomic_compare_exchange
+ if (condition) {
+ // Copy desired into target
+ dst = target;
+ src = desired;
+ } else {
+ // Copy target into expected
+ dst = expected;
+ src = target;
+ }
+ if (src == dst)
+ return;
+ CopyShadow(dst, src, size);
+ if (dfsan_get_track_origins()) {
+ // Duplicating code instead of calling __dfsan_mem_origin_transfer
+ // so that the getting the caller stack frame works correctly.
+ GET_CALLER_PC_BP;
+ GET_STORE_STACK_TRACE_PC_BP(pc, bp);
+ MoveOrigin(dst, src, size, &stack);
+ }
+}
+
namespace __dfsan {
bool dfsan_inited = false;
void dfsan_copy_memory(void *dst, const void *src, uptr size) {
internal_memcpy(dst, src, size);
- internal_memcpy((void *)shadow_for(dst), (const void *)shadow_for(src),
- size * sizeof(dfsan_label));
+ dfsan_mem_shadow_transfer(dst, src, size);
if (dfsan_get_track_origins())
dfsan_mem_origin_transfer(dst, src, size);
}
-} // namespace __dfsan
-
-// If the label s is tainted, set the size bytes from the address p to be a new
-// origin chain with the previous ID o and the current stack trace. This is
-// used by instrumentation to reduce code size when too much code is inserted.
-extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __dfsan_maybe_store_origin(
- dfsan_label s, void *p, uptr size, dfsan_origin o) {
- if (UNLIKELY(s)) {
- GET_CALLER_PC_BP_SP;
- (void)sp;
- GET_STORE_STACK_TRACE_PC_BP(pc, bp);
- SetOrigin(p, size, ChainOrigin(o, &stack));
- }
-}
-
// Releases the pages within the origin address range.
static void ReleaseOrigins(void *addr, uptr size) {
const uptr beg_origin_addr = (uptr)__dfsan::origin_for(addr);
Die();
}
+static void WriteZeroShadowInRange(uptr beg, uptr end) {
+ // Don't write the label if it is already the value we need it to be.
+ // In a program where most addresses are not labeled, it is common that
+ // a page of shadow memory is entirely zeroed. The Linux copy-on-write
+ // implementation will share all of the zeroed pages, making a copy of a
+ // page when any value is written. The un-sharing will happen even if
+ // the value written does not change the value in memory. Avoiding the
+ // write when both |label| and |*labelp| are zero dramatically reduces
+ // the amount of real memory used by large programs.
+ if (!mem_is_zero((const char *)beg, end - beg))
+ internal_memset((void *)beg, 0, end - beg);
+}
+
// Releases the pages within the shadow address range, and sets
// the shadow addresses not on the pages to be 0.
static void ReleaseOrClearShadows(void *addr, uptr size) {
const uptr end_shadow_addr = (uptr)__dfsan::shadow_for(end_addr);
if (end_shadow_addr - beg_shadow_addr <
- common_flags()->clear_shadow_mmap_threshold)
- return WriteShadowWithSize(0, beg_shadow_addr, size);
+ common_flags()->clear_shadow_mmap_threshold) {
+ WriteZeroShadowInRange(beg_shadow_addr, end_shadow_addr);
+ return;
+ }
const uptr page_size = GetPageSizeCached();
const uptr beg_aligned = RoundUpTo(beg_shadow_addr, page_size);
const uptr end_aligned = RoundDownTo(end_shadow_addr, page_size);
if (beg_aligned >= end_aligned) {
- WriteShadowWithSize(0, beg_shadow_addr, size);
+ WriteZeroShadowInRange(beg_shadow_addr, end_shadow_addr);
} else {
if (beg_aligned != beg_shadow_addr)
- WriteShadowInRange(0, beg_shadow_addr, beg_aligned);
+ WriteZeroShadowInRange(beg_shadow_addr, beg_aligned);
if (end_aligned != end_shadow_addr)
- WriteShadowInRange(0, end_aligned, end_shadow_addr);
+ WriteZeroShadowInRange(end_aligned, end_shadow_addr);
if (!MmapFixedSuperNoReserve(beg_aligned, end_aligned - beg_aligned))
Die();
}
void SetShadow(dfsan_label label, void *addr, uptr size, dfsan_origin origin) {
if (0 != label) {
const uptr beg_shadow_addr = (uptr)__dfsan::shadow_for(addr);
- WriteShadowWithSize(label, beg_shadow_addr, size);
+ internal_memset((void *)beg_shadow_addr, label, size);
if (dfsan_get_track_origins())
SetOrigin(addr, size, origin);
return;
ReleaseOrClearShadows(addr, size);
}
+} // namespace __dfsan
+
+// If the label s is tainted, set the size bytes from the address p to be a new
+// origin chain with the previous ID o and the current stack trace. This is
+// used by instrumentation to reduce code size when too much code is inserted.
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __dfsan_maybe_store_origin(
+ dfsan_label s, void *p, uptr size, dfsan_origin o) {
+ if (UNLIKELY(s)) {
+ GET_CALLER_PC_BP_SP;
+ (void)sp;
+ GET_STORE_STACK_TRACE_PC_BP(pc, bp);
+ SetOrigin(p, size, ChainOrigin(o, &stack));
+ }
+}
+
extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __dfsan_set_label(
dfsan_label label, dfsan_origin origin, void *addr, uptr size) {
- SetShadow(label, addr, size, origin);
+ __dfsan::SetShadow(label, addr, size, origin);
}
SANITIZER_INTERFACE_ATTRIBUTE
GET_STORE_STACK_TRACE_PC_BP(pc, bp);
init_origin = ChainOrigin(0, &stack, true);
}
- SetShadow(label, addr, size, init_origin);
+ __dfsan::SetShadow(label, addr, size, init_origin);
}
SANITIZER_INTERFACE_ATTRIBUTE
return (label & elem) == elem;
}
+namespace __dfsan {
+
+typedef void (*dfsan_conditional_callback_t)(dfsan_label label,
+ dfsan_origin origin);
+static dfsan_conditional_callback_t conditional_callback = nullptr;
+static dfsan_label labels_in_signal_conditional = 0;
+
+static void ConditionalCallback(dfsan_label label, dfsan_origin origin) {
+ // Programs have many branches. For efficiency the conditional sink callback
+ // handler needs to ignore as many as possible as early as possible.
+ if (label == 0) {
+ return;
+ }
+ if (conditional_callback == nullptr) {
+ return;
+ }
+
+ // This initial ConditionalCallback handler needs to be in here in dfsan
+ // runtime (rather than being an entirely user implemented hook) so that it
+ // has access to dfsan thread information.
+ DFsanThread *t = GetCurrentThread();
+ // A callback operation which does useful work (like record the flow) will
+ // likely be too long executed in a signal handler.
+ if (t && t->InSignalHandler()) {
+ // Record set of labels used in signal handler for completeness.
+ labels_in_signal_conditional |= label;
+ return;
+ }
+
+ conditional_callback(label, origin);
+}
+
+} // namespace __dfsan
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
+__dfsan_conditional_callback_origin(dfsan_label label, dfsan_origin origin) {
+ __dfsan::ConditionalCallback(label, origin);
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __dfsan_conditional_callback(
+ dfsan_label label) {
+ __dfsan::ConditionalCallback(label, 0);
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void dfsan_set_conditional_callback(
+ __dfsan::dfsan_conditional_callback_t callback) {
+ __dfsan::conditional_callback = callback;
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_label
+dfsan_get_labels_in_signal_conditional() {
+ return __dfsan::labels_in_signal_conditional;
+}
+
+namespace __dfsan {
+
+typedef void (*dfsan_reaches_function_callback_t)(dfsan_label label,
+ dfsan_origin origin,
+ const char *file,
+ unsigned int line,
+ const char *function);
+static dfsan_reaches_function_callback_t reaches_function_callback = nullptr;
+static dfsan_label labels_in_signal_reaches_function = 0;
+
+static void ReachesFunctionCallback(dfsan_label label, dfsan_origin origin,
+ const char *file, unsigned int line,
+ const char *function) {
+ if (label == 0) {
+ return;
+ }
+ if (reaches_function_callback == nullptr) {
+ return;
+ }
+
+ // This initial ReachesFunctionCallback handler needs to be in here in dfsan
+ // runtime (rather than being an entirely user implemented hook) so that it
+ // has access to dfsan thread information.
+ DFsanThread *t = GetCurrentThread();
+ // A callback operation which does useful work (like record the flow) will
+ // likely be too long executed in a signal handler.
+ if (t && t->InSignalHandler()) {
+ // Record set of labels used in signal handler for completeness.
+ labels_in_signal_reaches_function |= label;
+ return;
+ }
+
+ reaches_function_callback(label, origin, file, line, function);
+}
+
+} // namespace __dfsan
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
+__dfsan_reaches_function_callback_origin(dfsan_label label, dfsan_origin origin,
+ const char *file, unsigned int line,
+ const char *function) {
+ __dfsan::ReachesFunctionCallback(label, origin, file, line, function);
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
+__dfsan_reaches_function_callback(dfsan_label label, const char *file,
+ unsigned int line, const char *function) {
+ __dfsan::ReachesFunctionCallback(label, 0, file, line, function);
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
+dfsan_set_reaches_function_callback(
+ __dfsan::dfsan_reaches_function_callback_t callback) {
+ __dfsan::reaches_function_callback = callback;
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_label
+dfsan_get_labels_in_signal_reaches_function() {
+ return __dfsan::labels_in_signal_reaches_function;
+}
+
class Decorator : public __sanitizer::SanitizerCommonDecorator {
public:
Decorator() : SanitizerCommonDecorator() {}
d.Warning(), label, address, d.Default());
}
-bool PrintOriginTraceToStr(const void *addr, const char *description,
- InternalScopedString *out) {
- CHECK(out);
- CHECK(dfsan_get_track_origins());
+void PrintInvalidOriginIdWarning(dfsan_origin origin) {
Decorator d;
+ Printf(
+ " %sOrigin Id %d has invalid origin tracking. This can "
+ "be a DFSan bug.%s\n",
+ d.Warning(), origin, d.Default());
+}
- const dfsan_label label = *__dfsan::shadow_for(addr);
- CHECK(label);
-
- const dfsan_origin origin = *__dfsan::origin_for(addr);
-
- out->append(" %sTaint value 0x%x (at %p) origin tracking (%s)%s\n",
- d.Origin(), label, addr, description ? description : "",
- d.Default());
-
- Origin o = Origin::FromRawId(origin);
+bool PrintOriginTraceFramesToStr(Origin o, InternalScopedString *out) {
+ Decorator d;
bool found = false;
while (o.isChainedOrigin()) {
return found;
}
+bool PrintOriginTraceToStr(const void *addr, const char *description,
+ InternalScopedString *out) {
+ CHECK(out);
+ CHECK(dfsan_get_track_origins());
+ Decorator d;
+
+ const dfsan_label label = *__dfsan::shadow_for(addr);
+ CHECK(label);
+
+ const dfsan_origin origin = *__dfsan::origin_for(addr);
+
+ out->append(" %sTaint value 0x%x (at %p) origin tracking (%s)%s\n",
+ d.Origin(), label, addr, description ? description : "",
+ d.Default());
+
+ Origin o = Origin::FromRawId(origin);
+ return PrintOriginTraceFramesToStr(o, out);
+}
+
} // namespace
extern "C" SANITIZER_INTERFACE_ATTRIBUTE void dfsan_print_origin_trace(
PrintInvalidOriginWarning(label, addr);
}
-extern "C" SANITIZER_INTERFACE_ATTRIBUTE size_t
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE uptr
dfsan_sprint_origin_trace(const void *addr, const char *description,
- char *out_buf, size_t out_buf_size) {
+ char *out_buf, uptr out_buf_size) {
CHECK(out_buf);
if (!dfsan_get_track_origins()) {
return trace.length();
}
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void dfsan_print_origin_id_trace(
+ dfsan_origin origin) {
+ if (!dfsan_get_track_origins()) {
+ PrintNoOriginTrackingWarning();
+ return;
+ }
+ Origin o = Origin::FromRawId(origin);
+
+ InternalScopedString trace;
+ bool success = PrintOriginTraceFramesToStr(o, &trace);
+
+ if (trace.length())
+ Printf("%s", trace.data());
+
+ if (!success)
+ PrintInvalidOriginIdWarning(origin);
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE uptr dfsan_sprint_origin_id_trace(
+ dfsan_origin origin, char *out_buf, uptr out_buf_size) {
+ CHECK(out_buf);
+
+ if (!dfsan_get_track_origins()) {
+ PrintNoOriginTrackingWarning();
+ return 0;
+ }
+ Origin o = Origin::FromRawId(origin);
+
+ InternalScopedString trace;
+ bool success = PrintOriginTraceFramesToStr(o, &trace);
+
+ if (!success) {
+ PrintInvalidOriginIdWarning(origin);
+ return 0;
+ }
+
+ if (out_buf_size) {
+ internal_strncpy(out_buf, trace.data(), out_buf_size - 1);
+ out_buf[out_buf_size - 1] = '\0';
+ }
+
+ return trace.length();
+}
+
extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_origin
dfsan_get_init_origin(const void *addr) {
if (!dfsan_get_track_origins())
stack.Print();
}
-extern "C" SANITIZER_INTERFACE_ATTRIBUTE size_t
-dfsan_sprint_stack_trace(char *out_buf, size_t out_buf_size) {
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE uptr
+dfsan_sprint_stack_trace(char *out_buf, uptr out_buf_size) {
CHECK(out_buf);
GET_CALLER_PC_BP;
GET_STORE_STACK_TRACE_PC_BP(pc, bp);
}
}
+SANITIZER_INTERFACE_ATTRIBUTE
+void dfsan_set_arg_tls(uptr offset, dfsan_label label) {
+ // 2x to match ShadowTLSAlignment.
+ // ShadowTLSAlignment should probably be changed.
+ // TODO: Consider reducing ShadowTLSAlignment to 1.
+ // Aligning to 2 bytes is probably a remnant of fast16 mode.
+ ((dfsan_label *)__dfsan_arg_tls)[offset * 2] = label;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void dfsan_set_arg_origin_tls(uptr offset, dfsan_origin o) {
+ __dfsan_arg_origin_tls[offset] = o;
+}
+
extern "C" void dfsan_flush() {
const uptr maxVirtualAddress = GetMaxUserVirtualAddress();
for (unsigned i = 0; i < kMemoryLayoutSize; ++i) {
Die();
}
}
+ __dfsan::labels_in_signal_conditional = 0;
+ __dfsan::labels_in_signal_reaches_function = 0;
}
// TODO: CheckMemoryLayoutSanity is based on msan.
// Consider refactoring these into a shared implementation.
bool InitShadow(bool init_origins) {
// Let user know mapping parameters first.
- VPrintf(1, "dfsan_init %p\n", &__dfsan::dfsan_init);
+ VPrintf(1, "dfsan_init %p\n", (void *)&__dfsan::dfsan_init);
for (unsigned i = 0; i < kMemoryLayoutSize; ++i)
VPrintf(1, "%s: %zx - %zx\n", kMemoryLayout[i].name, kMemoryLayout[i].start,
kMemoryLayout[i].end - 1);
dfsan_allocator_init();
- DFsanThread *main_thread = DFsanThread::Create(nullptr, nullptr, nullptr);
+ DFsanThread *main_thread = DFsanThread::Create(nullptr, nullptr);
SetCurrentThread(main_thread);
- main_thread->ThreadStart();
+ main_thread->Init();
dfsan_init_is_running = false;
dfsan_inited = true;
// Zero out the TLS storage.
void dfsan_clear_thread_local_state();
+// Set DFSan label and origin TLS of argument for a call.
+// Note that offset may not correspond with argument number.
+// Some arguments (aggregate/array) will use several offsets.
+void dfsan_set_arg_tls(uptr offset, dfsan_label label);
+void dfsan_set_arg_origin_tls(uptr offset, dfsan_origin o);
+
// Return the origin associated with the first taint byte in the size bytes
// from the address addr.
dfsan_origin dfsan_read_origin_of_first_taint(const void *addr, uptr size);
// Copy or move the origins of the len bytes from src to dst.
void dfsan_mem_origin_transfer(const void *dst, const void *src, uptr len);
+
+// Copy shadow bytes from src to dst.
+// Note this preserves distinct taint labels at specific offsets.
+void dfsan_mem_shadow_transfer(void *dst, const void *src, uptr len);
} // extern "C"
template <typename T>
-void dfsan_set_label(dfsan_label label, T &data) { // NOLINT
+void dfsan_set_label(dfsan_label label, T &data) {
dfsan_set_label(label, (void *)&data, sizeof(T));
}
void OnUnmap(uptr p, uptr size) const { dfsan_set_label(0, (void *)p, size); }
};
-static const uptr kAllocatorSpace = 0x700000000000ULL;
-static const uptr kMaxAllowedMallocSize = 8UL << 30;
+#if defined(__aarch64__)
+const uptr kAllocatorSpace = 0xE00000000000ULL;
+#else
+const uptr kAllocatorSpace = 0x700000000000ULL;
+#endif
+const uptr kMaxAllowedMallocSize = 8UL << 30;
struct AP64 { // Allocator64 parameters. Deliberately using a short name.
static const uptr kSpaceBeg = kAllocatorSpace;
BufferedStackTrace stack;
ReportAllocationSizeTooBig(size, max_malloc_size, &stack);
}
+ if (UNLIKELY(IsRssLimitExceeded())) {
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ BufferedStackTrace stack;
+ ReportRssLimitExceeded(&stack);
+ }
DFsanThread *t = GetCurrentThread();
void *allocated;
if (t) {
}
static void *dfsan_memcpy(void *dest, const void *src, size_t n) {
- dfsan_label *sdest = shadow_for(dest);
- const dfsan_label *ssrc = shadow_for(src);
- internal_memcpy((void *)sdest, (const void *)ssrc, n * sizeof(dfsan_label));
+ dfsan_mem_shadow_transfer(dest, src, n);
return internal_memcpy(dest, src, n);
}
dfsan_label src_label,
dfsan_label *ret_label) {
size_t dest_len = strlen(dest);
- char *ret = strcat(dest, src); // NOLINT
- dfsan_label *sdest = shadow_for(dest + dest_len);
- const dfsan_label *ssrc = shadow_for(src);
- internal_memcpy((void *)sdest, (const void *)ssrc,
- strlen(src) * sizeof(dfsan_label));
+ char *ret = strcat(dest, src);
+ dfsan_mem_shadow_transfer(dest + dest_len, src, strlen(src));
*ret_label = dest_label;
return ret;
}
dfsan_label *ret_label, dfsan_origin dest_origin, dfsan_origin src_origin,
dfsan_origin *ret_origin) {
size_t dest_len = strlen(dest);
- char *ret = strcat(dest, src); // NOLINT
- dfsan_label *sdest = shadow_for(dest + dest_len);
- const dfsan_label *ssrc = shadow_for(src);
+ char *ret = strcat(dest, src);
size_t src_len = strlen(src);
dfsan_mem_origin_transfer(dest + dest_len, src, src_len);
- internal_memcpy((void *)sdest, (const void *)ssrc,
- src_len * sizeof(dfsan_label));
+ dfsan_mem_shadow_transfer(dest + dest_len, src, src_len);
*ret_label = dest_label;
*ret_origin = dest_origin;
return ret;
static void *DFsanThreadStartFunc(void *arg) {
DFsanThread *t = (DFsanThread *)arg;
SetCurrentThread(t);
+ t->Init();
+ SetSigProcMask(&t->starting_sigset_, nullptr);
return t->ThreadStart();
}
static int dfsan_pthread_create(pthread_t *thread, const pthread_attr_t *attr,
- void *start_routine_trampoline,
void *start_routine, void *arg,
dfsan_label *ret_label,
bool track_origins = false) {
AdjustStackSize((void *)(const_cast<pthread_attr_t *>(attr)));
DFsanThread *t =
- DFsanThread::Create(start_routine_trampoline,
- (thread_callback_t)start_routine, arg, track_origins);
+ DFsanThread::Create((thread_callback_t)start_routine, arg, track_origins);
+ ScopedBlockSignals block(&t->starting_sigset_);
int res = pthread_create(thread, attr, DFsanThreadStartFunc, t);
if (attr == &myattr)
}
SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_pthread_create(
- pthread_t *thread, const pthread_attr_t *attr,
- void *(*start_routine_trampoline)(void *, void *, dfsan_label,
- dfsan_label *),
- void *start_routine, void *arg, dfsan_label thread_label,
- dfsan_label attr_label, dfsan_label start_routine_label,
- dfsan_label arg_label, dfsan_label *ret_label) {
- return dfsan_pthread_create(thread, attr, (void *)start_routine_trampoline,
- start_routine, arg, ret_label);
+ pthread_t *thread, const pthread_attr_t *attr, void *start_routine,
+ void *arg, dfsan_label thread_label, dfsan_label attr_label,
+ dfsan_label start_routine_label, dfsan_label arg_label,
+ dfsan_label *ret_label) {
+ return dfsan_pthread_create(thread, attr, start_routine, arg, ret_label);
}
SANITIZER_INTERFACE_ATTRIBUTE int __dfso_pthread_create(
- pthread_t *thread, const pthread_attr_t *attr,
- void *(*start_routine_trampoline)(void *, void *, dfsan_label,
- dfsan_label *, dfsan_origin,
- dfsan_origin *),
- void *start_routine, void *arg, dfsan_label thread_label,
- dfsan_label attr_label, dfsan_label start_routine_label,
- dfsan_label arg_label, dfsan_label *ret_label, dfsan_origin thread_origin,
+ pthread_t *thread, const pthread_attr_t *attr, void *start_routine,
+ void *arg, dfsan_label thread_label, dfsan_label attr_label,
+ dfsan_label start_routine_label, dfsan_label arg_label,
+ dfsan_label *ret_label, dfsan_origin thread_origin,
dfsan_origin attr_origin, dfsan_origin start_routine_origin,
dfsan_origin arg_origin, dfsan_origin *ret_origin) {
- return dfsan_pthread_create(thread, attr, (void *)start_routine_trampoline,
- start_routine, arg, ret_label, true);
+ return dfsan_pthread_create(thread, attr, start_routine, arg, ret_label,
+ true);
}
SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_pthread_join(pthread_t thread,
}
struct dl_iterate_phdr_info {
- int (*callback_trampoline)(void *callback, struct dl_phdr_info *info,
- size_t size, void *data, dfsan_label info_label,
- dfsan_label size_label, dfsan_label data_label,
- dfsan_label *ret_label);
- void *callback;
- void *data;
-};
-
-struct dl_iterate_phdr_origin_info {
- int (*callback_trampoline)(void *callback, struct dl_phdr_info *info,
- size_t size, void *data, dfsan_label info_label,
- dfsan_label size_label, dfsan_label data_label,
- dfsan_label *ret_label, dfsan_origin info_origin,
- dfsan_origin size_origin, dfsan_origin data_origin,
- dfsan_origin *ret_origin);
- void *callback;
+ int (*callback)(struct dl_phdr_info *info, size_t size, void *data);
void *data;
};
dfsan_set_label(
0, const_cast<char *>(reinterpret_cast<const char *>(info->dlpi_phdr)),
sizeof(*info->dlpi_phdr) * info->dlpi_phnum);
- dfsan_label ret_label;
- return dipi->callback_trampoline(dipi->callback, info, size, dipi->data, 0, 0,
- 0, &ret_label);
-}
-int dl_iterate_phdr_origin_cb(struct dl_phdr_info *info, size_t size,
- void *data) {
- dl_iterate_phdr_origin_info *dipi = (dl_iterate_phdr_origin_info *)data;
- dfsan_set_label(0, *info);
- dfsan_set_label(0, const_cast<char *>(info->dlpi_name),
- strlen(info->dlpi_name) + 1);
- dfsan_set_label(
- 0, const_cast<char *>(reinterpret_cast<const char *>(info->dlpi_phdr)),
- sizeof(*info->dlpi_phdr) * info->dlpi_phnum);
- dfsan_label ret_label;
- dfsan_origin ret_origin;
- return dipi->callback_trampoline(dipi->callback, info, size, dipi->data, 0, 0,
- 0, &ret_label, 0, 0, 0, &ret_origin);
+ dfsan_clear_thread_local_state();
+ return dipi->callback(info, size, dipi->data);
}
SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_dl_iterate_phdr(
- int (*callback_trampoline)(void *callback, struct dl_phdr_info *info,
- size_t size, void *data, dfsan_label info_label,
- dfsan_label size_label, dfsan_label data_label,
- dfsan_label *ret_label),
- void *callback, void *data, dfsan_label callback_label,
- dfsan_label data_label, dfsan_label *ret_label) {
- dl_iterate_phdr_info dipi = { callback_trampoline, callback, data };
+ int (*callback)(struct dl_phdr_info *info, size_t size, void *data),
+ void *data, dfsan_label callback_label, dfsan_label data_label,
+ dfsan_label *ret_label) {
+ dl_iterate_phdr_info dipi = {callback, data};
*ret_label = 0;
return dl_iterate_phdr(dl_iterate_phdr_cb, &dipi);
}
SANITIZER_INTERFACE_ATTRIBUTE int __dfso_dl_iterate_phdr(
- int (*callback_trampoline)(void *callback, struct dl_phdr_info *info,
- size_t size, void *data, dfsan_label info_label,
- dfsan_label size_label, dfsan_label data_label,
- dfsan_label *ret_label, dfsan_origin info_origin,
- dfsan_origin size_origin,
- dfsan_origin data_origin,
- dfsan_origin *ret_origin),
- void *callback, void *data, dfsan_label callback_label,
- dfsan_label data_label, dfsan_label *ret_label,
- dfsan_origin callback_origin, dfsan_origin data_origin,
- dfsan_origin *ret_origin) {
- dl_iterate_phdr_origin_info dipi = {callback_trampoline, callback, data};
+ int (*callback)(struct dl_phdr_info *info, size_t size, void *data),
+ void *data, dfsan_label callback_label, dfsan_label data_label,
+ dfsan_label *ret_label, dfsan_origin callback_origin,
+ dfsan_origin data_origin, dfsan_origin *ret_origin) {
+ dl_iterate_phdr_info dipi = {callback, data};
*ret_label = 0;
- return dl_iterate_phdr(dl_iterate_phdr_origin_cb, &dipi);
+ return dl_iterate_phdr(dl_iterate_phdr_cb, &dipi);
}
// This function is only available for glibc 2.27 or newer. Mark it weak so
return __dfsw_get_current_dir_name(ret_label);
}
+// This function is only available for glibc 2.25 or newer. Mark it weak so
+// linking succeeds with older glibcs.
+SANITIZER_WEAK_ATTRIBUTE int getentropy(void *buffer, size_t length);
+
+SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_getentropy(void *buffer, size_t length,
+ dfsan_label buffer_label,
+ dfsan_label length_label,
+ dfsan_label *ret_label) {
+ int ret = getentropy(buffer, length);
+ if (ret == 0) {
+ dfsan_set_label(0, buffer, length);
+ }
+ *ret_label = 0;
+ return ret;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE int __dfso_getentropy(void *buffer, size_t length,
+ dfsan_label buffer_label,
+ dfsan_label length_label,
+ dfsan_label *ret_label,
+ dfsan_origin buffer_origin,
+ dfsan_origin length_origin,
+ dfsan_origin *ret_origin) {
+ return __dfsw_getentropy(buffer, length, buffer_label, length_label,
+ ret_label);
+}
+
SANITIZER_INTERFACE_ATTRIBUTE
int __dfsw_gethostname(char *name, size_t len, dfsan_label name_label,
dfsan_label len_label, dfsan_label *ret_label) {
SANITIZER_INTERFACE_ATTRIBUTE
char *__dfsw_strcpy(char *dest, const char *src, dfsan_label dst_label,
dfsan_label src_label, dfsan_label *ret_label) {
- char *ret = strcpy(dest, src); // NOLINT
+ char *ret = strcpy(dest, src);
if (ret) {
- internal_memcpy(shadow_for(dest), shadow_for(src),
- sizeof(dfsan_label) * (strlen(src) + 1));
+ dfsan_mem_shadow_transfer(dest, src, strlen(src) + 1);
}
*ret_label = dst_label;
return ret;
dfsan_label src_label, dfsan_label *ret_label,
dfsan_origin dst_origin, dfsan_origin src_origin,
dfsan_origin *ret_origin) {
- char *ret = strcpy(dest, src); // NOLINT
+ char *ret = strcpy(dest, src);
if (ret) {
size_t str_len = strlen(src) + 1;
dfsan_mem_origin_transfer(dest, src, str_len);
- internal_memcpy(shadow_for(dest), shadow_for(src),
- sizeof(dfsan_label) * str_len);
+ dfsan_mem_shadow_transfer(dest, src, str_len);
}
*ret_label = dst_label;
*ret_origin = dst_origin;
SignalHandlerScope signal_handler_scope;
ScopedClearThreadLocalState scoped_clear_tls;
- // Clear shadows for all inputs provided by system. This is why DFSan
- // instrumentation generates a trampoline function to each function pointer,
- // and uses the trampoline to clear shadows. However sigaction does not use
- // a function pointer directly, so we have to do this manually.
+ // Clear shadows for all inputs provided by system.
dfsan_clear_arg_tls(0, sizeof(dfsan_label));
typedef void (*signal_cb)(int x);
}
SANITIZER_INTERFACE_ATTRIBUTE
-sighandler_t __dfsw_signal(int signum,
- void *(*handler_trampoline)(void *, int, dfsan_label,
- dfsan_label *),
- sighandler_t handler, dfsan_label signum_label,
- dfsan_label handler_label, dfsan_label *ret_label) {
+sighandler_t __dfsw_signal(int signum, sighandler_t handler,
+ dfsan_label signum_label, dfsan_label handler_label,
+ dfsan_label *ret_label) {
return dfsan_signal(signum, handler, ret_label);
}
SANITIZER_INTERFACE_ATTRIBUTE
-sighandler_t __dfso_signal(
- int signum,
- void *(*handler_trampoline)(void *, int, dfsan_label, dfsan_label *,
- dfsan_origin, dfsan_origin *),
- sighandler_t handler, dfsan_label signum_label, dfsan_label handler_label,
- dfsan_label *ret_label, dfsan_origin signum_origin,
- dfsan_origin handler_origin, dfsan_origin *ret_origin) {
+sighandler_t __dfso_signal(int signum, sighandler_t handler,
+ dfsan_label signum_label, dfsan_label handler_label,
+ dfsan_label *ret_label, dfsan_origin signum_origin,
+ dfsan_origin handler_origin,
+ dfsan_origin *ret_origin) {
return dfsan_signal(signum, handler, ret_label);
}
addrlen_label, ret_label);
}
-// Type of the trampoline function passed to the custom version of
-// dfsan_set_write_callback.
-typedef void (*write_trampoline_t)(
- void *callback,
- int fd, const void *buf, ssize_t count,
- dfsan_label fd_label, dfsan_label buf_label, dfsan_label count_label);
-
-typedef void (*write_origin_trampoline_t)(
- void *callback, int fd, const void *buf, ssize_t count,
- dfsan_label fd_label, dfsan_label buf_label, dfsan_label count_label,
- dfsan_origin fd_origin, dfsan_origin buf_origin, dfsan_origin count_origin);
+// Type of the function passed to dfsan_set_write_callback.
+typedef void (*write_dfsan_callback_t)(int fd, const void *buf, ssize_t count);
// Calls to dfsan_set_write_callback() set the values in this struct.
// Calls to the custom version of write() read (and invoke) them.
static struct {
- write_trampoline_t write_callback_trampoline = nullptr;
- void *write_callback = nullptr;
+ write_dfsan_callback_t write_callback = nullptr;
} write_callback_info;
-static struct {
- write_origin_trampoline_t write_callback_trampoline = nullptr;
- void *write_callback = nullptr;
-} write_origin_callback_info;
-
-SANITIZER_INTERFACE_ATTRIBUTE void
-__dfsw_dfsan_set_write_callback(
- write_trampoline_t write_callback_trampoline,
- void *write_callback,
- dfsan_label write_callback_label,
+SANITIZER_INTERFACE_ATTRIBUTE void __dfsw_dfsan_set_write_callback(
+ write_dfsan_callback_t write_callback, dfsan_label write_callback_label,
dfsan_label *ret_label) {
- write_callback_info.write_callback_trampoline = write_callback_trampoline;
write_callback_info.write_callback = write_callback;
}
SANITIZER_INTERFACE_ATTRIBUTE void __dfso_dfsan_set_write_callback(
- write_origin_trampoline_t write_callback_trampoline, void *write_callback,
- dfsan_label write_callback_label, dfsan_label *ret_label,
- dfsan_origin write_callback_origin, dfsan_origin *ret_origin) {
- write_origin_callback_info.write_callback_trampoline =
- write_callback_trampoline;
- write_origin_callback_info.write_callback = write_callback;
+ write_dfsan_callback_t write_callback, dfsan_label write_callback_label,
+ dfsan_label *ret_label, dfsan_origin write_callback_origin,
+ dfsan_origin *ret_origin) {
+ write_callback_info.write_callback = write_callback;
+}
+
+static inline void setup_tls_args_for_write_callback(
+ dfsan_label fd_label, dfsan_label buf_label, dfsan_label count_label,
+ bool origins, dfsan_origin fd_origin, dfsan_origin buf_origin,
+ dfsan_origin count_origin) {
+ // The callback code will expect argument shadow labels in the args TLS,
+ // and origin labels in the origin args TLS.
+ // Previously this was done by a trampoline, but we want to remove this:
+ // https://github.com/llvm/llvm-project/issues/54172
+ //
+ // Instead, this code is manually setting up the args TLS data.
+ //
+ // The offsets used need to correspond with the instrumentation code,
+ // see llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
+ // DFSanFunction::getShadowForTLSArgument.
+ // https://github.com/llvm/llvm-project/blob/0acc9e4b5edd8b39ff3d4c6d0e17f02007671c4e/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp#L1684
+ // https://github.com/llvm/llvm-project/blob/0acc9e4b5edd8b39ff3d4c6d0e17f02007671c4e/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp#L125
+ //
+ // Here the arguments are all primitives, but it can be more complex
+ // to compute offsets for array/aggregate type arguments.
+ //
+ // TODO(browneee): Consider a builtin to improve maintainabliity.
+ // With a builtin, we would provide the argument labels via builtin,
+ // and the builtin would reuse parts of the instrumentation code to ensure
+ // that this code and the instrumentation can never be out of sync.
+ // Note: Currently DFSan instrumentation does not run on this code, so
+ // the builtin may need to be handled outside DFSan instrumentation.
+ dfsan_set_arg_tls(0, fd_label);
+ dfsan_set_arg_tls(1, buf_label);
+ dfsan_set_arg_tls(2, count_label);
+ if (origins) {
+ dfsan_set_arg_origin_tls(0, fd_origin);
+ dfsan_set_arg_origin_tls(1, buf_origin);
+ dfsan_set_arg_origin_tls(2, count_origin);
+ }
}
SANITIZER_INTERFACE_ATTRIBUTE int
dfsan_label fd_label, dfsan_label buf_label,
dfsan_label count_label, dfsan_label *ret_label) {
if (write_callback_info.write_callback) {
- write_callback_info.write_callback_trampoline(
- write_callback_info.write_callback,
- fd, buf, count,
- fd_label, buf_label, count_label);
+ setup_tls_args_for_write_callback(fd_label, buf_label, count_label, false,
+ 0, 0, 0);
+ write_callback_info.write_callback(fd, buf, count);
}
*ret_label = 0;
dfsan_label buf_label, dfsan_label count_label, dfsan_label *ret_label,
dfsan_origin fd_origin, dfsan_origin buf_origin, dfsan_origin count_origin,
dfsan_origin *ret_origin) {
- if (write_origin_callback_info.write_callback) {
- write_origin_callback_info.write_callback_trampoline(
- write_origin_callback_info.write_callback, fd, buf, count, fd_label,
- buf_label, count_label, fd_origin, buf_origin, count_origin);
+ if (write_callback_info.write_callback) {
+ setup_tls_args_for_write_callback(fd_label, buf_label, count_label, true,
+ fd_origin, buf_origin, count_origin);
+ write_callback_info.write_callback(fd, buf, count);
}
*ret_label = 0;
formatter.num_written_bytes(retval));
}
va_labels++;
- internal_memcpy(shadow_for(formatter.str_cur()), shadow_for(arg),
- sizeof(dfsan_label) *
- formatter.num_written_bytes(retval));
+ dfsan_mem_shadow_transfer(formatter.str_cur(), arg,
+ formatter.num_written_bytes(retval));
end_fmt = true;
break;
}
SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_pc_guard, u32 *) {}
SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_pc_guard_init, u32 *,
u32 *) {}
-SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_pcs_init, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_pcs_init, const uptr *beg,
+ const uptr *end) {}
SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_pc_indir, void) {}
SANITIZER_INTERFACE_WEAK_DEF(void, __dfsw___sanitizer_cov_trace_cmp, void) {}
#include "dfsan/dfsan.h"
#include "dfsan/dfsan_thread.h"
#include "interception/interception.h"
+#include "sanitizer_common/sanitizer_allocator_dlsym.h"
#include "sanitizer_common/sanitizer_allocator_interface.h"
#include "sanitizer_common/sanitizer_common.h"
#include "sanitizer_common/sanitizer_errno.h"
using namespace __sanitizer;
-namespace {
+static bool interceptors_initialized;
-bool interceptors_initialized;
-
-} // namespace
+struct DlsymAlloc : public DlSymAllocator<DlsymAlloc> {
+ static bool UseImpl() { return !__dfsan::dfsan_inited; }
+};
INTERCEPTOR(void *, reallocarray, void *ptr, SIZE_T nmemb, SIZE_T size) {
return __dfsan::dfsan_reallocarray(ptr, nmemb, size);
return __dfsan::dfsan_aligned_alloc(alignment, size);
}
-static uptr allocated_for_dlsym;
-static const uptr kDlsymAllocPoolSize = 1024;
-static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize];
-
-static bool IsInDlsymAllocPool(const void *ptr) {
- uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
- return off < sizeof(alloc_memory_for_dlsym);
-}
-
-static void *AllocateFromLocalPool(uptr size_in_bytes) {
- uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize;
- void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym];
- allocated_for_dlsym += size_in_words;
- CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize);
- return mem;
-}
-
INTERCEPTOR(void *, calloc, SIZE_T nmemb, SIZE_T size) {
- if (UNLIKELY(!__dfsan::dfsan_inited))
- // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
- return AllocateFromLocalPool(nmemb * size);
+ if (DlsymAlloc::Use())
+ return DlsymAlloc::Callocate(nmemb, size);
return __dfsan::dfsan_calloc(nmemb, size);
}
INTERCEPTOR(void *, realloc, void *ptr, SIZE_T size) {
- if (UNLIKELY(IsInDlsymAllocPool(ptr))) {
- uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
- uptr copy_size = Min(size, kDlsymAllocPoolSize - offset);
- void *new_ptr;
- if (UNLIKELY(!__dfsan::dfsan_inited)) {
- new_ptr = AllocateFromLocalPool(copy_size);
- } else {
- copy_size = size;
- new_ptr = __dfsan::dfsan_malloc(copy_size);
- }
- internal_memcpy(new_ptr, ptr, copy_size);
- return new_ptr;
- }
+ if (DlsymAlloc::Use() || DlsymAlloc::PointerIsMine(ptr))
+ return DlsymAlloc::Realloc(ptr, size);
return __dfsan::dfsan_realloc(ptr, size);
}
INTERCEPTOR(void *, malloc, SIZE_T size) {
- if (UNLIKELY(!__dfsan::dfsan_inited))
- // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym.
- return AllocateFromLocalPool(size);
+ if (DlsymAlloc::Use())
+ return DlsymAlloc::Allocate(size);
return __dfsan::dfsan_malloc(size);
}
INTERCEPTOR(void, free, void *ptr) {
- if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr)))
+ if (!ptr)
return;
+ if (DlsymAlloc::PointerIsMine(ptr))
+ return DlsymAlloc::Free(ptr);
return __dfsan::dfsan_deallocate(ptr);
}
INTERCEPTOR(void, cfree, void *ptr) {
- if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr)))
+ if (!ptr)
return;
+ if (DlsymAlloc::PointerIsMine(ptr))
+ return DlsymAlloc::Free(ptr);
return __dfsan::dfsan_deallocate(ptr);
}
if (__dfsan::dfsan_init_is_running) \
return REAL(func)(__VA_ARGS__); \
ENSURE_DFSAN_INITED(); \
- dfsan_set_label(0, __errno_location(), sizeof(int)); /* NOLINT */
+ dfsan_set_label(0, __errno_location(), sizeof(int));
INTERCEPTOR(void *, mmap, void *addr, SIZE_T length, int prot, int flags,
int fd, OFF_T offset) {
if (common_flags()->detect_write_exec)
- ReportMmapWriteExec(prot);
+ ReportMmapWriteExec(prot, flags);
if (!__dfsan::dfsan_inited)
return (void *)internal_mmap(addr, length, prot, flags, fd, offset);
COMMON_INTERCEPTOR_ENTER(mmap, addr, length, prot, flags, fd, offset);
INTERCEPTOR(void *, mmap64, void *addr, SIZE_T length, int prot, int flags,
int fd, OFF64_T offset) {
if (common_flags()->detect_write_exec)
- ReportMmapWriteExec(prot);
+ ReportMmapWriteExec(prot, flags);
if (!__dfsan::dfsan_inited)
return (void *)internal_mmap(addr, length, prot, flags, fd, offset);
COMMON_INTERCEPTOR_ENTER(mmap64, addr, length, prot, flags, fd, offset);
#if SANITIZER_LINUX && SANITIZER_WORDSIZE == 64
+# if defined(__aarch64__)
+// The mapping assumes 48-bit VMA. AArch64 maps:
+// - 0x0000000000000-0x0100000000000: 39/42/48-bits program own segments
+// - 0x0a00000000000-0x0b00000000000: 48-bits PIE program segments
+// Ideally, this would extend to 0x0c00000000000 (2^45 bytes - the
+// maximum ASLR region for 48-bit VMA) but it is too hard to fit in
+// the larger app/shadow/origin regions.
+// - 0x0e00000000000-0x1000000000000: 48-bits libraries segments
+const MappingDesc kMemoryLayout[] = {
+ {0X0000000000000, 0X0100000000000, MappingDesc::APP, "app-10-13"},
+ {0X0100000000000, 0X0200000000000, MappingDesc::SHADOW, "shadow-14"},
+ {0X0200000000000, 0X0300000000000, MappingDesc::INVALID, "invalid"},
+ {0X0300000000000, 0X0400000000000, MappingDesc::ORIGIN, "origin-14"},
+ {0X0400000000000, 0X0600000000000, MappingDesc::SHADOW, "shadow-15"},
+ {0X0600000000000, 0X0800000000000, MappingDesc::ORIGIN, "origin-15"},
+ {0X0800000000000, 0X0A00000000000, MappingDesc::INVALID, "invalid"},
+ {0X0A00000000000, 0X0B00000000000, MappingDesc::APP, "app-14"},
+ {0X0B00000000000, 0X0C00000000000, MappingDesc::SHADOW, "shadow-10-13"},
+ {0X0C00000000000, 0X0D00000000000, MappingDesc::INVALID, "invalid"},
+ {0X0D00000000000, 0X0E00000000000, MappingDesc::ORIGIN, "origin-10-13"},
+ {0X0E00000000000, 0X1000000000000, MappingDesc::APP, "app-15"},
+};
+# define MEM_TO_SHADOW(mem) ((uptr)mem ^ 0xB00000000000ULL)
+# define SHADOW_TO_ORIGIN(shadow) (((uptr)(shadow)) + 0x200000000000ULL)
+
+# else
// All of the following configurations are supported.
// ASLR disabled: main executable and DSOs at 0x555550000000
// PIE and ASLR: main executable and DSOs at 0x7f0000000000
{0x600000000000ULL, 0x610000000000ULL, MappingDesc::ORIGIN, "origin-1"},
{0x610000000000ULL, 0x700000000000ULL, MappingDesc::INVALID, "invalid"},
{0x700000000000ULL, 0x800000000000ULL, MappingDesc::APP, "app-3"}};
-# define MEM_TO_SHADOW(mem) (((uptr)(mem)) ^ 0x500000000000ULL)
-# define SHADOW_TO_ORIGIN(mem) (((uptr)(mem)) + 0x100000000000ULL)
+# define MEM_TO_SHADOW(mem) (((uptr)(mem)) ^ 0x500000000000ULL)
+# define SHADOW_TO_ORIGIN(mem) (((uptr)(mem)) + 0x100000000000ULL)
+# endif
#else
# error "Unsupported platform"
namespace __dfsan {
-DFsanThread *DFsanThread::Create(void *start_routine_trampoline,
- thread_callback_t start_routine, void *arg,
+DFsanThread *DFsanThread::Create(thread_callback_t start_routine, void *arg,
bool track_origins) {
uptr PageSize = GetPageSizeCached();
uptr size = RoundUpTo(sizeof(DFsanThread), PageSize);
DFsanThread *thread = (DFsanThread *)MmapOrDie(size, __func__);
- thread->start_routine_trampoline_ = start_routine_trampoline;
thread->start_routine_ = start_routine;
thread->arg_ = arg;
thread->track_origins_ = track_origins;
}
thread_return_t DFsanThread::ThreadStart() {
- Init();
-
if (!start_routine_) {
// start_routine_ == 0 if we're on the main thread or on one of the
// OS X libdispatch worker threads. But nobody is supposed to call
return 0;
}
- CHECK(start_routine_trampoline_);
-
- typedef void *(*thread_callback_trampoline_t)(void *, void *, dfsan_label,
- dfsan_label *);
- typedef void *(*thread_callback_origin_trampoline_t)(
- void *, void *, dfsan_label, dfsan_label *, dfsan_origin, dfsan_origin *);
-
- dfsan_label ret_label;
- if (!track_origins_)
- return ((thread_callback_trampoline_t)
- start_routine_trampoline_)((void *)start_routine_, arg_, 0,
- &ret_label);
+ // The only argument is void* arg.
+ //
+ // We have never supported propagating the pointer arg as tainted,
+ // __dfsw_pthread_create/__dfso_pthread_create ignore the taint label.
+ // Note that the bytes pointed-to (probably the much more common case)
+ // can still have taint labels attached to them.
+ dfsan_clear_thread_local_state();
- dfsan_origin ret_origin;
- return ((thread_callback_origin_trampoline_t)
- start_routine_trampoline_)((void *)start_routine_, arg_, 0,
- &ret_label, 0, &ret_origin);
+ return start_routine_(arg_);
}
DFsanThread::StackBounds DFsanThread::GetStackBounds() const {
-//===-- dfsan_thread.h -------------------------------------------*- C++
-//-*-===//
+//===-- dfsan_thread.h ------------------------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
#include "dfsan_allocator.h"
#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_posix.h"
namespace __dfsan {
// NOTE: There is no DFsanThread constructor. It is allocated
// via mmap() and *must* be valid in zero-initialized state.
- static DFsanThread *Create(void *start_routine_trampoline,
- thread_callback_t start_routine, void *arg,
+ static DFsanThread *Create(thread_callback_t start_routine, void *arg,
bool track_origins = false);
static void TSDDtor(void *tsd);
void Destroy();
DFsanThreadLocalMallocStorage &malloc_storage() { return malloc_storage_; }
int destructor_iterations_;
+ __sanitizer_sigset_t starting_sigset_;
private:
void SetThreadStackAndTls();
bool AddrIsInStack(uptr addr);
- void *start_routine_trampoline_;
thread_callback_t start_routine_;
void *arg_;
bool track_origins_;
fun:dfsan_flush=discard
fun:dfsan_print_origin_trace=uninstrumented
fun:dfsan_print_origin_trace=discard
+fun:dfsan_print_origin_id_trace=uninstrumented
+fun:dfsan_print_origin_id_trace=discard
fun:dfsan_sprint_origin_trace=uninstrumented
fun:dfsan_sprint_origin_trace=discard
+fun:dfsan_sprint_origin_id_trace=uninstrumented
+fun:dfsan_sprint_origin_id_trace=discard
fun:dfsan_sprint_stack_trace=uninstrumented
fun:dfsan_sprint_stack_trace=discard
fun:dfsan_get_origin=uninstrumented
fun:dfsan_get_origin=custom
+fun:dfsan_read_origin_of_first_taint=uninstrumented
+fun:dfsan_read_origin_of_first_taint=discard
fun:dfsan_get_init_origin=uninstrumented
fun:dfsan_get_init_origin=discard
fun:dfsan_get_track_origins=uninstrumented
fun:dfsan_get_track_origins=discard
+fun:dfsan_set_conditional_callback=uninstrumented
+fun:dfsan_set_conditional_callback=discard
+fun:dfsan_get_labels_in_signal_conditional=uninstrumented
+fun:dfsan_get_labels_in_signal_conditional=discard
+fun:dfsan_set_reaches_function_callback=uninstrumented
+fun:dfsan_set_reaches_function_callback=discard
+fun:dfsan_get_labels_in_signal_reaches_function=uninstrumented
+fun:dfsan_get_labels_in_signal_reaches_function=discard
+fun:dfsan_reaches_function_callback=uninstrumented
+fun:dfsan_reaches_function_callback=discard
###############################################################################
# glibc
fun:fstat=custom
fun:getcwd=custom
fun:get_current_dir_name=custom
+fun:getentropy=custom
fun:gethostname=custom
fun:getpeername=custom
fun:getrlimit=custom
fun:strstr=custom
# Functions which take action based on global state, such as running a callback
-# set by a sepperate function.
+# set by a separate function.
fun:write=custom
# Functions that take a callback (wrap the callback manually).
fun:getdomainname=uninstrumented
fun:getdtablesize=uninstrumented
fun:getegid=uninstrumented
+fun:getentropy=uninstrumented
fun:getenv=uninstrumented
fun:geteuid=uninstrumented
fun:getfsent=uninstrumented
# uninstrumented, thus allowing the instrumentation pass to treat calls to those
# functions correctly.
+# Typical usage will list runtime libraries which are not instrumented by dfsan.
+# This would include libc, and compiler builtins.
+#
+# ./build-libc-list.py \
+# --lib-file=/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2 \
+# --lib-file=/lib/x86_64-linux-gnu/libanl.so.1 \
+# --lib-file=/lib/x86_64-linux-gnu/libBrokenLocale.so.1 \
+# --lib-file=/lib/x86_64-linux-gnu/libcidn.so.1 \
+# --lib-file=/lib/x86_64-linux-gnu/libcrypt.so.1 \
+# --lib-file=/lib/x86_64-linux-gnu/libc.so.6 \
+# --lib-file=/lib/x86_64-linux-gnu/libdl.so.2 \
+# --lib-file=/lib/x86_64-linux-gnu/libm.so.6 \
+# --lib-file=/lib/x86_64-linux-gnu/libnsl.so.1 \
+# --lib-file=/lib/x86_64-linux-gnu/libpthread.so.0 \
+# --lib-file=/lib/x86_64-linux-gnu/libresolv.so.2 \
+# --lib-file=/lib/x86_64-linux-gnu/librt.so.1 \
+# --lib-file=/lib/x86_64-linux-gnu/libthread_db.so.1 \
+# --lib-file=/lib/x86_64-linux-gnu/libutil.so.1 \
+# --lib-file=/usr/lib/x86_64-linux-gnu/libc_nonshared.a \
+# --lib-file=/usr/lib/x86_64-linux-gnu/libpthread_nonshared.a \
+# --lib-file=/lib/x86_64-linux-gnu/libgcc_s.so.1 \
+# --lib-file=/usr/lib/gcc/x86_64-linux-gnu/4.6/libgcc.a \
+# --error-missing-lib
+
import os
import subprocess
import sys
p = OptionParser()
-p.add_option('--libc-dso-path', metavar='PATH',
- help='path to libc DSO directory',
- default='/lib/x86_64-linux-gnu')
-p.add_option('--libc-archive-path', metavar='PATH',
- help='path to libc archive directory',
- default='/usr/lib/x86_64-linux-gnu')
-
-p.add_option('--libgcc-dso-path', metavar='PATH',
- help='path to libgcc DSO directory',
- default='/lib/x86_64-linux-gnu')
-p.add_option('--libgcc-archive-path', metavar='PATH',
- help='path to libgcc archive directory',
- default='/usr/lib/gcc/x86_64-linux-gnu/4.6')
+p.add_option('--lib-file', action='append', metavar='PATH',
+ help='Specific library files to add.',
+ default=[])
-p.add_option('--with-libstdcxx', action='store_true',
- dest='with_libstdcxx',
- help='include libstdc++ in the list (inadvisable)')
-p.add_option('--libstdcxx-dso-path', metavar='PATH',
- help='path to libstdc++ DSO directory',
- default='/usr/lib/x86_64-linux-gnu')
+p.add_option('--error-missing-lib', action='store_true',
+ help='Make this script exit with an error code if any library is missing.',
+ dest='error_missing_lib', default=False)
(options, args) = p.parse_args()
-libs = [os.path.join(options.libc_dso_path, name) for name in
- ['ld-linux-x86-64.so.2',
- 'libanl.so.1',
- 'libBrokenLocale.so.1',
- 'libcidn.so.1',
- 'libcrypt.so.1',
- 'libc.so.6',
- 'libdl.so.2',
- 'libm.so.6',
- 'libnsl.so.1',
- 'libpthread.so.0',
- 'libresolv.so.2',
- 'librt.so.1',
- 'libthread_db.so.1',
- 'libutil.so.1']]
-libs += [os.path.join(options.libc_archive_path, name) for name in
- ['libc_nonshared.a',
- 'libpthread_nonshared.a']]
-
-libs.append(os.path.join(options.libgcc_dso_path, 'libgcc_s.so.1'))
-libs.append(os.path.join(options.libgcc_archive_path, 'libgcc.a'))
-
-if options.with_libstdcxx:
- libs.append(os.path.join(options.libstdcxx_dso_path, 'libstdc++.so.6'))
+libs = options.lib_file
+if not libs:
+ print >> sys.stderr, 'No libraries provided.'
+ exit(1)
+missing_lib = False
functions = []
for l in libs:
if os.path.exists(l):
functions += defined_function_list(l)
else:
+ missing_lib = True
print >> sys.stderr, 'warning: library %s not found' % l
+if options.error_missing_lib and missing_lib:
+ print >> sys.stderr, 'Exiting with failure code due to missing library.'
+ exit(1)
+
functions = list(set(functions))
functions.sort()
FuzzerExtFunctionsWeak.cpp
FuzzerExtFunctionsWindows.cpp
FuzzerExtraCounters.cpp
+ FuzzerExtraCountersDarwin.cpp
+ FuzzerExtraCountersWindows.cpp
FuzzerFork.cpp
FuzzerIO.cpp
FuzzerIOPosix.cpp
append_list_if(COMPILER_RT_HAS_NOSTDINCXX_FLAG -nostdinc++ LIBFUZZER_CFLAGS)
elseif(TARGET cxx-headers OR HAVE_LIBCXX)
# libFuzzer uses C++ standard library headers.
+ list(APPEND LIBFUZZER_CFLAGS ${COMPILER_RT_CXX_CFLAGS})
set(LIBFUZZER_DEPS cxx-headers)
endif()
append_list_if(COMPILER_RT_HAS_OMIT_FRAME_POINTER_FLAG -fno-omit-frame-pointer LIBFUZZER_CFLAGS)
if (CMAKE_CXX_FLAGS MATCHES "fsanitize-coverage")
- list(APPEND LIBFUZZER_CFLAGS -fno-sanitize-coverage=trace-pc-guard,edge,trace-cmp,indirect-calls,8bit-counters)
+ list(APPEND LIBFUZZER_CFLAGS -fsanitize-coverage=0)
endif()
if(MSVC)
# Silence warnings by turning off exceptions in MSVC headers and avoid an
- # error by unecessarily defining thread_local when it isn't even used on
+ # error by unnecessarily defining thread_local when it isn't even used on
# Windows.
list(APPEND LIBFUZZER_CFLAGS -D_HAS_EXCEPTIONS=0)
else()
COMPILER_RT_LIBCXX_PATH AND
COMPILER_RT_LIBCXXABI_PATH)
macro(partially_link_libcxx name dir arch)
- if(${arch} MATCHES "i386")
- set(EMULATION_ARGUMENT "-m" "elf_i386")
- else()
- set(EMULATION_ARGUMENT "")
+ get_target_flags_for_arch(${arch} target_cflags)
+ if(CMAKE_CXX_COMPILER_ID MATCHES Clang)
+ get_compiler_rt_target(${arch} target)
+ set(target_cflags --target=${target} ${target_cflags})
endif()
set(cxx_${arch}_merge_dir "${CMAKE_CURRENT_BINARY_DIR}/cxx_${arch}_merge.dir")
file(MAKE_DIRECTORY ${cxx_${arch}_merge_dir})
add_custom_command(TARGET clang_rt.${name}-${arch} POST_BUILD
- COMMAND ${CMAKE_LINKER} ${EMULATION_ARGUMENT} --whole-archive "$<TARGET_LINKER_FILE:clang_rt.${name}-${arch}>" --no-whole-archive ${dir}/lib/libc++.a -r -o ${name}.o
+ COMMAND ${CMAKE_CXX_COMPILER} ${target_cflags} -Wl,--whole-archive "$<TARGET_LINKER_FILE:clang_rt.${name}-${arch}>" -Wl,--no-whole-archive ${dir}/lib/libc++.a -r -o ${name}.o
COMMAND ${CMAKE_OBJCOPY} --localize-hidden ${name}.o
COMMAND ${CMAKE_COMMAND} -E remove "$<TARGET_LINKER_FILE:clang_rt.${name}-${arch}>"
COMMAND ${CMAKE_AR} qcs "$<TARGET_LINKER_FILE:clang_rt.${name}-${arch}>" ${name}.o
CMAKE_ARGS -DCMAKE_CXX_COMPILER_WORKS=ON
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
-DLIBCXXABI_ENABLE_EXCEPTIONS=OFF
- -DLIBCXX_ABI_NAMESPACE=__Fuzzer)
+ -DLIBCXX_ABI_NAMESPACE=__Fuzzer
+ -DLIBCXX_ENABLE_EXCEPTIONS=OFF)
target_compile_options(RTfuzzer.${arch} PRIVATE -isystem ${LIBCXX_${arch}_PREFIX}/include/c++/v1)
add_dependencies(RTfuzzer.${arch} libcxx_fuzzer_${arch}-build)
target_compile_options(RTfuzzer_main.${arch} PRIVATE -isystem ${LIBCXX_${arch}_PREFIX}/include/c++/v1)
#if !defined(_M_ARM) && !defined(_M_X64)
// Scan the high 32 bits.
if (_BitScanReverse(&LeadZeroIdx, static_cast<unsigned long>(X >> 32)))
- return static_cast<int>(63 - (LeadZeroIdx + 32)); // Create a bit offset from the MSB.
+ return static_cast<int>(
+ 63 - (LeadZeroIdx + 32)); // Create a bit offset from the MSB.
// Scan the low 32 bits.
if (_BitScanReverse(&LeadZeroIdx, static_cast<unsigned long>(X)))
return static_cast<int>(63 - LeadZeroIdx);
Command() : CombinedOutAndErr(false) {}
- explicit Command(const Vector<std::string> &ArgsToAdd)
+ explicit Command(const std::vector<std::string> &ArgsToAdd)
: Args(ArgsToAdd), CombinedOutAndErr(false) {}
explicit Command(const Command &Other)
// Gets all of the current command line arguments, **including** those after
// "-ignore-remaining-args=1".
- const Vector<std::string> &getArguments() const { return Args; }
+ const std::vector<std::string> &getArguments() const { return Args; }
// Adds the given argument before "-ignore_remaining_args=1", or at the end
// if that flag isn't present.
// Adds all given arguments before "-ignore_remaining_args=1", or at the end
// if that flag isn't present.
- void addArguments(const Vector<std::string> &ArgsToAdd) {
+ void addArguments(const std::vector<std::string> &ArgsToAdd) {
Args.insert(endMutableArgs(), ArgsToAdd.begin(), ArgsToAdd.end());
}
Command(Command &&Other) = delete;
Command &operator=(Command &&Other) = delete;
- Vector<std::string>::iterator endMutableArgs() {
+ std::vector<std::string>::iterator endMutableArgs() {
return std::find(Args.begin(), Args.end(), ignoreRemainingArgs());
}
- Vector<std::string>::const_iterator endMutableArgs() const {
+ std::vector<std::string>::const_iterator endMutableArgs() const {
return std::find(Args.begin(), Args.end(), ignoreRemainingArgs());
}
// The command arguments. Args[0] is the command name.
- Vector<std::string> Args;
+ std::vector<std::string> Args;
// True indicates stderr is redirected to stdout.
bool CombinedOutAndErr;
bool MayDeleteFile = false;
bool Reduced = false;
bool HasFocusFunction = false;
- Vector<uint32_t> UniqFeatureSet;
- Vector<uint8_t> DataFlowTraceForFocusFunction;
+ std::vector<uint32_t> UniqFeatureSet;
+ std::vector<uint8_t> DataFlowTraceForFocusFunction;
// Power schedule.
bool NeedsEnergyUpdate = false;
double Energy = 0.0;
double SumIncidence = 0.0;
- Vector<std::pair<uint32_t, uint16_t>> FeatureFreqs;
+ std::vector<std::pair<uint32_t, uint16_t>> FeatureFreqs;
// Delete feature Idx and its frequency from FeatureFreqs.
bool DeleteFeatureFreq(uint32_t Idx) {
InputInfo *AddToCorpus(const Unit &U, size_t NumFeatures, bool MayDeleteFile,
bool HasFocusFunction, bool NeverReduce,
std::chrono::microseconds TimeOfUnit,
- const Vector<uint32_t> &FeatureSet,
+ const std::vector<uint32_t> &FeatureSet,
const DataFlowTrace &DFT, const InputInfo *BaseII) {
assert(!U.empty());
if (FeatureDebug)
}
// Debug-only
- void PrintFeatureSet(const Vector<uint32_t> &FeatureSet) {
+ void PrintFeatureSet(const std::vector<uint32_t> &FeatureSet) {
if (!FeatureDebug) return;
Printf("{");
for (uint32_t Feature: FeatureSet)
}
}
- void Replace(InputInfo *II, const Unit &U) {
+ void Replace(InputInfo *II, const Unit &U,
+ std::chrono::microseconds TimeOfUnit) {
assert(II->U.size() > U.size());
Hashes.erase(Sha1ToString(II->Sha1));
DeleteFile(*II);
Hashes.insert(Sha1ToString(II->Sha1));
II->U = U;
II->Reduced = true;
+ II->TimeOfUnit = TimeOfUnit;
DistributionNeedsUpdate = true;
}
const auto &II = *Inputs[i];
Printf(" [% 3zd %s] sz: % 5zd runs: % 5zd succ: % 5zd focus: %d\n", i,
Sha1ToString(II.Sha1).c_str(), II.U.size(),
- II.NumExecutedMutations, II.NumSuccessfullMutations, II.HasFocusFunction);
+ II.NumExecutedMutations, II.NumSuccessfullMutations,
+ II.HasFocusFunction);
}
}
}
std::piecewise_constant_distribution<double> CorpusDistribution;
- Vector<double> Intervals;
- Vector<double> Weights;
+ std::vector<double> Intervals;
+ std::vector<double> Weights;
std::unordered_set<std::string> Hashes;
- Vector<InputInfo*> Inputs;
+ std::vector<InputInfo *> Inputs;
size_t NumAddedFeatures = 0;
size_t NumUpdatedFeatures = 0;
bool DistributionNeedsUpdate = true;
uint16_t FreqOfMostAbundantRareFeature = 0;
uint16_t GlobalFeatureFreqs[kFeatureSetSize] = {};
- Vector<uint32_t> RareFeatures;
+ std::vector<uint32_t> RareFeatures;
std::string OutputCorpus;
};
// Coverage lines have this form:
// CN X Y Z T
// where N is the number of the function, T is the total number of instrumented
-// BBs, and X,Y,Z, if present, are the indecies of covered BB.
+// BBs, and X,Y,Z, if present, are the indices of covered BB.
// BB #0, which is the entry block, is not explicitly listed.
bool BlockCoverage::AppendCoverage(std::istream &IN) {
std::string L;
continue;
}
if (L[0] != 'C') continue;
- Vector<uint32_t> CoveredBlocks;
+ std::vector<uint32_t> CoveredBlocks;
while (true) {
uint32_t BB = 0;
SS >> BB;
auto It = Functions.find(FunctionId);
auto &Counters =
It == Functions.end()
- ? Functions.insert({FunctionId, Vector<uint32_t>(NumBlocks)})
+ ? Functions.insert({FunctionId, std::vector<uint32_t>(NumBlocks)})
.first->second
: It->second;
// * any uncovered function gets weight 0.
// * a function with lots of uncovered blocks gets bigger weight.
// * a function with a less frequently executed code gets bigger weight.
-Vector<double> BlockCoverage::FunctionWeights(size_t NumFunctions) const {
- Vector<double> Res(NumFunctions);
+std::vector<double> BlockCoverage::FunctionWeights(size_t NumFunctions) const {
+ std::vector<double> Res(NumFunctions);
for (auto It : Functions) {
auto FunctionID = It.first;
auto Counters = It.second;
}
void DataFlowTrace::ReadCoverage(const std::string &DirPath) {
- Vector<SizedFile> Files;
+ std::vector<SizedFile> Files;
GetSizedFilesFromDir(DirPath, &Files);
for (auto &SF : Files) {
auto Name = Basename(SF.File);
}
}
-static void DFTStringAppendToVector(Vector<uint8_t> *DFT,
+static void DFTStringAppendToVector(std::vector<uint8_t> *DFT,
const std::string &DFTString) {
assert(DFT->size() == DFTString.size());
for (size_t I = 0, Len = DFT->size(); I < Len; I++)
(*DFT)[I] = DFTString[I] == '1';
}
-// converts a string of '0' and '1' into a Vector<uint8_t>
-static Vector<uint8_t> DFTStringToVector(const std::string &DFTString) {
- Vector<uint8_t> DFT(DFTString.size());
+// converts a string of '0' and '1' into a std::vector<uint8_t>
+static std::vector<uint8_t> DFTStringToVector(const std::string &DFTString) {
+ std::vector<uint8_t> DFT(DFTString.size());
DFTStringAppendToVector(&DFT, DFTString);
return DFT;
}
}
bool DataFlowTrace::Init(const std::string &DirPath, std::string *FocusFunction,
- Vector<SizedFile> &CorporaFiles, Random &Rand) {
+ std::vector<SizedFile> &CorporaFiles, Random &Rand) {
if (DirPath.empty()) return false;
Printf("INFO: DataFlowTrace: reading from '%s'\n", DirPath.c_str());
- Vector<SizedFile> Files;
+ std::vector<SizedFile> Files;
GetSizedFilesFromDir(DirPath, &Files);
std::string L;
size_t FocusFuncIdx = SIZE_MAX;
- Vector<std::string> FunctionNames;
+ std::vector<std::string> FunctionNames;
// Collect the hashes of the corpus files.
for (auto &SF : CorporaFiles)
// * chooses a random function according to the weights.
ReadCoverage(DirPath);
auto Weights = Coverage.FunctionWeights(NumFunctions);
- Vector<double> Intervals(NumFunctions + 1);
+ std::vector<double> Intervals(NumFunctions + 1);
std::iota(Intervals.begin(), Intervals.end(), 0);
auto Distribution = std::piecewise_constant_distribution<double>(
Intervals.begin(), Intervals.end(), Weights.begin());
}
int CollectDataFlow(const std::string &DFTBinary, const std::string &DirPath,
- const Vector<SizedFile> &CorporaFiles) {
+ const std::vector<SizedFile> &CorporaFiles) {
Printf("INFO: collecting data flow: bin: %s dir: %s files: %zd\n",
DFTBinary.c_str(), DirPath.c_str(), CorporaFiles.size());
if (CorporaFiles.empty()) {
// we then request tags in [0,Size/2) and [Size/2, Size), and so on.
// Function number => DFT.
auto OutPath = DirPlusFile(DirPath, Hash(FileToVector(F.File)));
- std::unordered_map<size_t, Vector<uint8_t>> DFTMap;
+ std::unordered_map<size_t, std::vector<uint8_t>> DFTMap;
std::unordered_set<std::string> Cov;
Command Cmd;
Cmd.addArgument(DFTBinary);
namespace fuzzer {
int CollectDataFlow(const std::string &DFTBinary, const std::string &DirPath,
- const Vector<SizedFile> &CorporaFiles);
+ const std::vector<SizedFile> &CorporaFiles);
class BlockCoverage {
public:
return Result;
}
- Vector<double> FunctionWeights(size_t NumFunctions) const;
+ std::vector<double> FunctionWeights(size_t NumFunctions) const;
void clear() { Functions.clear(); }
private:
- typedef Vector<uint32_t> CoverageVector;
+ typedef std::vector<uint32_t> CoverageVector;
uint32_t NumberOfCoveredBlocks(const CoverageVector &Counters) const {
uint32_t Res = 0;
public:
void ReadCoverage(const std::string &DirPath);
bool Init(const std::string &DirPath, std::string *FocusFunction,
- Vector<SizedFile> &CorporaFiles, Random &Rand);
+ std::vector<SizedFile> &CorporaFiles, Random &Rand);
void Clear() { Traces.clear(); }
- const Vector<uint8_t> *Get(const std::string &InputSha1) const {
+ const std::vector<uint8_t> *Get(const std::string &InputSha1) const {
auto It = Traces.find(InputSha1);
if (It != Traces.end())
return &It->second;
private:
// Input's sha1 => DFT for the FocusFunction.
- std::unordered_map<std::string, Vector<uint8_t> > Traces;
- BlockCoverage Coverage;
- std::unordered_set<std::string> CorporaHashes;
+ std::unordered_map<std::string, std::vector<uint8_t>> Traces;
+ BlockCoverage Coverage;
+ std::unordered_set<std::string> CorporaHashes;
};
} // namespace fuzzer
// Global interface to functions that may or may not be available.
extern ExternalFunctions *EF;
-// We are using a custom allocator to give a different symbol name to STL
-// containers in order to avoid ODR violations.
-template<typename T>
- class fuzzer_allocator: public std::allocator<T> {
- public:
- fuzzer_allocator() = default;
-
- template<class U>
- fuzzer_allocator(const fuzzer_allocator<U>&) {}
-
- template<class Other>
- struct rebind { typedef fuzzer_allocator<Other> other; };
- };
-
-template<typename T>
-using Vector = std::vector<T, fuzzer_allocator<T>>;
-
-template<typename T>
-using Set = std::set<T, std::less<T>, fuzzer_allocator<T>>;
-
-typedef Vector<uint8_t> Unit;
-typedef Vector<Unit> UnitVector;
+typedef std::vector<uint8_t> Unit;
+typedef std::vector<Unit> UnitVector;
typedef int (*UserCallback)(const uint8_t *Data, size_t Size);
int FuzzerDriver(int *argc, char ***argv, UserCallback Callback);
public:
DictionaryEntry() {}
DictionaryEntry(Word W) : W(W) {}
- DictionaryEntry(Word W, size_t PositionHint) : W(W), PositionHint(PositionHint) {}
+ DictionaryEntry(Word W, size_t PositionHint)
+ : W(W), PositionHint(PositionHint) {}
const Word &GetW() const { return W; }
- bool HasPositionHint() const { return PositionHint != std::numeric_limits<size_t>::max(); }
+ bool HasPositionHint() const {
+ return PositionHint != std::numeric_limits<size_t>::max();
+ }
size_t GetPositionHint() const {
assert(HasPositionHint());
return PositionHint;
};
// Parses one dictionary entry.
-// If successful, write the enty to Unit and returns true,
+// If successful, writes the entry to Unit and returns true,
// otherwise returns false.
bool ParseOneDictionaryEntry(const std::string &Str, Unit *U);
// Parses the dictionary file, fills Units, returns true iff all lines
// were parsed successfully.
-bool ParseDictionaryFile(const std::string &Text, Vector<Unit> *Units);
+bool ParseDictionaryFile(const std::string &Text, std::vector<Unit> *Units);
} // namespace fuzzer
static const size_t kNumFlags =
sizeof(FlagDescriptions) / sizeof(FlagDescriptions[0]);
-static Vector<std::string> *Inputs;
+static std::vector<std::string> *Inputs;
static std::string *ProgName;
static void PrintHelp() {
}
// We don't use any library to minimize dependencies.
-static void ParseFlags(const Vector<std::string> &Args,
+static void ParseFlags(const std::vector<std::string> &Args,
const ExternalFunctions *EF) {
for (size_t F = 0; F < kNumFlags; F++) {
if (FlagDescriptions[F].IntFlag)
"Disabling -len_control by default.\n", EF->LLVMFuzzerCustomMutator);
}
- Inputs = new Vector<std::string>;
+ Inputs = new std::vector<std::string>;
for (size_t A = 1; A < Args.size(); A++) {
if (ParseOneFlag(Args[A].c_str())) {
if (Flags.ignore_remaining_args)
exit(1);
}
-std::string CloneArgsWithoutX(const Vector<std::string> &Args,
+std::string CloneArgsWithoutX(const std::vector<std::string> &Args,
const char *X1, const char *X2) {
std::string Cmd;
for (auto &S : Args) {
return Cmd;
}
-static int RunInMultipleProcesses(const Vector<std::string> &Args,
+static int RunInMultipleProcesses(const std::vector<std::string> &Args,
unsigned NumWorkers, unsigned NumJobs) {
std::atomic<unsigned> Counter(0);
std::atomic<bool> HasErrors(false);
Command Cmd(Args);
Cmd.removeFlag("jobs");
Cmd.removeFlag("workers");
- Vector<std::thread> V;
+ std::vector<std::thread> V;
std::thread Pulse(PulseThread);
Pulse.detach();
for (unsigned i = 0; i < NumWorkers; i++)
- V.push_back(std::thread(WorkerThread, std::ref(Cmd), &Counter, NumJobs, &HasErrors));
+ V.push_back(std::thread(WorkerThread, std::ref(Cmd), &Counter, NumJobs,
+ &HasErrors));
for (auto &T : V)
T.join();
return HasErrors ? 1 : 0;
return S.substr(Beg, End - Beg);
}
-int CleanseCrashInput(const Vector<std::string> &Args,
- const FuzzingOptions &Options) {
+int CleanseCrashInput(const std::vector<std::string> &Args,
+ const FuzzingOptions &Options) {
if (Inputs->size() != 1 || !Flags.exact_artifact_path) {
Printf("ERROR: -cleanse_crash should be given one input file and"
" -exact_artifact_path\n");
auto U = FileToVector(CurrentFilePath);
size_t Size = U.size();
- const Vector<uint8_t> ReplacementBytes = {' ', 0xff};
+ const std::vector<uint8_t> ReplacementBytes = {' ', 0xff};
for (int NumAttempts = 0; NumAttempts < 5; NumAttempts++) {
bool Changed = false;
for (size_t Idx = 0; Idx < Size; Idx++) {
return 0;
}
-int MinimizeCrashInput(const Vector<std::string> &Args,
+int MinimizeCrashInput(const std::vector<std::string> &Args,
const FuzzingOptions &Options) {
if (Inputs->size() != 1) {
Printf("ERROR: -minimize_crash should be given one input file\n");
return 0;
}
-void Merge(Fuzzer *F, FuzzingOptions &Options, const Vector<std::string> &Args,
- const Vector<std::string> &Corpora, const char *CFPathOrNull) {
+void Merge(Fuzzer *F, FuzzingOptions &Options,
+ const std::vector<std::string> &Args,
+ const std::vector<std::string> &Corpora, const char *CFPathOrNull) {
if (Corpora.size() < 2) {
Printf("INFO: Merge requires two or more corpus dirs\n");
exit(0);
}
- Vector<SizedFile> OldCorpus, NewCorpus;
+ std::vector<SizedFile> OldCorpus, NewCorpus;
GetSizedFilesFromDir(Corpora[0], &OldCorpus);
for (size_t i = 1; i < Corpora.size(); i++)
GetSizedFilesFromDir(Corpora[i], &NewCorpus);
std::sort(NewCorpus.begin(), NewCorpus.end());
std::string CFPath = CFPathOrNull ? CFPathOrNull : TempPath("Merge", ".txt");
- Vector<std::string> NewFiles;
- Set<uint32_t> NewFeatures, NewCov;
+ std::vector<std::string> NewFiles;
+ std::set<uint32_t> NewFeatures, NewCov;
CrashResistantMerge(Args, OldCorpus, NewCorpus, &NewFiles, {}, &NewFeatures,
- {}, &NewCov, CFPath, true);
+ {}, &NewCov, CFPath, true, Flags.set_cover_merge);
for (auto &Path : NewFiles)
F->WriteToOutputCorpus(FileToVector(Path, Options.MaxLen));
// We are done, delete the control file if it was a temporary one.
exit(0);
}
-int AnalyzeDictionary(Fuzzer *F, const Vector<Unit>& Dict,
- UnitVector& Corpus) {
+int AnalyzeDictionary(Fuzzer *F, const std::vector<Unit> &Dict,
+ UnitVector &Corpus) {
Printf("Started dictionary minimization (up to %d tests)\n",
Dict.size() * Corpus.size() * 2);
// Scores and usage count for each dictionary unit.
- Vector<int> Scores(Dict.size());
- Vector<int> Usages(Dict.size());
+ std::vector<int> Scores(Dict.size());
+ std::vector<int> Usages(Dict.size());
- Vector<size_t> InitialFeatures;
- Vector<size_t> ModifiedFeatures;
+ std::vector<size_t> InitialFeatures;
+ std::vector<size_t> ModifiedFeatures;
for (auto &C : Corpus) {
// Get coverage for the testcase without modifications.
F->ExecuteCallback(C.data(), C.size());
});
for (size_t i = 0; i < Dict.size(); ++i) {
- Vector<uint8_t> Data = C;
+ std::vector<uint8_t> Data = C;
auto StartPos = std::search(Data.begin(), Data.end(),
Dict[i].begin(), Dict[i].end());
// Skip dictionary unit, if the testcase does not contain it.
return 0;
}
-Vector<std::string> ParseSeedInuts(const char *seed_inputs) {
+std::vector<std::string> ParseSeedInuts(const char *seed_inputs) {
// Parse -seed_inputs=file1,file2,... or -seed_inputs=@seed_inputs_file
- Vector<std::string> Files;
+ std::vector<std::string> Files;
if (!seed_inputs) return Files;
std::string SeedInputs;
if (Flags.seed_inputs[0] == '@')
return Files;
}
-static Vector<SizedFile> ReadCorpora(const Vector<std::string> &CorpusDirs,
- const Vector<std::string> &ExtraSeedFiles) {
- Vector<SizedFile> SizedFiles;
+static std::vector<SizedFile>
+ReadCorpora(const std::vector<std::string> &CorpusDirs,
+ const std::vector<std::string> &ExtraSeedFiles) {
+ std::vector<SizedFile> SizedFiles;
size_t LastNumFiles = 0;
for (auto &Dir : CorpusDirs) {
GetSizedFilesFromDir(Dir, &SizedFiles);
EF->LLVMFuzzerInitialize(argc, argv);
if (EF->__msan_scoped_disable_interceptor_checks)
EF->__msan_scoped_disable_interceptor_checks();
- const Vector<std::string> Args(*argv, *argv + *argc);
+ const std::vector<std::string> Args(*argv, *argv + *argc);
assert(!Args.empty());
ProgName = new std::string(Args[0]);
if (Argv0 != *ProgName) {
ValidateDirectoryExists(DirName(Options.ExactArtifactPath),
Flags.create_missing_dirs);
}
- Vector<Unit> Dictionary;
+ std::vector<Unit> Dictionary;
if (Flags.dict)
if (!ParseDictionaryFile(FileToString(Flags.dict), &Dictionary))
return 1;
if (Flags.verbosity)
Printf("INFO: Seed: %u\n", Seed);
- if (Flags.collect_data_flow && !Flags.fork && !Flags.merge) {
+ if (Flags.collect_data_flow && !Flags.fork &&
+ !(Flags.merge || Flags.set_cover_merge)) {
if (RunIndividualFiles)
return CollectDataFlow(Flags.collect_data_flow, Flags.data_flow_trace,
ReadCorpora({}, *Inputs));
exit(0);
}
+ Options.ForkCorpusGroups = Flags.fork_corpus_groups;
if (Flags.fork)
FuzzWithFork(F->GetMD().GetRand(), Options, Args, *Inputs, Flags.fork);
- if (Flags.merge)
+ if (Flags.merge || Flags.set_cover_merge)
Merge(F, Options, Args, *Inputs, Flags.merge_control_file);
if (Flags.merge_inner) {
if (Options.MaxLen == 0)
F->SetMaxInputLen(kDefaultMaxMergeLen);
assert(Flags.merge_control_file);
- F->CrashResistantMergeInternalStep(Flags.merge_control_file);
+ F->CrashResistantMergeInternalStep(Flags.merge_control_file,
+ !strncmp(Flags.merge_inner, "2", 1));
exit(0);
}
} // namespace fuzzer
-#else
-// TODO: implement for other platforms.
-namespace fuzzer {
-uint8_t *ExtraCountersBegin() { return nullptr; }
-uint8_t *ExtraCountersEnd() { return nullptr; }
-void ClearExtraCounters() {}
-} // namespace fuzzer
-
#endif
--- /dev/null
+//===- FuzzerExtraCountersDarwin.cpp - Extra coverage counters for Darwin -===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// Extra coverage counters defined by user code for Darwin.
+//===----------------------------------------------------------------------===//
+
+#include "FuzzerPlatform.h"
+#include <cstdint>
+
+#if LIBFUZZER_APPLE
+
+namespace fuzzer {
+uint8_t *ExtraCountersBegin() { return nullptr; }
+uint8_t *ExtraCountersEnd() { return nullptr; }
+void ClearExtraCounters() {}
+} // namespace fuzzer
+
+#endif
--- /dev/null
+//===- FuzzerExtraCountersWindows.cpp - Extra coverage counters for Win32 -===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// Extra coverage counters defined by user code for Windows.
+//===----------------------------------------------------------------------===//
+
+#include "FuzzerPlatform.h"
+#include <cstdint>
+
+#if LIBFUZZER_WINDOWS
+#include <windows.h>
+
+namespace fuzzer {
+
+//
+// The __start___libfuzzer_extra_counters variable is align 16, size 16 to
+// ensure the padding between it and the next variable in this section (either
+// __libfuzzer_extra_counters or __stop___libfuzzer_extra_counters) will be
+// located at (__start___libfuzzer_extra_counters +
+// sizeof(__start___libfuzzer_extra_counters)). Otherwise, the calculation of
+// (stop - (start + sizeof(start))) might be skewed.
+//
+// The section name, __libfuzzer_extra_countaaa ends with "aaa", so it sorts
+// before __libfuzzer_extra_counters alphabetically. We want the start symbol to
+// be placed in the section just before the user supplied counters (if present).
+//
+#pragma section(".data$__libfuzzer_extra_countaaa")
+ATTRIBUTE_ALIGNED(16)
+__declspec(allocate(".data$__libfuzzer_extra_countaaa")) uint8_t
+ __start___libfuzzer_extra_counters[16] = {0};
+
+//
+// Example of what the user-supplied counters should look like. First, the
+// pragma to create the section name. It will fall alphabetically between
+// ".data$__libfuzzer_extra_countaaa" and ".data$__libfuzzer_extra_countzzz".
+// Next, the declspec to allocate the variable inside the specified section.
+// Finally, some array, struct, whatever that is used to track the counter data.
+// The size of this variable is computed at runtime by finding the difference of
+// __stop___libfuzzer_extra_counters and __start___libfuzzer_extra_counters +
+// sizeof(__start___libfuzzer_extra_counters).
+//
+
+//
+// #pragma section(".data$__libfuzzer_extra_counters")
+// __declspec(allocate(".data$__libfuzzer_extra_counters"))
+// uint8_t any_name_variable[64 * 1024];
+//
+
+//
+// Here, the section name, __libfuzzer_extra_countzzz ends with "zzz", so it
+// sorts after __libfuzzer_extra_counters alphabetically. We want the stop
+// symbol to be placed in the section just after the user supplied counters (if
+// present). Align to 1 so there isn't any padding placed between this and the
+// previous variable.
+//
+#pragma section(".data$__libfuzzer_extra_countzzz")
+ATTRIBUTE_ALIGNED(1)
+__declspec(allocate(".data$__libfuzzer_extra_countzzz")) uint8_t
+ __stop___libfuzzer_extra_counters = 0;
+
+uint8_t *ExtraCountersBegin() {
+ return __start___libfuzzer_extra_counters +
+ sizeof(__start___libfuzzer_extra_counters);
+}
+
+uint8_t *ExtraCountersEnd() { return &__stop___libfuzzer_extra_counters; }
+
+ATTRIBUTE_NO_SANITIZE_ALL
+void ClearExtraCounters() {
+ uint8_t *Beg = ExtraCountersBegin();
+ SecureZeroMemory(Beg, ExtraCountersEnd() - Beg);
+}
+
+} // namespace fuzzer
+
+#endif
FUZZER_FLAG_INT(help, 0, "Print help.")
FUZZER_FLAG_INT(fork, 0, "Experimental mode where fuzzing happens "
"in a subprocess")
+FUZZER_FLAG_INT(fork_corpus_groups, 0, "For fork mode, enable the corpus-group "
+ "strategy, The main corpus will be grouped according to size, "
+ "and each sub-process will randomly select seeds from different "
+ "groups as the sub-corpus.")
FUZZER_FLAG_INT(ignore_timeouts, 1, "Ignore timeouts in fork mode")
FUZZER_FLAG_INT(ignore_ooms, 1, "Ignore OOMs in fork mode")
FUZZER_FLAG_INT(ignore_crashes, 0, "Ignore crashes in fork mode")
FUZZER_FLAG_INT(merge, 0, "If 1, the 2-nd, 3-rd, etc corpora will be "
"merged into the 1-st corpus. Only interesting units will be taken. "
"This flag can be used to minimize a corpus.")
+FUZZER_FLAG_INT(set_cover_merge, 0, "If 1, the 2-nd, 3-rd, etc corpora will be "
+ "merged into the 1-st corpus. Same as the 'merge' flag, but uses the "
+ "standard greedy algorithm for the set cover problem to "
+ "compute an approximation of the minimum set of testcases that "
+ "provide the same coverage as the initial corpora")
FUZZER_FLAG_STRING(stop_file, "Stop fuzzing ASAP if this file exists")
FUZZER_FLAG_STRING(merge_inner, "internal flag")
FUZZER_FLAG_STRING(merge_control_file,
};
struct GlobalEnv {
- Vector<std::string> Args;
- Vector<std::string> CorpusDirs;
+ std::vector<std::string> Args;
+ std::vector<std::string> CorpusDirs;
std::string MainCorpusDir;
std::string TempDir;
std::string DFTDir;
std::string DataFlowBinary;
- Set<uint32_t> Features, Cov;
- Set<std::string> FilesWithDFT;
- Vector<std::string> Files;
+ std::set<uint32_t> Features, Cov;
+ std::set<std::string> FilesWithDFT;
+ std::vector<std::string> Files;
+ std::vector<std::size_t> FilesSizes;
Random *Rand;
std::chrono::system_clock::time_point ProcessStartTime;
int Verbosity = 0;
+ int Group = 0;
+ int NumCorpuses = 8;
size_t NumTimeouts = 0;
size_t NumOOMs = 0;
if (size_t CorpusSubsetSize =
std::min(Files.size(), (size_t)sqrt(Files.size() + 2))) {
auto Time1 = std::chrono::system_clock::now();
- for (size_t i = 0; i < CorpusSubsetSize; i++) {
- auto &SF = Files[Rand->SkewTowardsLast(Files.size())];
- Seeds += (Seeds.empty() ? "" : ",") + SF;
- CollectDFT(SF);
+ if (Group) { // whether to group the corpus.
+ size_t AverageCorpusSize = Files.size() / NumCorpuses + 1;
+ size_t StartIndex = ((JobId - 1) % NumCorpuses) * AverageCorpusSize;
+ for (size_t i = 0; i < CorpusSubsetSize; i++) {
+ size_t RandNum = (*Rand)(AverageCorpusSize);
+ size_t Index = RandNum + StartIndex;
+ Index = Index < Files.size() ? Index
+ : Rand->SkewTowardsLast(Files.size());
+ auto &SF = Files[Index];
+ Seeds += (Seeds.empty() ? "" : ",") + SF;
+ CollectDFT(SF);
+ }
+ } else {
+ for (size_t i = 0; i < CorpusSubsetSize; i++) {
+ auto &SF = Files[Rand->SkewTowardsLast(Files.size())];
+ Seeds += (Seeds.empty() ? "" : ",") + SF;
+ CollectDFT(SF);
+ }
}
auto Time2 = std::chrono::system_clock::now();
auto DftTimeInSeconds = duration_cast<seconds>(Time2 - Time1).count();
auto Stats = ParseFinalStatsFromLog(Job->LogPath);
NumRuns += Stats.number_of_executed_units;
- Vector<SizedFile> TempFiles, MergeCandidates;
+ std::vector<SizedFile> TempFiles, MergeCandidates;
// Read all newly created inputs and their feature sets.
// Choose only those inputs that have new features.
GetSizedFilesFromDir(Job->CorpusDir, &TempFiles);
FeatureFile.replace(0, Job->CorpusDir.size(), Job->FeaturesDir);
auto FeatureBytes = FileToVector(FeatureFile, 0, false);
assert((FeatureBytes.size() % sizeof(uint32_t)) == 0);
- Vector<uint32_t> NewFeatures(FeatureBytes.size() / sizeof(uint32_t));
+ std::vector<uint32_t> NewFeatures(FeatureBytes.size() / sizeof(uint32_t));
memcpy(NewFeatures.data(), FeatureBytes.data(), FeatureBytes.size());
for (auto Ft : NewFeatures) {
if (!Features.count(Ft)) {
}
}
// if (!FilesToAdd.empty() || Job->ExitCode != 0)
- Printf("#%zd: cov: %zd ft: %zd corp: %zd exec/s %zd "
+ Printf("#%zd: cov: %zd ft: %zd corp: %zd exec/s: %zd "
"oom/timeout/crash: %zd/%zd/%zd time: %zds job: %zd dft_time: %d\n",
NumRuns, Cov.size(), Features.size(), Files.size(),
Stats.average_exec_per_sec, NumOOMs, NumTimeouts, NumCrashes,
if (MergeCandidates.empty()) return;
- Vector<std::string> FilesToAdd;
- Set<uint32_t> NewFeatures, NewCov;
+ std::vector<std::string> FilesToAdd;
+ std::set<uint32_t> NewFeatures, NewCov;
+ bool IsSetCoverMerge =
+ !Job->Cmd.getFlagValue("set_cover_merge").compare("1");
CrashResistantMerge(Args, {}, MergeCandidates, &FilesToAdd, Features,
- &NewFeatures, Cov, &NewCov, Job->CFPath, false);
+ &NewFeatures, Cov, &NewCov, Job->CFPath, false,
+ IsSetCoverMerge);
for (auto &Path : FilesToAdd) {
auto U = FileToVector(Path);
auto NewPath = DirPlusFile(MainCorpusDir, Hash(U));
WriteToFile(U, NewPath);
- Files.push_back(NewPath);
+ if (Group) { // Insert the queue according to the size of the seed.
+ size_t UnitSize = U.size();
+ auto Idx =
+ std::upper_bound(FilesSizes.begin(), FilesSizes.end(), UnitSize) -
+ FilesSizes.begin();
+ FilesSizes.insert(FilesSizes.begin() + Idx, UnitSize);
+ Files.insert(Files.begin() + Idx, NewPath);
+ } else {
+ Files.push_back(NewPath);
+ }
}
Features.insert(NewFeatures.begin(), NewFeatures.end());
Cov.insert(NewCov.begin(), NewCov.end());
if (TPC.PcIsFuncEntry(TE))
PrintPC(" NEW_FUNC: %p %F %L\n", "",
TPC.GetNextInstructionPc(TE->PC));
-
}
-
void CollectDFT(const std::string &InputPath) {
if (DataFlowBinary.empty()) return;
if (!FilesWithDFT.insert(InputPath).second) return;
// This is just a skeleton of an experimental -fork=1 feature.
void FuzzWithFork(Random &Rand, const FuzzingOptions &Options,
- const Vector<std::string> &Args,
- const Vector<std::string> &CorpusDirs, int NumJobs) {
+ const std::vector<std::string> &Args,
+ const std::vector<std::string> &CorpusDirs, int NumJobs) {
Printf("INFO: -fork=%d: fuzzing in separate process(s)\n", NumJobs);
GlobalEnv Env;
Env.Verbosity = Options.Verbosity;
Env.ProcessStartTime = std::chrono::system_clock::now();
Env.DataFlowBinary = Options.CollectDataFlow;
+ Env.Group = Options.ForkCorpusGroups;
- Vector<SizedFile> SeedFiles;
+ std::vector<SizedFile> SeedFiles;
for (auto &Dir : CorpusDirs)
GetSizedFilesFromDir(Dir, &SeedFiles);
std::sort(SeedFiles.begin(), SeedFiles.end());
Env.Files.push_back(File.File);
} else {
auto CFPath = DirPlusFile(Env.TempDir, "merge.txt");
- Set<uint32_t> NewFeatures, NewCov;
+ std::set<uint32_t> NewFeatures, NewCov;
CrashResistantMerge(Env.Args, {}, SeedFiles, &Env.Files, Env.Features,
- &NewFeatures, Env.Cov, &NewCov, CFPath, false);
+ &NewFeatures, Env.Cov, &NewCov, CFPath,
+ /*Verbose=*/false, /*IsSetCoverMerge=*/false);
Env.Features.insert(NewFeatures.begin(), NewFeatures.end());
Env.Cov.insert(NewFeatures.begin(), NewFeatures.end());
RemoveFile(CFPath);
}
+
+ if (Env.Group) {
+ for (auto &path : Env.Files)
+ Env.FilesSizes.push_back(FileSize(path));
+ }
+
Printf("INFO: -fork=%d: %zd seed inputs, starting to fuzz in %s\n", NumJobs,
Env.Files.size(), Env.TempDir.c_str());
WriteToFile(Unit({1}), Env.StopFile());
};
+ size_t MergeCycle = 20;
+ size_t JobExecuted = 0;
size_t JobId = 1;
- Vector<std::thread> Threads;
+ std::vector<std::thread> Threads;
for (int t = 0; t < NumJobs; t++) {
Threads.push_back(std::thread(WorkerThread, &FuzzQ, &MergeQ));
FuzzQ.Push(Env.CreateNewJob(JobId++));
Env.RunOneMergeJob(Job.get());
- // Continue if our crash is one of the ignorred ones.
+ // merge the corpus .
+ JobExecuted++;
+ if (Env.Group && JobExecuted >= MergeCycle) {
+ std::vector<SizedFile> CurrentSeedFiles;
+ for (auto &Dir : CorpusDirs)
+ GetSizedFilesFromDir(Dir, &CurrentSeedFiles);
+ std::sort(CurrentSeedFiles.begin(), CurrentSeedFiles.end());
+
+ auto CFPath = DirPlusFile(Env.TempDir, "merge.txt");
+ std::set<uint32_t> TmpNewFeatures, TmpNewCov;
+ std::set<uint32_t> TmpFeatures, TmpCov;
+ Env.Files.clear();
+ Env.FilesSizes.clear();
+ CrashResistantMerge(Env.Args, {}, CurrentSeedFiles, &Env.Files,
+ TmpFeatures, &TmpNewFeatures, TmpCov, &TmpNewCov,
+ CFPath, /*Verbose=*/false, /*IsSetCoverMerge=*/false);
+ for (auto &path : Env.Files)
+ Env.FilesSizes.push_back(FileSize(path));
+ RemoveFile(CFPath);
+ JobExecuted = 0;
+ MergeCycle += 5;
+ }
+
+ // Since the number of corpus seeds will gradually increase, in order to
+ // control the number in each group to be about three times the number of
+ // seeds selected each time, the number of groups is dynamically adjusted.
+ if (Env.Files.size() < 2000)
+ Env.NumCorpuses = 12;
+ else if (Env.Files.size() < 6000)
+ Env.NumCorpuses = 20;
+ else if (Env.Files.size() < 12000)
+ Env.NumCorpuses = 32;
+ else if (Env.Files.size() < 16000)
+ Env.NumCorpuses = 40;
+ else if (Env.Files.size() < 24000)
+ Env.NumCorpuses = 60;
+ else
+ Env.NumCorpuses = 80;
+
+ // Continue if our crash is one of the ignored ones.
if (Options.IgnoreTimeouts && ExitCode == Options.TimeoutExitCode)
Env.NumTimeouts++;
else if (Options.IgnoreOOMs && ExitCode == Options.OOMExitCode)
namespace fuzzer {
void FuzzWithFork(Random &Rand, const FuzzingOptions &Options,
- const Vector<std::string> &Args,
- const Vector<std::string> &CorpusDirs, int NumJobs);
+ const std::vector<std::string> &Args,
+ const std::vector<std::string> &CorpusDirs, int NumJobs);
} // namespace fuzzer
#endif // LLVM_FUZZER_FORK_H
static FILE *OutputFile = stderr;
+FILE *GetOutputFile() {
+ return OutputFile;
+}
+
+void SetOutputFile(FILE *NewOutputFile) {
+ OutputFile = NewOutputFile;
+}
+
long GetEpoch(const std::string &Path) {
struct stat St;
if (stat(Path.c_str(), &St))
fclose(Out);
}
-void ReadDirToVectorOfUnits(const char *Path, Vector<Unit> *V, long *Epoch,
+void ReadDirToVectorOfUnits(const char *Path, std::vector<Unit> *V, long *Epoch,
size_t MaxSize, bool ExitOnError,
- Vector<std::string> *VPaths) {
+ std::vector<std::string> *VPaths) {
long E = Epoch ? *Epoch : 0;
- Vector<std::string> Files;
+ std::vector<std::string> Files;
ListFilesInDirRecursive(Path, Epoch, &Files, /*TopDir*/true);
size_t NumLoaded = 0;
for (size_t i = 0; i < Files.size(); i++) {
}
}
-void GetSizedFilesFromDir(const std::string &Dir, Vector<SizedFile> *V) {
- Vector<std::string> Files;
+void GetSizedFilesFromDir(const std::string &Dir, std::vector<SizedFile> *V) {
+ std::vector<std::string> Files;
ListFilesInDirRecursive(Dir, 0, &Files, /*TopDir*/true);
for (auto &File : Files)
if (size_t Size = FileSize(File))
void AppendToFile(const uint8_t *Data, size_t Size, const std::string &Path);
void AppendToFile(const std::string &Data, const std::string &Path);
-void ReadDirToVectorOfUnits(const char *Path, Vector<Unit> *V, long *Epoch,
+void ReadDirToVectorOfUnits(const char *Path, std::vector<Unit> *V, long *Epoch,
size_t MaxSize, bool ExitOnError,
- Vector<std::string> *VPaths = 0);
+ std::vector<std::string> *VPaths = 0);
// Returns "Dir/FileName" or equivalent for the current OS.
std::string DirPlusFile(const std::string &DirPath,
void CloseStdout();
+// For testing.
+FILE *GetOutputFile();
+void SetOutputFile(FILE *NewOutputFile);
+
void Printf(const char *Fmt, ...);
void VPrintf(bool Verbose, const char *Fmt, ...);
size_t FileSize(const std::string &Path);
void ListFilesInDirRecursive(const std::string &Dir, long *Epoch,
- Vector<std::string> *V, bool TopDir);
+ std::vector<std::string> *V, bool TopDir);
bool MkDirRecursive(const std::string &Dir);
void RmDirRecursive(const std::string &Dir);
bool operator<(const SizedFile &B) const { return Size < B.Size; }
};
-void GetSizedFilesFromDir(const std::string &Dir, Vector<SizedFile> *V);
+void GetSizedFilesFromDir(const std::string &Dir, std::vector<SizedFile> *V);
char GetSeparator();
bool IsSeparator(char C);
}
void ListFilesInDirRecursive(const std::string &Dir, long *Epoch,
- Vector<std::string> *V, bool TopDir) {
+ std::vector<std::string> *V, bool TopDir) {
auto E = GetEpoch(Dir);
if (Epoch)
if (E && *Epoch >= E) return;
*Epoch = E;
}
-
void IterateDirRecursive(const std::string &Dir,
void (*DirPreCallback)(const std::string &Dir),
void (*DirPostCallback)(const std::string &Dir),
}
void ListFilesInDirRecursive(const std::string &Dir, long *Epoch,
- Vector<std::string> *V, bool TopDir) {
+ std::vector<std::string> *V, bool TopDir) {
auto E = GetEpoch(Dir);
if (Epoch)
if (E && *Epoch >= E) return;
*Epoch = E;
}
-
void IterateDirRecursive(const std::string &Dir,
void (*DirPreCallback)(const std::string &Dir),
void (*DirPostCallback)(const std::string &Dir),
return Pos - Offset;
}
-// Parse the given Ref string from the position Offset, to exactly match the given
-// string Patt.
-// Returns number of characters considered if successful.
+// Parse the given Ref string from the position Offset, to exactly match the
+// given string Patt. Returns number of characters considered if successful.
static size_t ParseCustomString(const std::string &Ref, size_t Offset,
const char *Patt) {
size_t Len = strlen(Patt);
Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD,
FuzzingOptions Options);
~Fuzzer();
- void Loop(Vector<SizedFile> &CorporaFiles);
- void ReadAndExecuteSeedCorpora(Vector<SizedFile> &CorporaFiles);
+ void Loop(std::vector<SizedFile> &CorporaFiles);
+ void ReadAndExecuteSeedCorpora(std::vector<SizedFile> &CorporaFiles);
void MinimizeCrashLoop(const Unit &U);
void RereadOutputCorpus(size_t MaxSize);
static void StaticFileSizeExceedCallback();
static void StaticGracefulExitCallback();
- void ExecuteCallback(const uint8_t *Data, size_t Size);
+ // Executes the target callback on {Data, Size} once.
+ // Returns false if the input was rejected by the target (target returned -1),
+ // and true otherwise.
+ bool ExecuteCallback(const uint8_t *Data, size_t Size);
bool RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile = false,
InputInfo *II = nullptr, bool ForceAddToCorpus = false,
bool *FoundUniqFeatures = nullptr);
void TPCUpdateObservedPCs();
// Merge Corpora[1:] into Corpora[0].
- void Merge(const Vector<std::string> &Corpora);
- void CrashResistantMergeInternalStep(const std::string &ControlFilePath);
+ void Merge(const std::vector<std::string> &Corpora);
+ void CrashResistantMergeInternalStep(const std::string &ControlFilePath,
+ bool IsSetCoverMerge);
MutationDispatcher &GetMD() { return MD; }
void PrintFinalStats();
void SetMaxInputLen(size_t MaxInputLen);
void HandleMalloc(size_t Size);
static void MaybeExitGracefully();
+ static int InterruptExitCode();
std::string WriteToOutputCorpus(const Unit &U);
private:
size_t MaxMutationLen = 0;
size_t TmpMaxMutationLen = 0;
- Vector<uint32_t> UniqFeatureSetTmp;
+ std::vector<uint32_t> UniqFeatureSetTmp;
// Need to know our own thread.
static thread_local bool IsMyThread;
_Exit(0);
}
+int Fuzzer::InterruptExitCode() {
+ assert(F);
+ return F->Options.InterruptExitCode;
+}
+
void Fuzzer::InterruptCallback() {
Printf("==%lu== libFuzzer: run interrupted; exiting\n", GetPid());
PrintFinalStats();
void Fuzzer::CheckExitOnSrcPosOrItem() {
if (!Options.ExitOnSrcPos.empty()) {
- static auto *PCsSet = new Set<uintptr_t>;
+ static auto *PCsSet = new std::set<uintptr_t>;
auto HandlePC = [&](const TracePC::PCTableEntry *TE) {
if (!PCsSet->insert(TE->PC).second)
return;
void Fuzzer::RereadOutputCorpus(size_t MaxSize) {
if (Options.OutputCorpus.empty() || !Options.ReloadIntervalSec)
return;
- Vector<Unit> AdditionalCorpus;
- Vector<std::string> AdditionalCorpusPaths;
+ std::vector<Unit> AdditionalCorpus;
+ std::vector<std::string> AdditionalCorpusPaths;
ReadDirToVectorOfUnits(
Options.OutputCorpus.c_str(), &AdditionalCorpus,
&EpochOfLastReadOfOutputCorpus, MaxSize,
static void WriteFeatureSetToFile(const std::string &FeaturesDir,
const std::string &FileName,
- const Vector<uint32_t> &FeatureSet) {
+ const std::vector<uint32_t> &FeatureSet) {
if (FeaturesDir.empty() || FeatureSet.empty()) return;
WriteToFile(reinterpret_cast<const uint8_t *>(FeatureSet.data()),
FeatureSet.size() * sizeof(FeatureSet[0]),
// Largest input length should be INT_MAX.
assert(Size < std::numeric_limits<uint32_t>::max());
- ExecuteCallback(Data, Size);
+ if(!ExecuteCallback(Data, Size)) return false;
auto TimeOfUnit = duration_cast<microseconds>(UnitStopTime - UnitStartTime);
UniqFeatureSetTmp.clear();
FoundUniqFeaturesOfII == II->UniqFeatureSet.size() &&
II->U.size() > Size) {
auto OldFeaturesFile = Sha1ToString(II->Sha1);
- Corpus.Replace(II, {Data, Data + Size});
+ Corpus.Replace(II, {Data, Data + Size}, TimeOfUnit);
RenameFeatureSetFile(Options.FeaturesDir, OldFeaturesFile,
Sha1ToString(II->Sha1));
return true;
// This method is not inlined because it would cause a test to fail where it
// is part of the stack unwinding. See D97975 for details.
-ATTRIBUTE_NOINLINE void Fuzzer::ExecuteCallback(const uint8_t *Data,
+ATTRIBUTE_NOINLINE bool Fuzzer::ExecuteCallback(const uint8_t *Data,
size_t Size) {
TPC.RecordInitialStack();
TotalNumberOfRuns++;
if (CurrentUnitData && CurrentUnitData != Data)
memcpy(CurrentUnitData, Data, Size);
CurrentUnitSize = Size;
+ int CBRes = 0;
{
ScopedEnableMsanInterceptorChecks S;
AllocTracer.Start(Options.TraceMalloc);
UnitStartTime = system_clock::now();
TPC.ResetMaps();
RunningUserCallback = true;
- int Res = CB(DataCopy, Size);
+ CBRes = CB(DataCopy, Size);
RunningUserCallback = false;
UnitStopTime = system_clock::now();
- (void)Res;
- assert(Res == 0);
+ assert(CBRes == 0 || CBRes == -1);
HasMoreMallocsThanFrees = AllocTracer.Stop();
}
if (!LooseMemeq(DataCopy, Data, Size))
CrashOnOverwrittenData();
CurrentUnitSize = 0;
delete[] DataCopy;
+ return CBRes == 0;
}
std::string Fuzzer::WriteToOutputCorpus(const Unit &U) {
LastAllocatorPurgeAttemptTime = system_clock::now();
}
-void Fuzzer::ReadAndExecuteSeedCorpora(Vector<SizedFile> &CorporaFiles) {
+void Fuzzer::ReadAndExecuteSeedCorpora(std::vector<SizedFile> &CorporaFiles) {
const size_t kMaxSaneLen = 1 << 20;
const size_t kMinDefaultLen = 4096;
size_t MaxSize = 0;
}
if (Corpus.empty() && Options.MaxNumberOfRuns) {
- Printf("ERROR: no interesting inputs were found. "
- "Is the code instrumented for coverage? Exiting.\n");
- exit(1);
+ Printf("WARNING: no interesting inputs were found so far. "
+ "Is the code instrumented for coverage?\n"
+ "This may also happen if the target rejected all inputs we tried so "
+ "far\n");
+ // The remaining logic requires that the corpus is not empty,
+ // so we add one fake input to the in-memory corpus.
+ Corpus.AddToCorpus({'\n'}, /*NumFeatures=*/1, /*MayDeleteFile=*/true,
+ /*HasFocusFunction=*/false, /*NeverReduce=*/false,
+ /*TimeOfUnit=*/duration_cast<microseconds>(0s), {0}, DFT,
+ /*BaseII*/ nullptr);
}
}
-void Fuzzer::Loop(Vector<SizedFile> &CorporaFiles) {
+void Fuzzer::Loop(std::vector<SizedFile> &CorporaFiles) {
auto FocusFunctionOrAuto = Options.FocusFunction;
DFT.Init(Options.DataFlowTrace, &FocusFunctionOrAuto, CorporaFiles,
MD.GetRand());
size_t ExpectedStartMarker = 0;
const size_t kInvalidStartMarker = -1;
size_t LastSeenStartMarker = kInvalidStartMarker;
- Vector<uint32_t> TmpFeatures;
- Set<uint32_t> PCs;
+ std::vector<uint32_t> TmpFeatures;
+ std::set<uint32_t> PCs;
while (std::getline(IS, Line, '\n')) {
std::istringstream ISS1(Line);
std::string Marker;
// Decides which files need to be merged (add those to NewFiles).
// Returns the number of new features added.
-size_t Merger::Merge(const Set<uint32_t> &InitialFeatures,
- Set<uint32_t> *NewFeatures,
- const Set<uint32_t> &InitialCov, Set<uint32_t> *NewCov,
- Vector<std::string> *NewFiles) {
+size_t Merger::Merge(const std::set<uint32_t> &InitialFeatures,
+ std::set<uint32_t> *NewFeatures,
+ const std::set<uint32_t> &InitialCov,
+ std::set<uint32_t> *NewCov,
+ std::vector<std::string> *NewFiles) {
NewFiles->clear();
NewFeatures->clear();
NewCov->clear();
assert(NumFilesInFirstCorpus <= Files.size());
- Set<uint32_t> AllFeatures = InitialFeatures;
+ std::set<uint32_t> AllFeatures = InitialFeatures;
// What features are in the initial corpus?
for (size_t i = 0; i < NumFilesInFirstCorpus; i++) {
// Remove all features that we already know from all other inputs.
for (size_t i = NumFilesInFirstCorpus; i < Files.size(); i++) {
auto &Cur = Files[i].Features;
- Vector<uint32_t> Tmp;
+ std::vector<uint32_t> Tmp;
std::set_difference(Cur.begin(), Cur.end(), AllFeatures.begin(),
AllFeatures.end(), std::inserter(Tmp, Tmp.begin()));
Cur.swap(Tmp);
return NewFeatures->size();
}
-Set<uint32_t> Merger::AllFeatures() const {
- Set<uint32_t> S;
+std::set<uint32_t> Merger::AllFeatures() const {
+ std::set<uint32_t> S;
for (auto &File : Files)
S.insert(File.Features.begin(), File.Features.end());
return S;
}
// Inner process. May crash if the target crashes.
-void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) {
+void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath,
+ bool IsSetCoverMerge) {
Printf("MERGE-INNER: using the control file '%s'\n", CFPath.c_str());
Merger M;
std::ifstream IF(CFPath);
M.Files.size() - M.FirstNotProcessedFile);
std::ofstream OF(CFPath, std::ofstream::out | std::ofstream::app);
- Set<size_t> AllFeatures;
+ std::set<size_t> AllFeatures;
auto PrintStatsWrapper = [this, &AllFeatures](const char* Where) {
this->PrintStats(Where, "\n", 0, AllFeatures.size());
};
- Set<const TracePC::PCTableEntry *> AllPCs;
+ std::set<const TracePC::PCTableEntry *> AllPCs;
for (size_t i = M.FirstNotProcessedFile; i < M.Files.size(); i++) {
Fuzzer::MaybeExitGracefully();
auto U = FileToVector(M.Files[i].Name);
// Collect coverage. We are iterating over the files in this order:
// * First, files in the initial corpus ordered by size, smallest first.
// * Then, all other files, smallest first.
- // So it makes no sense to record all features for all files, instead we
- // only record features that were not seen before.
- Set<size_t> UniqFeatures;
- TPC.CollectFeatures([&](size_t Feature) {
- if (AllFeatures.insert(Feature).second)
- UniqFeatures.insert(Feature);
- });
+ std::set<size_t> Features;
+ if (IsSetCoverMerge)
+ TPC.CollectFeatures([&](size_t Feature) { Features.insert(Feature); });
+ else
+ TPC.CollectFeatures([&](size_t Feature) {
+ if (AllFeatures.insert(Feature).second)
+ Features.insert(Feature);
+ });
TPC.UpdateObservedPCs();
// Show stats.
if (!(TotalNumberOfRuns & (TotalNumberOfRuns - 1)))
PrintStatsWrapper("LOADED");
// Write the post-run marker and the coverage.
OF << "FT " << i;
- for (size_t F : UniqFeatures)
+ for (size_t F : Features)
OF << " " << F;
OF << "\n";
OF << "COV " << i;
PrintStatsWrapper("DONE ");
}
-static size_t WriteNewControlFile(const std::string &CFPath,
- const Vector<SizedFile> &OldCorpus,
- const Vector<SizedFile> &NewCorpus,
- const Vector<MergeFileInfo> &KnownFiles) {
+// Merges all corpora into the first corpus. A file is added into
+// the first corpus only if it adds new features. Unlike `Merger::Merge`,
+// this implementation calculates an approximation of the minimum set
+// of corpora files, that cover all known features (set cover problem).
+// Generally, this means that files with more features are preferred for
+// merge into the first corpus. When two files have the same number of
+// features, the smaller one is preferred.
+size_t Merger::SetCoverMerge(const std::set<uint32_t> &InitialFeatures,
+ std::set<uint32_t> *NewFeatures,
+ const std::set<uint32_t> &InitialCov,
+ std::set<uint32_t> *NewCov,
+ std::vector<std::string> *NewFiles) {
+ assert(NumFilesInFirstCorpus <= Files.size());
+ NewFiles->clear();
+ NewFeatures->clear();
+ NewCov->clear();
+ std::set<uint32_t> AllFeatures;
+ // 1 << 21 - 1 is the maximum feature index.
+ // See 'kFeatureSetSize' in 'FuzzerCorpus.h'.
+ const uint32_t kFeatureSetSize = 1 << 21;
+ std::vector<bool> Covered(kFeatureSetSize, false);
+ size_t NumCovered = 0;
+
+ std::set<uint32_t> ExistingFeatures = InitialFeatures;
+ for (size_t i = 0; i < NumFilesInFirstCorpus; ++i)
+ ExistingFeatures.insert(Files[i].Features.begin(), Files[i].Features.end());
+
+ // Mark the existing features as covered.
+ for (const auto &F : ExistingFeatures) {
+ if (!Covered[F % kFeatureSetSize]) {
+ ++NumCovered;
+ Covered[F % kFeatureSetSize] = true;
+ }
+ // Calculate an underestimation of the set of covered features
+ // since the `Covered` bitvector is smaller than the feature range.
+ AllFeatures.insert(F % kFeatureSetSize);
+ }
+
+ std::set<size_t> RemainingFiles;
+ for (size_t i = NumFilesInFirstCorpus; i < Files.size(); ++i) {
+ // Construct an incremental sequence which represent the
+ // indices to all files (excluding those in the initial corpus).
+ // RemainingFiles = range(NumFilesInFirstCorpus..Files.size()).
+ RemainingFiles.insert(i);
+ // Insert this file's unique features to all features.
+ for (const auto &F : Files[i].Features)
+ AllFeatures.insert(F % kFeatureSetSize);
+ }
+
+ // Integrate files into Covered until set is complete.
+ while (NumCovered != AllFeatures.size()) {
+ // Index to file with largest number of unique features.
+ size_t MaxFeaturesIndex = NumFilesInFirstCorpus;
+ // Indices to remove from RemainingFiles.
+ std::set<size_t> RemoveIndices;
+ // Running max unique feature count.
+ // Updated upon finding a file with more features.
+ size_t MaxNumFeatures = 0;
+
+ // Iterate over all files not yet integrated into Covered,
+ // to find the file which has the largest number of
+ // features that are not already in Covered.
+ for (const auto &i : RemainingFiles) {
+ const auto &File = Files[i];
+ size_t CurrentUnique = 0;
+ // Count number of features in this file
+ // which are not yet in Covered.
+ for (const auto &F : File.Features)
+ if (!Covered[F % kFeatureSetSize])
+ ++CurrentUnique;
+
+ if (CurrentUnique == 0) {
+ // All features in this file are already in Covered: skip next time.
+ RemoveIndices.insert(i);
+ } else if (CurrentUnique > MaxNumFeatures ||
+ (CurrentUnique == MaxNumFeatures &&
+ File.Size < Files[MaxFeaturesIndex].Size)) {
+ // Update the max features file based on unique features
+ // Break ties by selecting smaller files.
+ MaxNumFeatures = CurrentUnique;
+ MaxFeaturesIndex = i;
+ }
+ }
+ // Must be a valid index/
+ assert(MaxFeaturesIndex < Files.size());
+ // Remove any feature-less files found.
+ for (const auto &i : RemoveIndices)
+ RemainingFiles.erase(i);
+ if (MaxNumFeatures == 0) {
+ // Did not find a file that adds unique features.
+ // This means that we should have no remaining files.
+ assert(RemainingFiles.size() == 0);
+ assert(NumCovered == AllFeatures.size());
+ break;
+ }
+
+ // MaxFeaturesIndex must be an element of Remaining.
+ assert(RemainingFiles.find(MaxFeaturesIndex) != RemainingFiles.end());
+ // Remove the file with the most features from Remaining.
+ RemainingFiles.erase(MaxFeaturesIndex);
+ const auto &MaxFeatureFile = Files[MaxFeaturesIndex];
+ // Add the features of the max feature file to Covered.
+ for (const auto &F : MaxFeatureFile.Features) {
+ if (!Covered[F % kFeatureSetSize]) {
+ ++NumCovered;
+ Covered[F % kFeatureSetSize] = true;
+ NewFeatures->insert(F);
+ }
+ }
+ // Add the index to this file to the result.
+ NewFiles->push_back(MaxFeatureFile.Name);
+ // Update NewCov with the additional coverage
+ // that MaxFeatureFile provides.
+ for (const auto &C : MaxFeatureFile.Cov)
+ if (InitialCov.find(C) == InitialCov.end())
+ NewCov->insert(C);
+ }
+
+ return NewFeatures->size();
+}
+
+static size_t
+WriteNewControlFile(const std::string &CFPath,
+ const std::vector<SizedFile> &OldCorpus,
+ const std::vector<SizedFile> &NewCorpus,
+ const std::vector<MergeFileInfo> &KnownFiles) {
std::unordered_set<std::string> FilesToSkip;
for (auto &SF: KnownFiles)
FilesToSkip.insert(SF.Name);
- Vector<std::string> FilesToUse;
+ std::vector<std::string> FilesToUse;
auto MaybeUseFile = [=, &FilesToUse](std::string Name) {
if (FilesToSkip.find(Name) == FilesToSkip.end())
FilesToUse.push_back(Name);
}
// Outer process. Does not call the target code and thus should not fail.
-void CrashResistantMerge(const Vector<std::string> &Args,
- const Vector<SizedFile> &OldCorpus,
- const Vector<SizedFile> &NewCorpus,
- Vector<std::string> *NewFiles,
- const Set<uint32_t> &InitialFeatures,
- Set<uint32_t> *NewFeatures,
- const Set<uint32_t> &InitialCov,
- Set<uint32_t> *NewCov,
- const std::string &CFPath,
- bool V /*Verbose*/) {
+void CrashResistantMerge(const std::vector<std::string> &Args,
+ const std::vector<SizedFile> &OldCorpus,
+ const std::vector<SizedFile> &NewCorpus,
+ std::vector<std::string> *NewFiles,
+ const std::set<uint32_t> &InitialFeatures,
+ std::set<uint32_t> *NewFeatures,
+ const std::set<uint32_t> &InitialCov,
+ std::set<uint32_t> *NewCov, const std::string &CFPath,
+ bool V, /*Verbose*/
+ bool IsSetCoverMerge) {
if (NewCorpus.empty() && OldCorpus.empty()) return; // Nothing to merge.
size_t NumAttempts = 0;
- Vector<MergeFileInfo> KnownFiles;
+ std::vector<MergeFileInfo> KnownFiles;
if (FileSize(CFPath)) {
VPrintf(V, "MERGE-OUTER: non-empty control file provided: '%s'\n",
CFPath.c_str());
// Every inner process should execute at least one input.
Command BaseCmd(Args);
BaseCmd.removeFlag("merge");
+ BaseCmd.removeFlag("set_cover_merge");
BaseCmd.removeFlag("fork");
BaseCmd.removeFlag("collect_data_flow");
for (size_t Attempt = 1; Attempt <= NumAttempts; Attempt++) {
VPrintf(V, "MERGE-OUTER: attempt %zd\n", Attempt);
Command Cmd(BaseCmd);
Cmd.addFlag("merge_control_file", CFPath);
- Cmd.addFlag("merge_inner", "1");
+ // If we are going to use the set cover implementation for
+ // minimization add the merge_inner=2 internal flag.
+ Cmd.addFlag("merge_inner", IsSetCoverMerge ? "2" : "1");
if (!V) {
Cmd.setOutputFile(getDevNull());
Cmd.combineOutAndErr();
}
auto ExitCode = ExecuteCommand(Cmd);
if (!ExitCode) {
- VPrintf(V, "MERGE-OUTER: succesfull in %zd attempt(s)\n", Attempt);
+ VPrintf(V, "MERGE-OUTER: successful in %zd attempt(s)\n", Attempt);
break;
}
}
M.ApproximateMemoryConsumption() >> 20, GetPeakRSSMb());
M.Files.insert(M.Files.end(), KnownFiles.begin(), KnownFiles.end());
- M.Merge(InitialFeatures, NewFeatures, InitialCov, NewCov, NewFiles);
+ if (IsSetCoverMerge)
+ M.SetCoverMerge(InitialFeatures, NewFeatures, InitialCov, NewCov, NewFiles);
+ else
+ M.Merge(InitialFeatures, NewFeatures, InitialCov, NewCov, NewFiles);
VPrintf(V, "MERGE-OUTER: %zd new files with %zd new features added; "
"%zd new coverage edges\n",
NewFiles->size(), NewFeatures->size(), NewCov->size());
#define LLVM_FUZZER_MERGE_H
#include "FuzzerDefs.h"
+#include "FuzzerIO.h"
#include <istream>
#include <ostream>
struct MergeFileInfo {
std::string Name;
size_t Size = 0;
- Vector<uint32_t> Features, Cov;
+ std::vector<uint32_t> Features, Cov;
};
struct Merger {
- Vector<MergeFileInfo> Files;
+ std::vector<MergeFileInfo> Files;
size_t NumFilesInFirstCorpus = 0;
size_t FirstNotProcessedFile = 0;
std::string LastFailure;
bool Parse(std::istream &IS, bool ParseCoverage);
bool Parse(const std::string &Str, bool ParseCoverage);
void ParseOrExit(std::istream &IS, bool ParseCoverage);
- size_t Merge(const Set<uint32_t> &InitialFeatures, Set<uint32_t> *NewFeatures,
- const Set<uint32_t> &InitialCov, Set<uint32_t> *NewCov,
- Vector<std::string> *NewFiles);
+ size_t Merge(const std::set<uint32_t> &InitialFeatures,
+ std::set<uint32_t> *NewFeatures,
+ const std::set<uint32_t> &InitialCov, std::set<uint32_t> *NewCov,
+ std::vector<std::string> *NewFiles);
+ size_t SetCoverMerge(const std::set<uint32_t> &InitialFeatures,
+ std::set<uint32_t> *NewFeatures,
+ const std::set<uint32_t> &InitialCov,
+ std::set<uint32_t> *NewCov,
+ std::vector<std::string> *NewFiles);
size_t ApproximateMemoryConsumption() const;
- Set<uint32_t> AllFeatures() const;
+ std::set<uint32_t> AllFeatures() const;
};
-void CrashResistantMerge(const Vector<std::string> &Args,
- const Vector<SizedFile> &OldCorpus,
- const Vector<SizedFile> &NewCorpus,
- Vector<std::string> *NewFiles,
- const Set<uint32_t> &InitialFeatures,
- Set<uint32_t> *NewFeatures,
- const Set<uint32_t> &InitialCov,
- Set<uint32_t> *NewCov,
- const std::string &CFPath,
- bool Verbose);
+void CrashResistantMerge(const std::vector<std::string> &Args,
+ const std::vector<SizedFile> &OldCorpus,
+ const std::vector<SizedFile> &NewCorpus,
+ std::vector<std::string> *NewFiles,
+ const std::set<uint32_t> &InitialFeatures,
+ std::set<uint32_t> *NewFeatures,
+ const std::set<uint32_t> &InitialCov,
+ std::set<uint32_t> *NewCov, const std::string &CFPath,
+ bool Verbose, bool IsSetCoverMerge);
} // namespace fuzzer
}
void MutationDispatcher::PrintRecommendedDictionary() {
- Vector<DictionaryEntry> V;
+ std::vector<DictionaryEntry> V;
for (auto &DE : PersistentAutoDictionary)
if (!ManualDictionary.ContainsWord(DE.GetW()))
V.push_back(DE);
// Mutates Data in place, returns new size.
size_t MutationDispatcher::MutateImpl(uint8_t *Data, size_t Size,
size_t MaxSize,
- Vector<Mutator> &Mutators) {
+ std::vector<Mutator> &Mutators) {
assert(MaxSize > 0);
// Some mutations may fail (e.g. can't insert more bytes if Size == MaxSize),
// in which case they will return 0.
// Mask represents the set of Data bytes that are worth mutating.
size_t MutationDispatcher::MutateWithMask(uint8_t *Data, size_t Size,
size_t MaxSize,
- const Vector<uint8_t> &Mask) {
+ const std::vector<uint8_t> &Mask) {
size_t MaskedSize = std::min(Size, Mask.size());
// * Copy the worthy bytes into a temporary array T
// * Mutate T
/// that have '1' in Mask.
/// Mask.size() should be >= Size.
size_t MutateWithMask(uint8_t *Data, size_t Size, size_t MaxSize,
- const Vector<uint8_t> &Mask);
+ const std::vector<uint8_t> &Mask);
/// Applies one of the default mutations. Provided as a service
/// to mutation authors.
size_t AddWordFromDictionary(Dictionary &D, uint8_t *Data, size_t Size,
size_t MaxSize);
size_t MutateImpl(uint8_t *Data, size_t Size, size_t MaxSize,
- Vector<Mutator> &Mutators);
+ std::vector<Mutator> &Mutators);
size_t InsertPartOf(const uint8_t *From, size_t FromSize, uint8_t *To,
size_t ToSize, size_t MaxToSize);
// entries that led to successful discoveries in the past mutations.
Dictionary PersistentAutoDictionary;
- Vector<DictionaryEntry *> CurrentDictionaryEntrySequence;
+ std::vector<DictionaryEntry *> CurrentDictionaryEntrySequence;
static const size_t kCmpDictionaryEntriesDequeSize = 16;
DictionaryEntry CmpDictionaryEntriesDeque[kCmpDictionaryEntriesDequeSize];
size_t CmpDictionaryEntriesDequeIdx = 0;
const Unit *CrossOverWith = nullptr;
- Vector<uint8_t> MutateInPlaceHere;
- Vector<uint8_t> MutateWithMaskTemp;
+ std::vector<uint8_t> MutateInPlaceHere;
+ std::vector<uint8_t> MutateWithMaskTemp;
// CustomCrossOver needs its own buffer as a custom implementation may call
// LLVMFuzzerMutate, which in turn may resize MutateInPlaceHere.
- Vector<uint8_t> CustomCrossOverInPlaceHere;
+ std::vector<uint8_t> CustomCrossOverInPlaceHere;
- Vector<Mutator> Mutators;
- Vector<Mutator> DefaultMutators;
- Vector<Mutator> CurrentMutatorSequence;
+ std::vector<Mutator> Mutators;
+ std::vector<Mutator> DefaultMutators;
+ std::vector<Mutator> CurrentMutatorSequence;
};
} // namespace fuzzer
int ReportSlowUnits = 10;
bool OnlyASCII = false;
bool Entropic = true;
+ bool ForkCorpusGroups = false;
size_t EntropicFeatureFrequencyThreshold = 0xFF;
size_t EntropicNumberOfRarestFeatures = 100;
bool EntropicScalePerExecTime = false;
// so we return (pc-2) in that case in order to be safe.
// For A32 mode we return (pc-4) because all instructions are 32 bit long.
return (PC - 3) & (~1);
-#elif defined(__powerpc__) || defined(__powerpc64__) || defined(__aarch64__)
- // PCs are always 4 byte aligned.
- return PC - 4;
#elif defined(__sparc__) || defined(__mips__)
return PC - 8;
-#else
+#elif defined(__riscv__)
+ return PC - 2;
+#elif defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) || defined(_M_X64)
return PC - 1;
+#else
+ return PC - 4;
#endif
}
}
void TracePC::UpdateObservedPCs() {
- Vector<uintptr_t> CoveredFuncs;
+ std::vector<uintptr_t> CoveredFuncs;
auto ObservePC = [&](const PCTableEntry *TE) {
if (ObservedPCs.insert(TE).second && DoPrintNewPCs) {
PrintPC("\tNEW_PC: %p %F %L", "\tNEW_PC: %p",
FunctionStr = FunctionStr.substr(3);
std::string LineStr = DescribePC("%l", VisualizePC);
size_t NumEdges = Last - First;
- Vector<uintptr_t> UncoveredPCs;
- Vector<uintptr_t> CoveredPCs;
+ std::vector<uintptr_t> UncoveredPCs;
+ std::vector<uintptr_t> CoveredPCs;
for (auto TE = First; TE < Last; TE++)
if (!ObservedPCs.count(TE))
UncoveredPCs.push_back(TE->PC);
ValueProfileMap.AddValue(PC * 128 + 64 + AbsoluteDistance);
}
+ATTRIBUTE_NO_SANITIZE_MEMORY
static size_t InternalStrnlen(const char *S, size_t MaxLen) {
size_t Len = 0;
for (; Len < MaxLen && S[Len]; Len++) {}
}
// Finds min of (strlen(S1), strlen(S2)).
-// Needed bacause one of these strings may actually be non-zero terminated.
+// Needed because one of these strings may actually be non-zero terminated.
+ATTRIBUTE_NO_SANITIZE_MEMORY
static size_t InternalStrnlen2(const char *S1, const char *S2) {
size_t Len = 0;
for (; S1[Len] && S2[Len]; Len++) {}
size_t NumPCTables;
size_t NumPCsInPCTables;
- Set<const PCTableEntry*> ObservedPCs;
+ std::set<const PCTableEntry *> ObservedPCs;
std::unordered_map<uintptr_t, uintptr_t> ObservedFuncs; // PC => Counter.
uint8_t *FocusFunctionCounterPtr = nullptr;
else if (Byte >= 32 && Byte < 127)
Printf("%c", Byte);
else
- Printf("\\x%02x", Byte);
+ Printf("\\%03o", Byte);
}
void PrintASCII(const uint8_t *Data, size_t Size, const char *PrintAfter) {
return true;
}
-bool ParseDictionaryFile(const std::string &Text, Vector<Unit> *Units) {
+bool ParseDictionaryFile(const std::string &Text, std::vector<Unit> *Units) {
if (Text.empty()) {
Printf("ParseDictionaryFile: file does not exist or is empty\n");
return false;
const void *SearchMemory(const void *haystack, size_t haystacklen,
const void *needle, size_t needlelen);
-std::string CloneArgsWithoutX(const Vector<std::string> &Args,
+std::string CloneArgsWithoutX(const std::vector<std::string> &Args,
const char *X1, const char *X2);
-inline std::string CloneArgsWithoutX(const Vector<std::string> &Args,
+inline std::string CloneArgsWithoutX(const std::vector<std::string> &Args,
const char *X) {
return CloneArgsWithoutX(Args, X, X);
}
namespace {
+// The signal handler thread uses Zircon exceptions to resume crashed threads
+// into libFuzzer's POSIX signal handlers. The associated event is used to
+// signal when the thread is running, and when it should stop.
+std::thread SignalHandler;
+zx_handle_t SignalHandlerEvent = ZX_HANDLE_INVALID;
+
// Helper function to handle Zircon syscall failures.
void ExitOnErr(zx_status_t Status, const char *Syscall) {
if (Status != ZX_OK) {
}
}
-// CFAOffset is used to reference the stack pointer before entering the
-// trampoline (Stack Pointer + CFAOffset = prev Stack Pointer). Before jumping
-// to the trampoline we copy all the registers onto the stack. We need to make
-// sure that the new stack has enough space to store all the registers.
-//
-// The trampoline holds CFI information regarding the registers stored in the
-// stack, which is then used by the unwinder to restore them.
-#if defined(__x86_64__)
-// In x86_64 the crashing function might also be using the red zone (128 bytes
-// on top of their rsp).
-constexpr size_t CFAOffset = 128 + sizeof(zx_thread_state_general_regs_t);
-#elif defined(__aarch64__)
-// In aarch64 we need to always have the stack pointer aligned to 16 bytes, so we
-// make sure that we are keeping that same alignment.
-constexpr size_t CFAOffset = (sizeof(zx_thread_state_general_regs_t) + 15) & -(uintptr_t)16;
-#endif
-
// For the crash handler, we need to call Fuzzer::StaticCrashSignalCallback
// without POSIX signal handlers. To achieve this, we use an assembly function
// to add the necessary CFI unwinding information and a C function to bridge
// Produces an assembler immediate operand for the named or numbered register.
// This operand contains the offset of the register relative to the CFA.
-#define ASM_OPERAND_REG(reg) \
- [reg] "i"(offsetof(zx_thread_state_general_regs_t, reg) - CFAOffset),
-#define ASM_OPERAND_NUM(num) \
- [x##num] "i"(offsetof(zx_thread_state_general_regs_t, r[num]) - CFAOffset),
+#define ASM_OPERAND_REG(reg) \
+ [reg] "i"(offsetof(zx_thread_state_general_regs_t, reg)),
+#define ASM_OPERAND_NUM(num) \
+ [x##num] "i"(offsetof(zx_thread_state_general_regs_t, r[num])),
// Trampoline to bridge from the assembly below to the static C++ crash
// callback.
}
}
-// Creates the trampoline with the necessary CFI information to unwind through
-// to the crashing call stack:
-// * Defining the CFA so that it points to the stack pointer at the point
-// of crash.
-// * Storing all registers at the point of crash in the stack and refer to them
-// via CFI information (relative to the CFA).
-// * Setting the return column so the unwinder knows how to continue unwinding.
-// * (x86_64) making sure rsp is aligned before calling StaticCrashHandler.
-// * Calling StaticCrashHandler that will trigger the unwinder.
+// This trampoline function has the necessary CFI information to unwind
+// and get a backtrace:
+// * The stack contains a copy of all the registers at the point of crash,
+// the code has CFI directives specifying how to restore them.
+// * A call to StaticCrashHandler, which will print the stacktrace and exit
+// the fuzzer, generating a crash artifact.
//
// The __attribute__((used)) is necessary because the function
// is never called; it's just a container around the assembly to allow it to
// use operands for compile-time computed constants.
__attribute__((used))
void MakeTrampoline() {
- __asm__(".cfi_endproc\n"
- ".pushsection .text.CrashTrampolineAsm\n"
- ".type CrashTrampolineAsm,STT_FUNC\n"
-"CrashTrampolineAsm:\n"
- ".cfi_startproc simple\n"
- ".cfi_signal_frame\n"
+ __asm__(
+ ".cfi_endproc\n"
+ ".pushsection .text.CrashTrampolineAsm\n"
+ ".type CrashTrampolineAsm,STT_FUNC\n"
+ "CrashTrampolineAsm:\n"
+ ".cfi_startproc simple\n"
+ ".cfi_signal_frame\n"
#if defined(__x86_64__)
- ".cfi_return_column rip\n"
- ".cfi_def_cfa rsp, %c[CFAOffset]\n"
- FOREACH_REGISTER(CFI_OFFSET_REG, CFI_OFFSET_NUM)
- "mov %%rsp, %%rbp\n"
- ".cfi_def_cfa_register rbp\n"
- "andq $-16, %%rsp\n"
- "call %c[StaticCrashHandler]\n"
- "ud2\n"
+ ".cfi_return_column rip\n"
+ ".cfi_def_cfa rsp, 0\n"
+ FOREACH_REGISTER(CFI_OFFSET_REG, CFI_OFFSET_NUM)
+ "call %c[StaticCrashHandler]\n"
+ "ud2\n"
#elif defined(__aarch64__)
- ".cfi_return_column 33\n"
- ".cfi_def_cfa sp, %c[CFAOffset]\n"
- FOREACH_REGISTER(CFI_OFFSET_REG, CFI_OFFSET_NUM)
- ".cfi_offset 33, %c[pc]\n"
- ".cfi_offset 30, %c[lr]\n"
- "bl %c[StaticCrashHandler]\n"
- "brk 1\n"
+ ".cfi_return_column 33\n"
+ ".cfi_def_cfa sp, 0\n"
+ FOREACH_REGISTER(CFI_OFFSET_REG, CFI_OFFSET_NUM)
+ ".cfi_offset 33, %c[pc]\n"
+ ".cfi_offset 30, %c[lr]\n"
+ "bl %c[StaticCrashHandler]\n"
+ "brk 1\n"
#else
#error "Unsupported architecture for fuzzing on Fuchsia"
#endif
- ".cfi_endproc\n"
- ".size CrashTrampolineAsm, . - CrashTrampolineAsm\n"
- ".popsection\n"
- ".cfi_startproc\n"
- : // No outputs
- : FOREACH_REGISTER(ASM_OPERAND_REG, ASM_OPERAND_NUM)
+ ".cfi_endproc\n"
+ ".size CrashTrampolineAsm, . - CrashTrampolineAsm\n"
+ ".popsection\n"
+ ".cfi_startproc\n"
+ : // No outputs
+ : FOREACH_REGISTER(ASM_OPERAND_REG, ASM_OPERAND_NUM)
#if defined(__aarch64__)
- ASM_OPERAND_REG(pc)
- ASM_OPERAND_REG(lr)
+ ASM_OPERAND_REG(pc) ASM_OPERAND_REG(lr)
#endif
- [StaticCrashHandler] "i" (StaticCrashHandler),
- [CFAOffset] "i" (CFAOffset));
+ [StaticCrashHandler] "i"(StaticCrashHandler));
}
-void CrashHandler(zx_handle_t *Event) {
+void CrashHandler() {
+ assert(SignalHandlerEvent != ZX_HANDLE_INVALID);
+
// This structure is used to ensure we close handles to objects we create in
// this handler.
struct ScopedHandle {
Self, ZX_EXCEPTION_CHANNEL_DEBUGGER, &Channel.Handle),
"_zx_task_create_exception_channel");
- ExitOnErr(_zx_object_signal(*Event, 0, ZX_USER_SIGNAL_0),
+ ExitOnErr(_zx_object_signal(SignalHandlerEvent, 0, ZX_USER_SIGNAL_0),
"_zx_object_signal");
// This thread lives as long as the process in order to keep handling
// crashes. In practice, the first crashed thread to reach the end of the
// StaticCrashHandler will end the process.
while (true) {
- ExitOnErr(_zx_object_wait_one(Channel.Handle, ZX_CHANNEL_READABLE,
- ZX_TIME_INFINITE, nullptr),
- "_zx_object_wait_one");
+ zx_wait_item_t WaitItems[] = {
+ {
+ .handle = SignalHandlerEvent,
+ .waitfor = ZX_SIGNAL_HANDLE_CLOSED,
+ .pending = 0,
+ },
+ {
+ .handle = Channel.Handle,
+ .waitfor = ZX_CHANNEL_READABLE | ZX_CHANNEL_PEER_CLOSED,
+ .pending = 0,
+ },
+ };
+ auto Status = _zx_object_wait_many(
+ WaitItems, sizeof(WaitItems) / sizeof(WaitItems[0]), ZX_TIME_INFINITE);
+ if (Status != ZX_OK || (WaitItems[1].pending & ZX_CHANNEL_READABLE) == 0) {
+ break;
+ }
zx_exception_info_t ExceptionInfo;
ScopedHandle Exception;
// onto the stack and jump into a trampoline with CFI instructions on how
// to restore it.
#if defined(__x86_64__)
- uintptr_t StackPtr = GeneralRegisters.rsp - CFAOffset;
+ uintptr_t StackPtr =
+ (GeneralRegisters.rsp - (128 + sizeof(GeneralRegisters))) &
+ -(uintptr_t)16;
__unsanitized_memcpy(reinterpret_cast<void *>(StackPtr), &GeneralRegisters,
sizeof(GeneralRegisters));
GeneralRegisters.rsp = StackPtr;
GeneralRegisters.rip = reinterpret_cast<zx_vaddr_t>(CrashTrampolineAsm);
#elif defined(__aarch64__)
- uintptr_t StackPtr = GeneralRegisters.sp - CFAOffset;
+ uintptr_t StackPtr =
+ (GeneralRegisters.sp - sizeof(GeneralRegisters)) & -(uintptr_t)16;
__unsanitized_memcpy(reinterpret_cast<void *>(StackPtr), &GeneralRegisters,
sizeof(GeneralRegisters));
GeneralRegisters.sp = StackPtr;
}
}
+void StopSignalHandler() {
+ _zx_handle_close(SignalHandlerEvent);
+ if (SignalHandler.joinable()) {
+ SignalHandler.join();
+ }
+}
+
} // namespace
// Platform specific functions.
return;
// Set up the crash handler and wait until it is ready before proceeding.
- zx_handle_t Event;
- ExitOnErr(_zx_event_create(0, &Event), "_zx_event_create");
+ ExitOnErr(_zx_event_create(0, &SignalHandlerEvent), "_zx_event_create");
- std::thread T(CrashHandler, &Event);
- zx_status_t Status =
- _zx_object_wait_one(Event, ZX_USER_SIGNAL_0, ZX_TIME_INFINITE, nullptr);
- _zx_handle_close(Event);
+ SignalHandler = std::thread(CrashHandler);
+ zx_status_t Status = _zx_object_wait_one(SignalHandlerEvent, ZX_USER_SIGNAL_0,
+ ZX_TIME_INFINITE, nullptr);
ExitOnErr(Status, "_zx_object_wait_one");
- T.detach();
+ std::atexit(StopSignalHandler);
}
void SleepSeconds(int Seconds) {
#if LIBFUZZER_LINUX || LIBFUZZER_NETBSD || LIBFUZZER_FREEBSD || \
LIBFUZZER_EMSCRIPTEN
#include "FuzzerCommand.h"
+#include "FuzzerInternal.h"
+#include <signal.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/wait.h>
int exit_code = system(CmdLine.c_str());
if (WIFEXITED(exit_code))
return WEXITSTATUS(exit_code);
+ if (WIFSIGNALED(exit_code) && WTERMSIG(exit_code) == SIGINT)
+ return Fuzzer::InterruptExitCode();
return exit_code;
}
}
std::string DisassembleCmd(const std::string &FileName) {
- Vector<std::string> command_vector;
+ std::vector<std::string> command_vector;
command_vector.push_back("dumpbin /summary > nul");
if (ExecuteCommand(Command(command_vector)) == 0)
return "dumpbin /disasm " + FileName;
if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND
COMPILER_RT_LIBCXX_PATH AND
COMPILER_RT_LIBCXXABI_PATH)
- list(APPEND LIBFUZZER_UNITTEST_CFLAGS -nostdinc++)
+ list(APPEND LIBFUZZER_UNITTEST_CFLAGS -nostdinc++ -fno-exceptions)
+ list(APPEND LIBFUZZER_UNITTEST_LINK_FLAGS -nostdlib++ -fno-exceptions)
endif()
if ("-fvisibility=hidden" IN_LIST LIBFUZZER_CFLAGS)
FuzzerUnitTests "Fuzzer-${arch}-Test" ${arch}
SOURCES FuzzerUnittest.cpp ${COMPILER_RT_GTEST_SOURCE}
RUNTIME ${LIBFUZZER_TEST_RUNTIME}
- DEPS gtest ${LIBFUZZER_TEST_RUNTIME_DEPS}
+ DEPS llvm_gtest ${LIBFUZZER_TEST_RUNTIME_DEPS}
CFLAGS ${LIBFUZZER_UNITTEST_CFLAGS} ${LIBFUZZER_TEST_RUNTIME_CFLAGS}
LINK_FLAGS ${LIBFUZZER_UNITTEST_LINK_FLAGS} ${LIBFUZZER_TEST_RUNTIME_LINK_FLAGS})
set_target_properties(FuzzerUnitTests PROPERTIES
generate_compiler_rt_tests(FuzzedDataProviderTestObjects
FuzzedDataProviderUnitTests "FuzzerUtils-${arch}-Test" ${arch}
SOURCES FuzzedDataProviderUnittest.cpp ${COMPILER_RT_GTEST_SOURCE}
- DEPS gtest ${LIBFUZZER_TEST_RUNTIME_DEPS} ${COMPILER_RT_SOURCE_DIR}/include/fuzzer/FuzzedDataProvider.h
+ DEPS llvm_gtest ${LIBFUZZER_TEST_RUNTIME_DEPS} ${COMPILER_RT_SOURCE_DIR}/include/fuzzer/FuzzedDataProvider.h
CFLAGS ${LIBFUZZER_UNITTEST_CFLAGS} ${LIBFUZZER_TEST_RUNTIME_CFLAGS}
LINK_FLAGS ${LIBFUZZER_UNITTEST_LINK_FLAGS} ${LIBFUZZER_TEST_RUNTIME_LINK_FLAGS})
set_target_properties(FuzzedDataProviderUnitTests PROPERTIES
using namespace fuzzer;
// For now, have LLVMFuzzerTestOneInput just to make it link.
-// Later we may want to make unittests that actually call LLVMFuzzerTestOneInput.
+// Later we may want to make unittests that actually call
+// LLVMFuzzerTestOneInput.
extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
abort();
}
{ 0, 5, 6, 7, 1, 2 }
};
for (size_t Len = 1; Len < 8; Len++) {
- Set<Unit> FoundUnits, ExpectedUnitsWitThisLength;
+ std::set<Unit> FoundUnits, ExpectedUnitsWitThisLength;
for (int Iter = 0; Iter < 3000; Iter++) {
C.resize(Len);
size_t NewSize = MD->CrossOver(A.data(), A.size(), B.data(), B.size(),
}
TEST(FuzzerMutate, InsertRepeatedBytes1) {
- TestInsertRepeatedBytes(&MutationDispatcher::Mutate_InsertRepeatedBytes, 10000);
+ TestInsertRepeatedBytes(&MutationDispatcher::Mutate_InsertRepeatedBytes,
+ 10000);
}
TEST(FuzzerMutate, InsertRepeatedBytes2) {
TestInsertRepeatedBytes(&MutationDispatcher::Mutate, 300000);
}
TEST(FuzzerDictionary, ParseDictionaryFile) {
- Vector<Unit> Units;
+ std::vector<Unit> Units;
EXPECT_FALSE(ParseDictionaryFile("zzz\n", &Units));
EXPECT_FALSE(ParseDictionaryFile("", &Units));
EXPECT_TRUE(ParseDictionaryFile("\n", &Units));
EXPECT_TRUE(ParseDictionaryFile(" #zzzz\n", &Units));
EXPECT_EQ(Units.size(), 0U);
EXPECT_TRUE(ParseDictionaryFile(" #zzzz\naaa=\"aa\"", &Units));
- EXPECT_EQ(Units, Vector<Unit>({Unit({'a', 'a'})}));
+ EXPECT_EQ(Units, std::vector<Unit>({Unit({'a', 'a'})}));
EXPECT_TRUE(
ParseDictionaryFile(" #zzzz\naaa=\"aa\"\n\nabc=\"abc\"", &Units));
EXPECT_EQ(Units,
- Vector<Unit>({Unit({'a', 'a'}), Unit({'a', 'b', 'c'})}));
+ std::vector<Unit>({Unit({'a', 'a'}), Unit({'a', 'b', 'c'})}));
}
TEST(FuzzerUtil, Base64) {
EXPECT_EQ("YWJjeHl6", Base64({'a', 'b', 'c', 'x', 'y', 'z'}));
}
+#ifdef __GLIBC__
+class PrintfCapture {
+ public:
+ PrintfCapture() {
+ OldOutputFile = GetOutputFile();
+ SetOutputFile(open_memstream(&Buffer, &Size));
+ }
+ ~PrintfCapture() {
+ fclose(GetOutputFile());
+ SetOutputFile(OldOutputFile);
+ free(Buffer);
+ }
+ std::string str() { return std::string(Buffer, Size); }
+
+ private:
+ char *Buffer;
+ size_t Size;
+ FILE *OldOutputFile;
+};
+
+TEST(FuzzerUtil, PrintASCII) {
+ auto f = [](const char *Str, const char *PrintAfter = "") {
+ PrintfCapture Capture;
+ PrintASCII(reinterpret_cast<const uint8_t*>(Str), strlen(Str), PrintAfter);
+ return Capture.str();
+ };
+ EXPECT_EQ("hello", f("hello"));
+ EXPECT_EQ("c:\\\\", f("c:\\"));
+ EXPECT_EQ("\\\"hi\\\"", f("\"hi\""));
+ EXPECT_EQ("\\011a", f("\ta"));
+ EXPECT_EQ("\\0111", f("\t1"));
+ EXPECT_EQ("hello\\012", f("hello\n"));
+ EXPECT_EQ("hello\n", f("hello", "\n"));
+}
+#endif
+
TEST(Corpus, Distribution) {
DataFlowTrace DFT;
Random Rand(0);
/*FeatureSet*/ {}, DFT,
/*BaseII*/ nullptr);
- Vector<size_t> Hist(N);
+ std::vector<size_t> Hist(N);
for (size_t i = 0; i < N * TriesPerUnit; i++) {
Hist[C->ChooseUnitIdxToMutate(Rand)]++;
}
}
}
-template <typename T> void EQ(const Vector<T> &A, const Vector<T> &B) {
+TEST(Corpus, Replace) {
+ DataFlowTrace DFT;
+ struct EntropicOptions Entropic = {false, 0xFF, 100, false};
+ std::unique_ptr<InputCorpus> C(
+ new InputCorpus(/*OutputCorpus*/ "", Entropic));
+ InputInfo *FirstII =
+ C->AddToCorpus(Unit{0x01, 0x00}, /*NumFeatures*/ 1,
+ /*MayDeleteFile*/ false, /*HasFocusFunction*/ false,
+ /*ForceAddToCorpus*/ false,
+ /*TimeOfUnit*/ std::chrono::microseconds(1234),
+ /*FeatureSet*/ {}, DFT,
+ /*BaseII*/ nullptr);
+ InputInfo *SecondII =
+ C->AddToCorpus(Unit{0x02}, /*NumFeatures*/ 1,
+ /*MayDeleteFile*/ false, /*HasFocusFunction*/ false,
+ /*ForceAddToCorpus*/ false,
+ /*TimeOfUnit*/ std::chrono::microseconds(5678),
+ /*FeatureSet*/ {}, DFT,
+ /*BaseII*/ nullptr);
+ Unit ReplacedU = Unit{0x03};
+
+ C->Replace(FirstII, ReplacedU,
+ /*TimeOfUnit*/ std::chrono::microseconds(321));
+
+ // FirstII should be replaced.
+ EXPECT_EQ(FirstII->U, Unit{0x03});
+ EXPECT_EQ(FirstII->Reduced, true);
+ EXPECT_EQ(FirstII->TimeOfUnit, std::chrono::microseconds(321));
+ std::vector<uint8_t> ExpectedSha1(kSHA1NumBytes);
+ ComputeSHA1(ReplacedU.data(), ReplacedU.size(), ExpectedSha1.data());
+ std::vector<uint8_t> IISha1(FirstII->Sha1, FirstII->Sha1 + kSHA1NumBytes);
+ EXPECT_EQ(IISha1, ExpectedSha1);
+
+ // SecondII should not be replaced.
+ EXPECT_EQ(SecondII->U, Unit{0x02});
+ EXPECT_EQ(SecondII->Reduced, false);
+ EXPECT_EQ(SecondII->TimeOfUnit, std::chrono::microseconds(5678));
+}
+
+template <typename T>
+void EQ(const std::vector<T> &A, const std::vector<T> &B) {
EXPECT_EQ(A, B);
}
-template <typename T> void EQ(const Set<T> &A, const Vector<T> &B) {
- EXPECT_EQ(A, Set<T>(B.begin(), B.end()));
+template <typename T> void EQ(const std::set<T> &A, const std::vector<T> &B) {
+ EXPECT_EQ(A, std::set<T>(B.begin(), B.end()));
}
-void EQ(const Vector<MergeFileInfo> &A, const Vector<std::string> &B) {
- Set<std::string> a;
+void EQ(const std::vector<MergeFileInfo> &A,
+ const std::vector<std::string> &B) {
+ std::set<std::string> a;
for (const auto &File : A)
a.insert(File.Name);
- Set<std::string> b(B.begin(), B.end());
+ std::set<std::string> b(B.begin(), B.end());
EXPECT_EQ(a, b);
}
TEST(Merger, Merge) {
Merger M;
- Set<uint32_t> Features, NewFeatures;
- Set<uint32_t> Cov, NewCov;
- Vector<std::string> NewFiles;
+ std::set<uint32_t> Features, NewFeatures;
+ std::set<uint32_t> Cov, NewCov;
+ std::vector<std::string> NewFiles;
// Adds new files and features
EXPECT_TRUE(M.Parse("3\n0\nA\nB\nC\n"
TRACED_EQ(NewFeatures, {1, 2, 3});
}
+TEST(Merger, SetCoverMerge) {
+ Merger M;
+ std::set<uint32_t> Features, NewFeatures;
+ std::set<uint32_t> Cov, NewCov;
+ std::vector<std::string> NewFiles;
+
+ // Adds new files and features
+ EXPECT_TRUE(M.Parse("3\n0\nA\nB\nC\n"
+ "STARTED 0 1000\n"
+ "FT 0 1 2 3\n"
+ "STARTED 1 1001\n"
+ "FT 1 4 5 6 \n"
+ "STARTED 2 1002\n"
+ "FT 2 6 1 3\n"
+ "",
+ true));
+ EXPECT_EQ(M.SetCoverMerge(Features, &NewFeatures, Cov, &NewCov, &NewFiles),
+ 6U);
+ TRACED_EQ(M.Files, {"A", "B", "C"});
+ TRACED_EQ(NewFiles, {"A", "B"});
+ TRACED_EQ(NewFeatures, {1, 2, 3, 4, 5, 6});
+
+ // Doesn't return features or files in the initial corpus.
+ EXPECT_TRUE(M.Parse("3\n1\nA\nB\nC\n"
+ "STARTED 0 1000\n"
+ "FT 0 1 2 3\n"
+ "STARTED 1 1001\n"
+ "FT 1 4 5 6 \n"
+ "STARTED 2 1002\n"
+ "FT 2 6 1 3\n"
+ "",
+ true));
+ EXPECT_EQ(M.SetCoverMerge(Features, &NewFeatures, Cov, &NewCov, &NewFiles),
+ 3U);
+ TRACED_EQ(M.Files, {"A", "B", "C"});
+ TRACED_EQ(NewFiles, {"B"});
+ TRACED_EQ(NewFeatures, {4, 5, 6});
+
+ // No new features, so no new files
+ EXPECT_TRUE(M.Parse("3\n2\nA\nB\nC\n"
+ "STARTED 0 1000\n"
+ "FT 0 1 2 3\n"
+ "STARTED 1 1001\n"
+ "FT 1 4 5 6 \n"
+ "STARTED 2 1002\n"
+ "FT 2 6 1 3\n"
+ "",
+ true));
+ EXPECT_EQ(M.SetCoverMerge(Features, &NewFeatures, Cov, &NewCov, &NewFiles),
+ 0U);
+ TRACED_EQ(M.Files, {"A", "B", "C"});
+ TRACED_EQ(NewFiles, {});
+ TRACED_EQ(NewFeatures, {});
+
+ // Can pass initial features and coverage.
+ Features = {1, 2, 3};
+ Cov = {};
+ EXPECT_TRUE(M.Parse("2\n0\nA\nB\n"
+ "STARTED 0 1000\n"
+ "FT 0 1 2 3\n"
+ "STARTED 1 1001\n"
+ "FT 1 4 5 6\n"
+ "",
+ true));
+ EXPECT_EQ(M.SetCoverMerge(Features, &NewFeatures, Cov, &NewCov, &NewFiles),
+ 3U);
+ TRACED_EQ(M.Files, {"A", "B"});
+ TRACED_EQ(NewFiles, {"B"});
+ TRACED_EQ(NewFeatures, {4, 5, 6});
+ Features.clear();
+ Cov.clear();
+
+ // Prefer files with a lot of features first (C has 4 features)
+ // Then prefer B over A due to the smaller size. After choosing C and B,
+ // A and D have no new features to contribute.
+ EXPECT_TRUE(M.Parse("4\n0\nA\nB\nC\nD\n"
+ "STARTED 0 2000\n"
+ "FT 0 3 5 6\n"
+ "STARTED 1 1000\n"
+ "FT 1 4 5 6 \n"
+ "STARTED 2 1000\n"
+ "FT 2 1 2 3 4 \n"
+ "STARTED 3 500\n"
+ "FT 3 1 \n"
+ "",
+ true));
+ EXPECT_EQ(M.SetCoverMerge(Features, &NewFeatures, Cov, &NewCov, &NewFiles),
+ 6U);
+ TRACED_EQ(M.Files, {"A", "B", "C", "D"});
+ TRACED_EQ(NewFiles, {"C", "B"});
+ TRACED_EQ(NewFeatures, {1, 2, 3, 4, 5, 6});
+
+ // Only 1 file covers all features.
+ EXPECT_TRUE(M.Parse("4\n1\nA\nB\nC\nD\n"
+ "STARTED 0 2000\n"
+ "FT 0 4 5 6 7 8\n"
+ "STARTED 1 1100\n"
+ "FT 1 1 2 3 \n"
+ "STARTED 2 1100\n"
+ "FT 2 2 3 \n"
+ "STARTED 3 1000\n"
+ "FT 3 1 \n"
+ "",
+ true));
+ EXPECT_EQ(M.SetCoverMerge(Features, &NewFeatures, Cov, &NewCov, &NewFiles),
+ 3U);
+ TRACED_EQ(M.Files, {"A", "B", "C", "D"});
+ TRACED_EQ(NewFiles, {"B"});
+ TRACED_EQ(NewFeatures, {1, 2, 3});
+
+ // A Feature has a value greater than (1 << 21) and hence
+ // there are collisions in the underlying `covered features`
+ // bitvector.
+ EXPECT_TRUE(M.Parse("3\n0\nA\nB\nC\n"
+ "STARTED 0 2000\n"
+ "FT 0 1 2 3\n"
+ "STARTED 1 1000\n"
+ "FT 1 3 4 5 \n"
+ "STARTED 2 1000\n"
+ "FT 2 3 2097153 \n" // Last feature is (2^21 + 1).
+ "",
+ true));
+ EXPECT_EQ(M.SetCoverMerge(Features, &NewFeatures, Cov, &NewCov, &NewFiles),
+ 5U);
+ TRACED_EQ(M.Files, {"A", "B", "C"});
+ // File 'C' is not added because it's last feature is considered
+ // covered due to collision with feature 1.
+ TRACED_EQ(NewFiles, {"B", "A"});
+ TRACED_EQ(NewFeatures, {1, 2, 3, 4, 5});
+}
+
#undef TRACED_EQ
TEST(DFT, BlockCoverage) {
0, 0, 0, 0, 0, 0, 0, 8,
9, 9, 9, 9, 9, 9, 9, 9,
};
- typedef Vector<std::pair<size_t, uint8_t> > Vec;
+ typedef std::vector<std::pair<size_t, uint8_t>> Vec;
Vec Res, Expected;
auto CB = [&](size_t FirstFeature, size_t Idx, uint8_t V) {
Res.push_back({FirstFeature + Idx, V});
// FuzzerCommand unit tests. The arguments in the two helper methods below must
// match.
-static void makeCommandArgs(Vector<std::string> *ArgsToAdd) {
+static void makeCommandArgs(std::vector<std::string> *ArgsToAdd) {
assert(ArgsToAdd);
ArgsToAdd->clear();
ArgsToAdd->push_back("foo");
EXPECT_EQ(CmdLine, "");
// Explicit constructor
- Vector<std::string> ArgsToAdd;
+ std::vector<std::string> ArgsToAdd;
makeCommandArgs(&ArgsToAdd);
Command InitializedCmd(ArgsToAdd);
}
TEST(FuzzerCommand, ModifyArguments) {
- Vector<std::string> ArgsToAdd;
+ std::vector<std::string> ArgsToAdd;
makeCommandArgs(&ArgsToAdd);
Command Cmd;
std::string CmdLine;
}
TEST(FuzzerCommand, ModifyFlags) {
- Vector<std::string> ArgsToAdd;
+ std::vector<std::string> ArgsToAdd;
makeCommandArgs(&ArgsToAdd);
Command Cmd(ArgsToAdd);
std::string Value, CmdLine;
}
TEST(FuzzerCommand, SetOutput) {
- Vector<std::string> ArgsToAdd;
+ std::vector<std::string> ArgsToAdd;
makeCommandArgs(&ArgsToAdd);
Command Cmd(ArgsToAdd);
std::string CmdLine;
struct EntropicOptions Entropic = {true, 0xFF, 100, false};
std::unique_ptr<InputCorpus> C(new InputCorpus("", Entropic));
std::unique_ptr<InputInfo> II(new InputInfo());
- Vector<std::pair<uint32_t, uint16_t>> FeatureFreqs = {{1, 3}, {2, 3}, {3, 3}};
+ std::vector<std::pair<uint32_t, uint16_t>> FeatureFreqs = {
+ {1, 3}, {2, 3}, {3, 3}};
II->FeatureFreqs = FeatureFreqs;
II->NumExecutedMutations = 0;
II->UpdateEnergy(4, false, std::chrono::microseconds(0));
set(GWP_ASAN_CFLAGS ${SANITIZER_COMMON_CFLAGS} -fno-rtti -fno-exceptions
-nostdinc++ -pthread -fno-omit-frame-pointer)
append_list_if(COMPILER_RT_HAS_FPIC_FLAG -fPIC GWP_ASAN_CFLAGS)
+# append_list_if(COMPILER_RT_HAS_SANITIZER_COMMON ${SANITIZER_COMMON_CFLAGS} GWP_ASAN_CFLAGS)
# Remove -stdlib= which is unused when passing -nostdinc++.
-string(REGEX REPLACE "-stdlib=[a-zA-Z+]*" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
+string(REGEX REPLACE "-stdlib=[a-zA-Z+]*" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
# Options parsing support is optional. This is an optional library that can be
# used by an allocator to automatically parse GwpAsan options from the
return addrToSlot(this, Ptr + PageSize); // Round up.
}
+uintptr_t AllocatorState::internallyDetectedErrorFaultAddress() const {
+ return GuardedPagePoolEnd - 0x10;
+}
+
} // namespace gwp_asan
#include <stdint.h>
namespace gwp_asan {
-enum class Error {
+
+// Magic header that resides in the AllocatorState so that GWP-ASan bugreports
+// can be understood by tools at different versions. Out-of-process crash
+// handlers, like crashpad on Fuchsia, take the raw contents of the
+// AllocationMetatada array and the AllocatorState, and shove them into the
+// minidump. Online unpacking of these structs needs to know from which version
+// of GWP-ASan it's extracting the information, as the structures are not
+// stable.
+struct AllocatorVersionMagic {
+ // The values are copied into the structure at runtime, during
+ // `GuardedPoolAllocator::init()` so that GWP-ASan remains completely in the
+ // `.bss` segment.
+ static constexpr uint8_t kAllocatorVersionMagic[4] = {'A', 'S', 'A', 'N'};
+ uint8_t Magic[4] = {};
+ // Update the version number when the AllocatorState or AllocationMetadata
+ // change.
+ static constexpr uint16_t kAllocatorVersion = 2;
+ uint16_t Version = 0;
+ uint16_t Reserved = 0;
+};
+
+enum class Error : uint8_t {
UNKNOWN,
USE_AFTER_FREE,
DOUBLE_FREE,
// Whether this allocation has been deallocated yet.
bool IsDeallocated = false;
+
+ // In recoverable mode, whether this allocation has had a crash associated
+ // with it. This has certain side effects, like meaning this allocation will
+ // permanently occupy a slot, and won't ever have another crash reported from
+ // it.
+ bool HasCrashed = false;
};
// This holds the state that's shared between the GWP-ASan allocator and the
// set of information required for understanding a GWP-ASan crash.
struct AllocatorState {
constexpr AllocatorState() {}
+ AllocatorVersionMagic VersionMagic{};
// Returns whether the provided pointer is a current sampled allocation that
// is owned by this pool.
// must be within memory owned by this pool, else the result is undefined.
bool isGuardPage(uintptr_t Ptr) const;
+ // Returns the address that's used by __gwp_asan_get_internal_crash_address()
+ // and GPA::raiseInternallyDetectedError() to communicate that the SEGV in
+ // question comes from an internally-detected error.
+ uintptr_t internallyDetectedErrorFaultAddress() const;
+
// The number of guarded slots that this pool holds.
size_t MaxSimultaneousAllocations = 0;
uintptr_t FailureAddress = 0;
};
+// Below are various compile-time checks that the layout of the internal
+// GWP-ASan structures are undisturbed. If they are disturbed, the version magic
+// number needs to be increased by one, and the asserts need to be updated.
+// Out-of-process crash handlers, like breakpad/crashpad, may copy the internal
+// GWP-ASan structures into a minidump for offline reconstruction of the crash.
+// In order to accomplish this, the offline reconstructor needs to know the
+// version of GWP-ASan internal structures that it's unpacking (along with the
+// architecture-specific layout info, which is left as an exercise to the crash
+// handler).
+static_assert(offsetof(AllocatorState, VersionMagic) == 0, "");
+static_assert(sizeof(AllocatorVersionMagic) == 8, "");
+#if defined(__x86_64__)
+static_assert(sizeof(AllocatorState) == 56, "");
+static_assert(offsetof(AllocatorState, FailureAddress) == 48, "");
+static_assert(sizeof(AllocationMetadata) == 568, "");
+static_assert(offsetof(AllocationMetadata, IsDeallocated) == 560, "");
+#elif defined(__aarch64__)
+static_assert(sizeof(AllocatorState) == 56, "");
+static_assert(offsetof(AllocatorState, FailureAddress) == 48, "");
+static_assert(sizeof(AllocationMetadata) == 568, "");
+static_assert(offsetof(AllocationMetadata, IsDeallocated) == 560, "");
+#elif defined(__i386__)
+static_assert(sizeof(AllocatorState) == 32, "");
+static_assert(offsetof(AllocatorState, FailureAddress) == 28, "");
+static_assert(sizeof(AllocationMetadata) == 548, "");
+static_assert(offsetof(AllocationMetadata, IsDeallocated) == 544, "");
+#elif defined(__arm__)
+static_assert(sizeof(AllocatorState) == 32, "");
+static_assert(offsetof(AllocatorState, FailureAddress) == 28, "");
+static_assert(sizeof(AllocationMetadata) == 560, "");
+static_assert(offsetof(AllocationMetadata, IsDeallocated) == 552, "");
+#endif // defined($ARCHITECTURE)
+
} // namespace gwp_asan
#endif // GWP_ASAN_COMMON_H_
}
uintptr_t
-__gwp_asan_get_internal_crash_address(const gwp_asan::AllocatorState *State) {
+__gwp_asan_get_internal_crash_address(const gwp_asan::AllocatorState *State,
+ uintptr_t ErrorPtr) {
+ // There can be a race between internally- and externally-raised faults. The
+ // fault address from the signal handler is used to discriminate whether it's
+ // internally- or externally-raised, and the pool maintains a special page at
+ // the end of the GuardedPagePool specifically for the internally-raised
+ // faults.
+ if (ErrorPtr != State->internallyDetectedErrorFaultAddress())
+ return 0u;
return State->FailureAddress;
}
if (State->FailureType != Error::UNKNOWN)
return State->FailureType;
- // Let's try and figure out what the source of this error is.
+ // Check for use-after-free.
+ if (addrToMetadata(State, Metadata, ErrorPtr)->IsDeallocated)
+ return Error::USE_AFTER_FREE;
+
+ // Check for buffer-overflow. Because of allocation alignment or left/right
+ // page placement, we can have buffer-overflows that don't touch a guarded
+ // page, but these are not possible to detect unless it's also a
+ // use-after-free, which is handled above.
if (State->isGuardPage(ErrorPtr)) {
size_t Slot = State->getNearestSlot(ErrorPtr);
const AllocationMetadata *SlotMeta =
return Error::BUFFER_UNDERFLOW;
}
- // Access wasn't a guard page, check for use-after-free.
- const AllocationMetadata *SlotMeta =
- addrToMetadata(State, Metadata, ErrorPtr);
- if (SlotMeta->IsDeallocated) {
- return Error::USE_AFTER_FREE;
- }
-
// If we have reached here, the error is still unknown.
return Error::UNKNOWN;
}
const gwp_asan::AllocationMetadata *Metadata,
uintptr_t ErrorPtr);
-// For internally-detected errors (double free, invalid free), this function
-// returns the pointer that the error occurred at. If the error is unrelated to
-// GWP-ASan, or if the error was caused by a non-internally detected failure,
-// this function returns zero.
+// This function, provided the fault address from the signal handler, returns
+// the following values:
+// 1. If the crash was caused by an internally-detected error (invalid free,
+// double free), this function returns the pointer that was used for the
+// internally-detected bad operation (i.e. the pointer given to free()).
+// 2. For externally-detected crashes (use-after-free, buffer-overflow), this
+// function returns zero.
+// 3. If GWP-ASan wasn't responsible for the crash at all, this function also
+// returns zero.
uintptr_t
-__gwp_asan_get_internal_crash_address(const gwp_asan::AllocatorState *State);
+__gwp_asan_get_internal_crash_address(const gwp_asan::AllocatorState *State,
+ uintptr_t ErrorPtr);
// Returns a pointer to the metadata for the allocation that's responsible for
// the crash. This metadata should not be dereferenced directly due to API
#include "gwp_asan/guarded_pool_allocator.h"
+#include "gwp_asan/crash_handler.h"
#include "gwp_asan/options.h"
#include "gwp_asan/utilities.h"
SingletonPtr = this;
Backtrace = Opts.Backtrace;
+ State.VersionMagic = {{AllocatorVersionMagic::kAllocatorVersionMagic[0],
+ AllocatorVersionMagic::kAllocatorVersionMagic[1],
+ AllocatorVersionMagic::kAllocatorVersionMagic[2],
+ AllocatorVersionMagic::kAllocatorVersionMagic[3]},
+ AllocatorVersionMagic::kAllocatorVersion,
+ 0};
+
State.MaxSimultaneousAllocations = Opts.MaxSimultaneousAllocations;
const size_t PageSize = getPlatformPageSize();
assert((PageSize & (PageSize - 1)) == 0);
State.PageSize = PageSize;
+ // Number of pages required =
+ // + MaxSimultaneousAllocations * maximumAllocationSize (N pages per slot)
+ // + MaxSimultaneousAllocations (one guard on the left side of each slot)
+ // + 1 (an extra guard page at the end of the pool, on the right side)
+ // + 1 (an extra page that's used for reporting internally-detected crashes,
+ // like double free and invalid free, to the signal handler; see
+ // raiseInternallyDetectedError() for more info)
size_t PoolBytesRequired =
- PageSize * (1 + State.MaxSimultaneousAllocations) +
+ PageSize * (2 + State.MaxSimultaneousAllocations) +
State.MaxSimultaneousAllocations * State.maximumAllocationSize();
assert(PoolBytesRequired % PageSize == 0);
void *GuardedPoolMemory = reserveGuardedPool(PoolBytesRequired);
return reinterpret_cast<void *>(UserPtr);
}
-void GuardedPoolAllocator::trapOnAddress(uintptr_t Address, Error E) {
+void GuardedPoolAllocator::raiseInternallyDetectedError(uintptr_t Address,
+ Error E) {
+ // Disable the allocator before setting the internal failure state. In
+ // non-recoverable mode, the allocator will be permanently disabled, and so
+ // things will be accessed without locks.
+ disable();
+
+ // Races between internally- and externally-raised faults can happen. Right
+ // now, in this thread we've locked the allocator in order to raise an
+ // internally-detected fault, and another thread could SIGSEGV to raise an
+ // externally-detected fault. What will happen is that the other thread will
+ // wait in the signal handler, as we hold the allocator's locks from the
+ // disable() above. We'll trigger the signal handler by touching the
+ // internal-signal-raising address below, and the signal handler from our
+ // thread will get to run first as we will continue to hold the allocator
+ // locks until the enable() at the end of this function. Be careful though, if
+ // this thread receives another SIGSEGV after the disable() above, but before
+ // touching the internal-signal-raising address below, then this thread will
+ // get an "externally-raised" SIGSEGV while *also* holding the allocator
+ // locks, which means this thread's signal handler will deadlock. This could
+ // be resolved with a re-entrant lock, but asking platforms to implement this
+ // seems unnecessary given the only way to get a SIGSEGV in this critical
+ // section is either a memory safety bug in the couple lines of code below (be
+ // careful!), or someone outside uses `kill(this_thread, SIGSEGV)`, which
+ // really shouldn't happen.
+
State.FailureType = E;
State.FailureAddress = Address;
- // Raise a SEGV by touching first guard page.
- volatile char *p = reinterpret_cast<char *>(State.GuardedPagePool);
+ // Raise a SEGV by touching a specific address that identifies to the crash
+ // handler that this is an internally-raised fault. Changing this address?
+ // Don't forget to update __gwp_asan_get_internal_crash_address.
+ volatile char *p =
+ reinterpret_cast<char *>(State.internallyDetectedErrorFaultAddress());
*p = 0;
- // Normally, would be __builtin_unreachable(), but because of
- // https://bugs.llvm.org/show_bug.cgi?id=47480, unreachable will DCE the
- // volatile store above, even though it has side effects.
- __builtin_trap();
-}
-void GuardedPoolAllocator::stop() {
- getThreadLocals()->RecursiveGuard = true;
- PoolMutex.tryLock();
+ // This should never be reached in non-recoverable mode. Ensure that the
+ // signal handler called handleRecoverablePostCrashReport(), which was
+ // responsible for re-setting these fields.
+ assert(State.FailureType == Error::UNKNOWN);
+ assert(State.FailureAddress == 0u);
+
+ // In recoverable mode, the signal handler (after dumping the crash) marked
+ // the page containing the InternalFaultSegvAddress as read/writeable, to
+ // allow the second touch to succeed after returning from the signal handler.
+ // Now, we need to mark the page as non-read/write-able again, so future
+ // internal faults can be raised.
+ deallocateInGuardedPool(
+ reinterpret_cast<void *>(getPageAddr(
+ State.internallyDetectedErrorFaultAddress(), State.PageSize)),
+ State.PageSize);
+
+ // And now we're done with patching ourselves back up, enable the allocator.
+ enable();
}
void GuardedPoolAllocator::deallocate(void *Ptr) {
size_t Slot = State.getNearestSlot(UPtr);
uintptr_t SlotStart = State.slotToAddr(Slot);
AllocationMetadata *Meta = addrToMetadata(UPtr);
+
+ // If this allocation is responsible for crash, never recycle it. Turn the
+ // deallocate() call into a no-op.
+ if (Meta->HasCrashed)
+ return;
+
if (Meta->Addr != UPtr) {
- // If multiple errors occur at the same time, use the first one.
- ScopedLock L(PoolMutex);
- trapOnAddress(UPtr, Error::INVALID_FREE);
+ raiseInternallyDetectedError(UPtr, Error::INVALID_FREE);
+ return;
+ }
+ if (Meta->IsDeallocated) {
+ raiseInternallyDetectedError(UPtr, Error::DOUBLE_FREE);
+ return;
}
// Intentionally scope the mutex here, so that other threads can access the
// pool during the expensive markInaccessible() call.
{
ScopedLock L(PoolMutex);
- if (Meta->IsDeallocated) {
- trapOnAddress(UPtr, Error::DOUBLE_FREE);
- }
// Ensure that the deallocation is recorded before marking the page as
// inaccessible. Otherwise, a racy use-after-free will have inconsistent
freeSlot(Slot);
}
+// Thread-compatible, protected by PoolMutex.
+static bool PreviousRecursiveGuard;
+
+void GuardedPoolAllocator::preCrashReport(void *Ptr) {
+ assert(pointerIsMine(Ptr) && "Pointer is not mine!");
+ uintptr_t InternalCrashAddr = __gwp_asan_get_internal_crash_address(
+ &State, reinterpret_cast<uintptr_t>(Ptr));
+ if (!InternalCrashAddr)
+ disable();
+
+ // If something in the signal handler calls malloc() while dumping the
+ // GWP-ASan report (e.g. backtrace_symbols()), make sure that GWP-ASan doesn't
+ // service that allocation. `PreviousRecursiveGuard` is protected by the
+ // allocator locks taken in disable(), either explicitly above for
+ // externally-raised errors, or implicitly in raiseInternallyDetectedError()
+ // for internally-detected errors.
+ PreviousRecursiveGuard = getThreadLocals()->RecursiveGuard;
+ getThreadLocals()->RecursiveGuard = true;
+}
+
+void GuardedPoolAllocator::postCrashReportRecoverableOnly(void *SignalPtr) {
+ uintptr_t SignalUPtr = reinterpret_cast<uintptr_t>(SignalPtr);
+ uintptr_t InternalCrashAddr =
+ __gwp_asan_get_internal_crash_address(&State, SignalUPtr);
+ uintptr_t ErrorUptr = InternalCrashAddr ?: SignalUPtr;
+
+ AllocationMetadata *Metadata = addrToMetadata(ErrorUptr);
+ Metadata->HasCrashed = true;
+
+ allocateInGuardedPool(
+ reinterpret_cast<void *>(getPageAddr(SignalUPtr, State.PageSize)),
+ State.PageSize);
+
+ // Clear the internal state in order to not confuse the crash handler if a
+ // use-after-free or buffer-overflow comes from a different allocation in the
+ // future.
+ if (InternalCrashAddr) {
+ State.FailureType = Error::UNKNOWN;
+ State.FailureAddress = 0;
+ }
+
+ size_t Slot = State.getNearestSlot(ErrorUptr);
+ // If the slot is available, remove it permanently.
+ for (size_t i = 0; i < FreeSlotsLength; ++i) {
+ if (FreeSlots[i] == Slot) {
+ FreeSlots[i] = FreeSlots[FreeSlotsLength - 1];
+ FreeSlotsLength -= 1;
+ break;
+ }
+ }
+
+ getThreadLocals()->RecursiveGuard = PreviousRecursiveGuard;
+ if (!InternalCrashAddr)
+ enable();
+}
+
size_t GuardedPoolAllocator::getSize(const void *Ptr) {
assert(pointerIsMine(Ptr));
ScopedLock L(PoolMutex);
// allocate.
void iterate(void *Base, size_t Size, iterate_callback Cb, void *Arg);
- // This function is used to signal the allocator to indefinitely stop
- // functioning, as a crash has occurred. This stops the allocator from
- // servicing any further allocations permanently.
- void stop();
-
// Return whether the allocation should be randomly chosen for sampling.
GWP_ASAN_ALWAYS_INLINE bool shouldSample() {
// NextSampleCounter == 0 means we "should regenerate the counter".
// Returns a pointer to the AllocatorState region.
const AllocatorState *getAllocatorState() const { return &State; }
+ // Functions that the signal handler is responsible for calling, while
+ // providing the SEGV pointer, prior to dumping the crash, and after dumping
+ // the crash (in recoverable mode only).
+ void preCrashReport(void *Ptr);
+ void postCrashReportRecoverableOnly(void *Ptr);
+
// Exposed as protected for testing.
protected:
// Returns the actual allocation size required to service an allocation with
// Raise a SEGV and set the corresponding fields in the Allocator's State in
// order to tell the crash handler what happened. Used when errors are
// detected internally (Double Free, Invalid Free).
- void trapOnAddress(uintptr_t Address, Error E);
+ void raiseInternallyDetectedError(uintptr_t Address, Error E);
static GuardedPoolAllocator *getSingleton();
return;
}
- StackTrace.Print();
+ __sanitizer::InternalScopedString buffer;
+ StackTrace.PrintTo(&buffer);
+ Printf("%s\n", buffer.data());
}
} // anonymous namespace
// before this function.
void installSignalHandlers(gwp_asan::GuardedPoolAllocator *GPA, Printf_t Printf,
gwp_asan::backtrace::PrintBacktrace_t PrintBacktrace,
- gwp_asan::backtrace::SegvBacktrace_t SegvBacktrace);
+ gwp_asan::backtrace::SegvBacktrace_t SegvBacktrace,
+ bool Recoverable = false);
// Uninistall the signal handlers, test-only.
void uninstallSignalHandlers();
void installSignalHandlers(gwp_asan::GuardedPoolAllocator * /* GPA */,
Printf_t /* Printf */,
backtrace::PrintBacktrace_t /* PrintBacktrace */,
- backtrace::SegvBacktrace_t /* SegvBacktrace */) {}
+ backtrace::SegvBacktrace_t /* SegvBacktrace */,
+ bool /* Recoverable */) {}
void uninstallSignalHandlers() {}
} // namespace segv_handler
// appended to a log file automatically per Printf() call.
constexpr size_t kDescriptionBufferLen = 128;
char DescriptionBuffer[kDescriptionBufferLen] = "";
+
+ bool AccessWasInBounds = false;
if (E != Error::UNKNOWN && Metadata != nullptr) {
uintptr_t Address = __gwp_asan_get_allocation_address(Metadata);
size_t Size = __gwp_asan_get_allocation_size(Metadata);
- if (E == Error::USE_AFTER_FREE) {
- snprintf(DescriptionBuffer, kDescriptionBufferLen,
- "(%zu byte%s into a %zu-byte allocation at 0x%zx) ",
- AccessPtr - Address, (AccessPtr - Address == 1) ? "" : "s", Size,
- Address);
- } else if (AccessPtr < Address) {
+ if (AccessPtr < Address) {
snprintf(DescriptionBuffer, kDescriptionBufferLen,
"(%zu byte%s to the left of a %zu-byte allocation at 0x%zx) ",
Address - AccessPtr, (Address - AccessPtr == 1) ? "" : "s", Size,
"(%zu byte%s to the right of a %zu-byte allocation at 0x%zx) ",
AccessPtr - Address, (AccessPtr - Address == 1) ? "" : "s", Size,
Address);
- } else {
+ } else if (E == Error::DOUBLE_FREE) {
snprintf(DescriptionBuffer, kDescriptionBufferLen,
"(a %zu-byte allocation) ", Size);
+ } else {
+ AccessWasInBounds = true;
+ snprintf(DescriptionBuffer, kDescriptionBufferLen,
+ "(%zu byte%s into a %zu-byte allocation at 0x%zx) ",
+ AccessPtr - Address, (AccessPtr - Address == 1) ? "" : "s", Size,
+ Address);
}
}
else
snprintf(ThreadBuffer, kThreadBufferLen, "%" PRIu64, ThreadID);
- Printf("%s at 0x%zx %sby thread %s here:\n", gwp_asan::ErrorToString(E),
- AccessPtr, DescriptionBuffer, ThreadBuffer);
+ const char *OutOfBoundsAndUseAfterFreeWarning = "";
+ if (E == Error::USE_AFTER_FREE && !AccessWasInBounds) {
+ OutOfBoundsAndUseAfterFreeWarning =
+ " (warning: buffer overflow/underflow detected on a free()'d "
+ "allocation. This either means you have a buffer-overflow and a "
+ "use-after-free at the same time, or you have a long-lived "
+ "use-after-free bug where the allocation/deallocation metadata below "
+ "has already been overwritten and is likely bogus)";
+ }
+
+ Printf("%s%s at 0x%zx %sby thread %s here:\n", gwp_asan::ErrorToString(E),
+ OutOfBoundsAndUseAfterFreeWarning, AccessPtr, DescriptionBuffer,
+ ThreadBuffer);
}
void dumpReport(uintptr_t ErrorPtr, const gwp_asan::AllocatorState *State,
assert(State && "dumpReport missing Allocator State.");
assert(Metadata && "dumpReport missing Metadata.");
assert(Printf && "dumpReport missing Printf.");
+ assert(__gwp_asan_error_is_mine(State, ErrorPtr) &&
+ "dumpReport() called on a non-GWP-ASan error.");
- if (!__gwp_asan_error_is_mine(State, ErrorPtr))
+ uintptr_t InternalErrorPtr =
+ __gwp_asan_get_internal_crash_address(State, ErrorPtr);
+ if (InternalErrorPtr)
+ ErrorPtr = InternalErrorPtr;
+
+ const gwp_asan::AllocationMetadata *AllocMeta =
+ __gwp_asan_get_metadata(State, Metadata, ErrorPtr);
+
+ // It's unusual for a signal handler to be invoked multiple times for the same
+ // allocation, but it's possible in various scenarios, like:
+ // 1. A double-free or invalid-free was invoked in one thread at the same
+ // time as a buffer-overflow or use-after-free in another thread, or
+ // 2. Two threads do a use-after-free or buffer-overflow at the same time.
+ // In these instances, we've already dumped a report for this allocation, so
+ // skip dumping this issue as well.
+ if (AllocMeta->HasCrashed)
return;
Printf("*** GWP-ASan detected a memory error ***\n");
ScopedEndOfReportDecorator Decorator(Printf);
- uintptr_t InternalErrorPtr = __gwp_asan_get_internal_crash_address(State);
- if (InternalErrorPtr != 0u)
- ErrorPtr = InternalErrorPtr;
-
Error E = __gwp_asan_diagnose_error(State, Metadata, ErrorPtr);
-
if (E == Error::UNKNOWN) {
Printf("GWP-ASan cannot provide any more information about this error. "
"This may occur due to a wild memory access into the GWP-ASan pool, "
return;
}
- const gwp_asan::AllocationMetadata *AllocMeta =
- __gwp_asan_get_metadata(State, Metadata, ErrorPtr);
-
// Print the error header.
printHeader(E, ErrorPtr, AllocMeta, Printf);
struct sigaction PreviousHandler;
bool SignalHandlerInstalled;
+bool RecoverableSignal;
gwp_asan::GuardedPoolAllocator *GPAForSignalHandler;
Printf_t PrintfForSignalHandler;
PrintBacktrace_t PrintBacktraceForSignalHandler;
SegvBacktrace_t BacktraceForSignalHandler;
static void sigSegvHandler(int sig, siginfo_t *info, void *ucontext) {
- if (GPAForSignalHandler) {
- GPAForSignalHandler->stop();
+ const gwp_asan::AllocatorState *State =
+ GPAForSignalHandler->getAllocatorState();
+ void *FaultAddr = info->si_addr;
+ uintptr_t FaultAddrUPtr = reinterpret_cast<uintptr_t>(FaultAddr);
- dumpReport(reinterpret_cast<uintptr_t>(info->si_addr),
- GPAForSignalHandler->getAllocatorState(),
- GPAForSignalHandler->getMetadataRegion(),
+ if (__gwp_asan_error_is_mine(State, FaultAddrUPtr)) {
+ GPAForSignalHandler->preCrashReport(FaultAddr);
+
+ dumpReport(FaultAddrUPtr, State, GPAForSignalHandler->getMetadataRegion(),
BacktraceForSignalHandler, PrintfForSignalHandler,
PrintBacktraceForSignalHandler, ucontext);
+
+ if (RecoverableSignal) {
+ GPAForSignalHandler->postCrashReportRecoverableOnly(FaultAddr);
+ return;
+ }
}
- // Process any previous handlers.
+ // Process any previous handlers as long as the crash wasn't a GWP-ASan crash
+ // in recoverable mode.
if (PreviousHandler.sa_flags & SA_SIGINFO) {
PreviousHandler.sa_sigaction(sig, info, ucontext);
} else if (PreviousHandler.sa_handler == SIG_DFL) {
void installSignalHandlers(gwp_asan::GuardedPoolAllocator *GPA, Printf_t Printf,
PrintBacktrace_t PrintBacktrace,
- SegvBacktrace_t SegvBacktrace) {
+ SegvBacktrace_t SegvBacktrace, bool Recoverable) {
assert(GPA && "GPA wasn't provided to installSignalHandlers.");
assert(Printf && "Printf wasn't provided to installSignalHandlers.");
assert(PrintBacktrace &&
PrintfForSignalHandler = Printf;
PrintBacktraceForSignalHandler = PrintBacktrace;
BacktraceForSignalHandler = SegvBacktrace;
+ RecoverableSignal = Recoverable;
struct sigaction Action = {};
Action.sa_sigaction = sigSegvHandler;
"the same. Note, if the previously installed SIGSEGV handler is SIG_IGN, "
"we terminate the process after dumping the error report.")
+GWP_ASAN_OPTION(
+ bool, Recoverable, false,
+ "Install GWP-ASan's signal handler in recoverable mode. This means that "
+ "upon GWP-ASan detecting an error, it'll print the error report, but *not* "
+ "crash. Only one crash per sampled allocation will ever be recorded, and "
+ "if a sampled allocation does actually cause a crash, it'll permanently "
+ "occupy a slot in the pool. The recoverable mode also means that "
+ "previously-installed signal handlers will only be triggered for "
+ "non-GWP-ASan errors, as all GWP-ASan errors won't be forwarded.")
+
GWP_ASAN_OPTION(bool, InstallForkHandlers, true,
"Install GWP-ASan atfork handlers to acquire internal locks "
"before fork and release them after.")
}
void GuardedPoolAllocator::installAtFork() {
+ static bool AtForkInstalled = false;
+ if (AtForkInstalled)
+ return;
+ AtForkInstalled = true;
auto Disable = []() {
if (auto *S = getSingleton())
S->disable();
-#!/bin/bash
+#!/usr/bin/env bash
# The lines that we're looking to symbolize look like this:
#0 ./a.out(_foo+0x3e6) [0x55a52e64c696]
set(GWP_ASAN_UNITTEST_CFLAGS
${COMPILER_RT_UNITTEST_CFLAGS}
${COMPILER_RT_GTEST_CFLAGS}
+ ${SANITIZER_TEST_CXX_CFLAGS}
+ -std=c++17
-I${COMPILER_RT_SOURCE_DIR}/lib/
-O2
-g
harness.cpp
enable_disable.cpp
late_init.cpp
- options.cpp)
+ options.cpp
+ recoverable.cpp)
set(GWP_ASAN_UNIT_TEST_HEADERS
${GWP_ASAN_HEADERS}
add_custom_target(GwpAsanUnitTests)
set_target_properties(GwpAsanUnitTests PROPERTIES FOLDER "Compiler-RT Tests")
-set(GWP_ASAN_UNITTEST_LINK_FLAGS ${COMPILER_RT_UNITTEST_LINK_FLAGS} -ldl)
+set(GWP_ASAN_UNITTEST_LINK_FLAGS
+ ${COMPILER_RT_UNITTEST_LINK_FLAGS} -ldl
+ ${COMPILER_RT_UNWINDER_LINK_LIBS}
+ ${SANITIZER_TEST_CXX_LIBRARIES})
list(APPEND GWP_ASAN_UNITTEST_LINK_FLAGS --driver-mode=g++)
if(NOT WIN32)
list(APPEND GWP_ASAN_UNITTEST_LINK_FLAGS -pthread)
GwpAsanUnitTests "GwpAsan-${arch}-Test" ${arch}
SOURCES ${GWP_ASAN_UNITTESTS} ${COMPILER_RT_GTEST_SOURCE}
RUNTIME ${GWP_ASAN_TEST_RUNTIME}
- DEPS gtest ${GWP_ASAN_UNIT_TEST_HEADERS}
+ DEPS llvm_gtest ${GWP_ASAN_UNIT_TEST_HEADERS}
CFLAGS ${GWP_ASAN_UNITTEST_CFLAGS}
LINK_FLAGS ${GWP_ASAN_UNITTEST_LINK_FLAGS})
set_target_properties(GwpAsanUnitTests PROPERTIES
// numerics of the testing.
TEST(AlignmentTest, LeftAlignedAllocs) {
// Alignment < Page Size.
- EXPECT_EQ(0x4000, AlignmentTestGPA::alignUp(
- /* Ptr */ 0x4000, /* Alignment */ 0x1));
+ EXPECT_EQ(0x4000u, AlignmentTestGPA::alignUp(
+ /* Ptr */ 0x4000, /* Alignment */ 0x1));
// Alignment == Page Size.
- EXPECT_EQ(0x4000, AlignmentTestGPA::alignUp(
- /* Ptr */ 0x4000, /* Alignment */ 0x1000));
+ EXPECT_EQ(0x4000u, AlignmentTestGPA::alignUp(
+ /* Ptr */ 0x4000, /* Alignment */ 0x1000));
// Alignment > Page Size.
- EXPECT_EQ(0x4000, AlignmentTestGPA::alignUp(
- /* Ptr */ 0x4000, /* Alignment */ 0x4000));
+ EXPECT_EQ(0x4000u, AlignmentTestGPA::alignUp(
+ /* Ptr */ 0x4000, /* Alignment */ 0x4000));
}
TEST(AlignmentTest, SingleByteAllocs) {
// Alignment < Page Size.
- EXPECT_EQ(0x1,
+ EXPECT_EQ(0x1u,
AlignmentTestGPA::getRequiredBackingSize(
/* Size */ 0x1, /* Alignment */ 0x1, /* PageSize */ 0x1000));
- EXPECT_EQ(0x7fff, AlignmentTestGPA::alignDown(
- /* Ptr */ 0x8000 - 0x1, /* Alignment */ 0x1));
+ EXPECT_EQ(0x7fffu, AlignmentTestGPA::alignDown(
+ /* Ptr */ 0x8000 - 0x1, /* Alignment */ 0x1));
// Alignment == Page Size.
- EXPECT_EQ(0x1,
+ EXPECT_EQ(0x1u,
AlignmentTestGPA::getRequiredBackingSize(
/* Size */ 0x1, /* Alignment */ 0x1000, /* PageSize */ 0x1000));
- EXPECT_EQ(0x7000, AlignmentTestGPA::alignDown(
- /* Ptr */ 0x8000 - 0x1, /* Alignment */ 0x1000));
+ EXPECT_EQ(0x7000u, AlignmentTestGPA::alignDown(
+ /* Ptr */ 0x8000 - 0x1, /* Alignment */ 0x1000));
// Alignment > Page Size.
- EXPECT_EQ(0x3001,
+ EXPECT_EQ(0x3001u,
AlignmentTestGPA::getRequiredBackingSize(
/* Size */ 0x1, /* Alignment */ 0x4000, /* PageSize */ 0x1000));
- EXPECT_EQ(0x4000, AlignmentTestGPA::alignDown(
- /* Ptr */ 0x8000 - 0x1, /* Alignment */ 0x4000));
+ EXPECT_EQ(0x4000u, AlignmentTestGPA::alignDown(
+ /* Ptr */ 0x8000 - 0x1, /* Alignment */ 0x4000));
}
TEST(AlignmentTest, PageSizedAllocs) {
// Alignment < Page Size.
- EXPECT_EQ(0x1000,
+ EXPECT_EQ(0x1000u,
AlignmentTestGPA::getRequiredBackingSize(
/* Size */ 0x1000, /* Alignment */ 0x1, /* PageSize */ 0x1000));
- EXPECT_EQ(0x7000, AlignmentTestGPA::alignDown(
- /* Ptr */ 0x8000 - 0x1000, /* Alignment */ 0x1));
+ EXPECT_EQ(0x7000u, AlignmentTestGPA::alignDown(
+ /* Ptr */ 0x8000 - 0x1000, /* Alignment */ 0x1));
// Alignment == Page Size.
- EXPECT_EQ(0x1000, AlignmentTestGPA::getRequiredBackingSize(
- /* Size */ 0x1000, /* Alignment */ 0x1000,
- /* PageSize */ 0x1000));
- EXPECT_EQ(0x7000, AlignmentTestGPA::alignDown(
- /* Ptr */ 0x8000 - 0x1000, /* Alignment */ 0x1000));
+ EXPECT_EQ(0x1000u, AlignmentTestGPA::getRequiredBackingSize(
+ /* Size */ 0x1000, /* Alignment */ 0x1000,
+ /* PageSize */ 0x1000));
+ EXPECT_EQ(0x7000u, AlignmentTestGPA::alignDown(
+ /* Ptr */ 0x8000 - 0x1000, /* Alignment */ 0x1000));
// Alignment > Page Size.
- EXPECT_EQ(0x4000, AlignmentTestGPA::getRequiredBackingSize(
- /* Size */ 0x1000, /* Alignment */ 0x4000,
- /* PageSize */ 0x1000));
- EXPECT_EQ(0x4000, AlignmentTestGPA::alignDown(
- /* Ptr */ 0x8000 - 0x1000, /* Alignment */ 0x4000));
+ EXPECT_EQ(0x4000u, AlignmentTestGPA::getRequiredBackingSize(
+ /* Size */ 0x1000, /* Alignment */ 0x4000,
+ /* PageSize */ 0x1000));
+ EXPECT_EQ(0x4000u, AlignmentTestGPA::alignDown(
+ /* Ptr */ 0x8000 - 0x1000, /* Alignment */ 0x4000));
}
TEST(AlignmentTest, MoreThanPageAllocs) {
// Alignment < Page Size.
- EXPECT_EQ(0x2fff,
+ EXPECT_EQ(0x2fffu,
AlignmentTestGPA::getRequiredBackingSize(
/* Size */ 0x2fff, /* Alignment */ 0x1, /* PageSize */ 0x1000));
- EXPECT_EQ(0x5001, AlignmentTestGPA::alignDown(
- /* Ptr */ 0x8000 - 0x2fff, /* Alignment */ 0x1));
+ EXPECT_EQ(0x5001u, AlignmentTestGPA::alignDown(
+ /* Ptr */ 0x8000 - 0x2fff, /* Alignment */ 0x1));
// Alignment == Page Size.
- EXPECT_EQ(0x2fff, AlignmentTestGPA::getRequiredBackingSize(
- /* Size */ 0x2fff, /* Alignment */ 0x1000,
- /* PageSize */ 0x1000));
- EXPECT_EQ(0x5000, AlignmentTestGPA::alignDown(
- /* Ptr */ 0x8000 - 0x2fff, /* Alignment */ 0x1000));
+ EXPECT_EQ(0x2fffu, AlignmentTestGPA::getRequiredBackingSize(
+ /* Size */ 0x2fff, /* Alignment */ 0x1000,
+ /* PageSize */ 0x1000));
+ EXPECT_EQ(0x5000u, AlignmentTestGPA::alignDown(
+ /* Ptr */ 0x8000 - 0x2fff, /* Alignment */ 0x1000));
// Alignment > Page Size.
- EXPECT_EQ(0x5fff, AlignmentTestGPA::getRequiredBackingSize(
- /* Size */ 0x2fff, /* Alignment */ 0x4000,
- /* PageSize */ 0x1000));
- EXPECT_EQ(0x4000, AlignmentTestGPA::alignDown(
- /* Ptr */ 0x8000 - 0x2fff, /* Alignment */ 0x4000));
+ EXPECT_EQ(0x5fffu, AlignmentTestGPA::getRequiredBackingSize(
+ /* Size */ 0x2fff, /* Alignment */ 0x4000,
+ /* PageSize */ 0x1000));
+ EXPECT_EQ(0x4000u, AlignmentTestGPA::alignDown(
+ /* Ptr */ 0x8000 - 0x2fff, /* Alignment */ 0x4000));
}
//
//===----------------------------------------------------------------------===//
+#include <regex>
#include <string>
#include "gwp_asan/common.h"
#include "gwp_asan/crash_handler.h"
#include "gwp_asan/tests/harness.h"
-// Optnone to ensure that the calls to these functions are not optimized away,
-// as we're looking for them in the backtraces.
-__attribute((optnone)) void *
-AllocateMemory(gwp_asan::GuardedPoolAllocator &GPA) {
- return GPA.allocate(1);
-}
-__attribute((optnone)) void
-DeallocateMemory(gwp_asan::GuardedPoolAllocator &GPA, void *Ptr) {
- GPA.deallocate(Ptr);
-}
-__attribute((optnone)) void
-DeallocateMemory2(gwp_asan::GuardedPoolAllocator &GPA, void *Ptr) {
- GPA.deallocate(Ptr);
-}
-__attribute__((optnone)) void TouchMemory(void *Ptr) {
- *(reinterpret_cast<volatile char *>(Ptr)) = 7;
-}
-
-TEST_F(BacktraceGuardedPoolAllocatorDeathTest, DoubleFree) {
+TEST_P(BacktraceGuardedPoolAllocatorDeathTest, DoubleFree) {
void *Ptr = AllocateMemory(GPA);
DeallocateMemory(GPA, Ptr);
- std::string DeathRegex = "Double Free.*";
- DeathRegex.append("DeallocateMemory2.*");
-
- DeathRegex.append("was deallocated.*");
- DeathRegex.append("DeallocateMemory.*");
-
- DeathRegex.append("was allocated.*");
- DeathRegex.append("AllocateMemory.*");
- ASSERT_DEATH(DeallocateMemory2(GPA, Ptr), DeathRegex);
+ std::string DeathRegex = "Double Free.*DeallocateMemory2.*";
+ DeathRegex.append("was deallocated.*DeallocateMemory[^2].*");
+ DeathRegex.append("was allocated.*AllocateMemory");
+ if (!Recoverable) {
+ ASSERT_DEATH(DeallocateMemory2(GPA, Ptr), DeathRegex);
+ return;
+ }
+
+ // For recoverable, assert that DeallocateMemory2() doesn't crash.
+ DeallocateMemory2(GPA, Ptr);
+ // Fuchsia's zxtest doesn't have an EXPECT_THAT(testing::MatchesRegex(), ...),
+ // so check the regex manually.
+ EXPECT_TRUE(std::regex_search(
+ GetOutputBuffer(),
+ std::basic_regex(DeathRegex, std::regex_constants::extended)))
+ << "Regex \"" << DeathRegex
+ << "\" was not found in input:\n============\n"
+ << GetOutputBuffer() << "\n============";
}
-TEST_F(BacktraceGuardedPoolAllocatorDeathTest, UseAfterFree) {
+TEST_P(BacktraceGuardedPoolAllocatorDeathTest, UseAfterFree) {
#if defined(__linux__) && __ARM_ARCH == 7
// Incomplete backtrace on Armv7 Linux
GTEST_SKIP();
void *Ptr = AllocateMemory(GPA);
DeallocateMemory(GPA, Ptr);
- std::string DeathRegex = "Use After Free.*";
- DeathRegex.append("TouchMemory.*");
-
- DeathRegex.append("was deallocated.*");
- DeathRegex.append("DeallocateMemory.*");
-
- DeathRegex.append("was allocated.*");
- DeathRegex.append("AllocateMemory.*");
- ASSERT_DEATH(TouchMemory(Ptr), DeathRegex);
+ std::string DeathRegex = "Use After Free.*TouchMemory.*";
+ DeathRegex.append("was deallocated.*DeallocateMemory[^2].*");
+ DeathRegex.append("was allocated.*AllocateMemory");
+
+ if (!Recoverable) {
+ ASSERT_DEATH(TouchMemory(Ptr), DeathRegex);
+ return;
+ }
+
+ // For recoverable, assert that TouchMemory() doesn't crash.
+ TouchMemory(Ptr);
+ // Fuchsia's zxtest doesn't have an EXPECT_THAT(testing::MatchesRegex(), ...),
+ // so check the regex manually.
+ EXPECT_TRUE(std::regex_search(
+ GetOutputBuffer(),
+ std::basic_regex(DeathRegex, std::regex_constants::extended)))
+ << "Regex \"" << DeathRegex
+ << "\" was not found in input:\n============\n"
+ << GetOutputBuffer() << "\n============";
+ ;
}
+INSTANTIATE_TEST_SUITE_P(RecoverableSignalDeathTest,
+ BacktraceGuardedPoolAllocatorDeathTest,
+ /* Recoverable */ testing::Bool());
+
TEST(Backtrace, Short) {
gwp_asan::AllocationMetadata Meta;
Meta.AllocationTrace.RecordBacktrace(
void setupState() {
State.GuardedPagePool = 0x2000;
- State.GuardedPagePoolEnd = 0xb000;
+ State.GuardedPagePoolEnd = 0xc000;
+ InternalFaultAddr = State.GuardedPagePoolEnd - 0x10;
State.MaxSimultaneousAllocations = 4; // 0x3000, 0x5000, 0x7000, 0x9000.
State.PageSize = 0x1000;
}
static uintptr_t BacktraceConstants[kNumBacktraceConstants];
AllocatorState State = {};
AllocationMetadata Metadata[4] = {};
+ uintptr_t InternalFaultAddr;
};
uintptr_t CrashHandlerAPITest::BacktraceConstants[kNumBacktraceConstants] = {
EXPECT_TRUE(__gwp_asan_error_is_mine(&State, FailureAddress));
EXPECT_EQ(Error::UNKNOWN,
__gwp_asan_diagnose_error(&State, Metadata, FailureAddress));
- EXPECT_EQ(0u, __gwp_asan_get_internal_crash_address(&State));
+ EXPECT_EQ(0u, __gwp_asan_get_internal_crash_address(&State, FailureAddress));
EXPECT_EQ(nullptr, __gwp_asan_get_metadata(&State, Metadata, FailureAddress));
}
EXPECT_TRUE(__gwp_asan_error_is_mine(&State));
EXPECT_EQ(Error::DOUBLE_FREE,
__gwp_asan_diagnose_error(&State, Metadata, 0x0));
- EXPECT_EQ(FailureAddress, __gwp_asan_get_internal_crash_address(&State));
+ EXPECT_EQ(FailureAddress,
+ __gwp_asan_get_internal_crash_address(&State, InternalFaultAddr));
checkMetadata(Index, FailureAddress);
}
EXPECT_TRUE(__gwp_asan_error_is_mine(&State));
EXPECT_EQ(Error::INVALID_FREE,
__gwp_asan_diagnose_error(&State, Metadata, 0x0));
- EXPECT_EQ(FailureAddress, __gwp_asan_get_internal_crash_address(&State));
+ EXPECT_EQ(FailureAddress,
+ __gwp_asan_get_internal_crash_address(&State, InternalFaultAddr));
checkMetadata(Index, FailureAddress);
}
EXPECT_TRUE(__gwp_asan_error_is_mine(&State));
EXPECT_EQ(Error::INVALID_FREE,
__gwp_asan_diagnose_error(&State, Metadata, 0x0));
- EXPECT_EQ(FailureAddress, __gwp_asan_get_internal_crash_address(&State));
+ EXPECT_EQ(FailureAddress,
+ __gwp_asan_get_internal_crash_address(&State, InternalFaultAddr));
EXPECT_EQ(nullptr, __gwp_asan_get_metadata(&State, Metadata, FailureAddress));
}
EXPECT_TRUE(__gwp_asan_error_is_mine(&State, FailureAddress));
EXPECT_EQ(Error::USE_AFTER_FREE,
__gwp_asan_diagnose_error(&State, Metadata, FailureAddress));
- EXPECT_EQ(0u, __gwp_asan_get_internal_crash_address(&State));
+ EXPECT_EQ(0u, __gwp_asan_get_internal_crash_address(&State, FailureAddress));
checkMetadata(Index, FailureAddress);
}
EXPECT_TRUE(__gwp_asan_error_is_mine(&State, FailureAddress));
EXPECT_EQ(Error::BUFFER_OVERFLOW,
__gwp_asan_diagnose_error(&State, Metadata, FailureAddress));
- EXPECT_EQ(0u, __gwp_asan_get_internal_crash_address(&State));
+ EXPECT_EQ(0u, __gwp_asan_get_internal_crash_address(&State, FailureAddress));
checkMetadata(Index, FailureAddress);
}
EXPECT_TRUE(__gwp_asan_error_is_mine(&State, FailureAddress));
EXPECT_EQ(Error::BUFFER_UNDERFLOW,
__gwp_asan_diagnose_error(&State, Metadata, FailureAddress));
- EXPECT_EQ(0u, __gwp_asan_get_internal_crash_address(&State));
+ EXPECT_EQ(0u, __gwp_asan_get_internal_crash_address(&State, FailureAddress));
checkMetadata(Index, FailureAddress);
}
}
} // namespace test
} // namespace gwp_asan
+
+// Optnone to ensure that the calls to these functions are not optimized away,
+// as we're looking for them in the backtraces.
+__attribute__((optnone)) char *
+AllocateMemory(gwp_asan::GuardedPoolAllocator &GPA) {
+ return static_cast<char *>(GPA.allocate(1));
+}
+__attribute__((optnone)) void
+DeallocateMemory(gwp_asan::GuardedPoolAllocator &GPA, void *Ptr) {
+ GPA.deallocate(Ptr);
+}
+__attribute__((optnone)) void
+DeallocateMemory2(gwp_asan::GuardedPoolAllocator &GPA, void *Ptr) {
+ GPA.deallocate(Ptr);
+}
+__attribute__((optnone)) void TouchMemory(void *Ptr) {
+ *(reinterpret_cast<volatile char *>(Ptr)) = 7;
+}
#if defined(__Fuchsia__)
#include <zxtest/zxtest.h>
using Test = ::zxtest::Test;
+template <typename T> using TestWithParam = ::zxtest::TestWithParam<T>;
#else
#include "gtest/gtest.h"
using Test = ::testing::Test;
+template <typename T> using TestWithParam = ::testing::TestWithParam<T>;
#endif
#include "gwp_asan/guarded_pool_allocator.h"
}; // namespace test
}; // namespace gwp_asan
+char *AllocateMemory(gwp_asan::GuardedPoolAllocator &GPA);
+void DeallocateMemory(gwp_asan::GuardedPoolAllocator &GPA, void *Ptr);
+void DeallocateMemory2(gwp_asan::GuardedPoolAllocator &GPA, void *Ptr);
+void TouchMemory(void *Ptr);
+
class DefaultGuardedPoolAllocator : public Test {
public:
void SetUp() override {
MaxSimultaneousAllocations;
};
-class BacktraceGuardedPoolAllocator : public Test {
+class BacktraceGuardedPoolAllocator
+ : public TestWithParam</* Recoverable */ bool> {
public:
void SetUp() override {
gwp_asan::options::Options Opts;
Opts.InstallForkHandlers = gwp_asan::test::OnlyOnce();
GPA.init(Opts);
+ // In recoverable mode, capture GWP-ASan logs to an internal buffer so that
+ // we can search it in unit tests. For non-recoverable tests, the default
+ // buffer is fine, as any tests should be EXPECT_DEATH()'d.
+ Recoverable = GetParam();
+ gwp_asan::Printf_t PrintfFunction = PrintfToBuffer;
+ GetOutputBuffer().clear();
+ if (!Recoverable)
+ PrintfFunction = gwp_asan::test::getPrintfFunction();
+
gwp_asan::segv_handler::installSignalHandlers(
- &GPA, gwp_asan::test::getPrintfFunction(),
- gwp_asan::backtrace::getPrintBacktraceFunction(),
- gwp_asan::backtrace::getSegvBacktraceFunction());
+ &GPA, PrintfFunction, gwp_asan::backtrace::getPrintBacktraceFunction(),
+ gwp_asan::backtrace::getSegvBacktraceFunction(),
+ /* Recoverable */ Recoverable);
}
void TearDown() override {
}
protected:
+ static std::string &GetOutputBuffer() {
+ static std::string Buffer;
+ return Buffer;
+ }
+
+ __attribute__((format(printf, 1, 2))) static void
+ PrintfToBuffer(const char *Format, ...) {
+ va_list AP;
+ va_start(AP, Format);
+ char Buffer[8192];
+ vsnprintf(Buffer, sizeof(Buffer), Format, AP);
+ GetOutputBuffer() += Buffer;
+ va_end(AP);
+ }
+
gwp_asan::GuardedPoolAllocator GPA;
+ bool Recoverable;
};
// https://github.com/google/googletest/blob/master/docs/advanced.md#death-tests-and-threads
#include "gwp_asan/tests/harness.h"
+#include <algorithm>
#include <set>
#include <vector>
--- /dev/null
+//===-- recoverable.cpp -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <atomic>
+#include <mutex>
+#include <regex>
+#include <string>
+#include <thread>
+#include <vector>
+
+#include "gwp_asan/common.h"
+#include "gwp_asan/crash_handler.h"
+#include "gwp_asan/tests/harness.h"
+
+void CheckOnlyOneGwpAsanCrash(const std::string &OutputBuffer) {
+ const char *kGwpAsanErrorString = "GWP-ASan detected a memory error";
+ size_t FirstIndex = OutputBuffer.find(kGwpAsanErrorString);
+ ASSERT_NE(FirstIndex, std::string::npos) << "Didn't detect a GWP-ASan crash";
+ ASSERT_EQ(OutputBuffer.find(kGwpAsanErrorString, FirstIndex + 1),
+ std::string::npos)
+ << "Detected more than one GWP-ASan crash:\n"
+ << OutputBuffer;
+}
+
+TEST_P(BacktraceGuardedPoolAllocator, MultipleDoubleFreeOnlyOneOutput) {
+ SCOPED_TRACE("");
+ void *Ptr = AllocateMemory(GPA);
+ DeallocateMemory(GPA, Ptr);
+ // First time should generate a crash report.
+ DeallocateMemory(GPA, Ptr);
+ CheckOnlyOneGwpAsanCrash(GetOutputBuffer());
+ ASSERT_NE(std::string::npos, GetOutputBuffer().find("Double Free"));
+
+ // Ensure the crash is only reported once.
+ GetOutputBuffer().clear();
+ for (size_t i = 0; i < 100; ++i) {
+ DeallocateMemory(GPA, Ptr);
+ ASSERT_TRUE(GetOutputBuffer().empty());
+ }
+}
+
+TEST_P(BacktraceGuardedPoolAllocator, MultipleInvalidFreeOnlyOneOutput) {
+ SCOPED_TRACE("");
+ char *Ptr = static_cast<char *>(AllocateMemory(GPA));
+ // First time should generate a crash report.
+ DeallocateMemory(GPA, Ptr + 1);
+ CheckOnlyOneGwpAsanCrash(GetOutputBuffer());
+ ASSERT_NE(std::string::npos, GetOutputBuffer().find("Invalid (Wild) Free"));
+
+ // Ensure the crash is only reported once.
+ GetOutputBuffer().clear();
+ for (size_t i = 0; i < 100; ++i) {
+ DeallocateMemory(GPA, Ptr + 1);
+ ASSERT_TRUE(GetOutputBuffer().empty());
+ }
+}
+
+TEST_P(BacktraceGuardedPoolAllocator, MultipleUseAfterFreeOnlyOneOutput) {
+ SCOPED_TRACE("");
+ void *Ptr = AllocateMemory(GPA);
+ DeallocateMemory(GPA, Ptr);
+ // First time should generate a crash report.
+ TouchMemory(Ptr);
+ ASSERT_NE(std::string::npos, GetOutputBuffer().find("Use After Free"));
+
+ // Ensure the crash is only reported once.
+ GetOutputBuffer().clear();
+ for (size_t i = 0; i < 100; ++i) {
+ TouchMemory(Ptr);
+ ASSERT_TRUE(GetOutputBuffer().empty());
+ }
+}
+
+TEST_P(BacktraceGuardedPoolAllocator, MultipleBufferOverflowOnlyOneOutput) {
+ SCOPED_TRACE("");
+ char *Ptr = static_cast<char *>(AllocateMemory(GPA));
+ // First time should generate a crash report.
+ TouchMemory(Ptr - 16);
+ TouchMemory(Ptr + 16);
+ CheckOnlyOneGwpAsanCrash(GetOutputBuffer());
+ if (GetOutputBuffer().find("Buffer Overflow") == std::string::npos &&
+ GetOutputBuffer().find("Buffer Underflow") == std::string::npos)
+ FAIL() << "Failed to detect buffer underflow/overflow:\n"
+ << GetOutputBuffer();
+
+ // Ensure the crash is only reported once.
+ GetOutputBuffer().clear();
+ for (size_t i = 0; i < 100; ++i) {
+ TouchMemory(Ptr - 16);
+ TouchMemory(Ptr + 16);
+ ASSERT_TRUE(GetOutputBuffer().empty()) << GetOutputBuffer();
+ }
+}
+
+TEST_P(BacktraceGuardedPoolAllocator, OneDoubleFreeOneUseAfterFree) {
+ SCOPED_TRACE("");
+ void *Ptr = AllocateMemory(GPA);
+ DeallocateMemory(GPA, Ptr);
+ // First time should generate a crash report.
+ DeallocateMemory(GPA, Ptr);
+ CheckOnlyOneGwpAsanCrash(GetOutputBuffer());
+ ASSERT_NE(std::string::npos, GetOutputBuffer().find("Double Free"));
+
+ // Ensure the crash is only reported once.
+ GetOutputBuffer().clear();
+ for (size_t i = 0; i < 100; ++i) {
+ DeallocateMemory(GPA, Ptr);
+ ASSERT_TRUE(GetOutputBuffer().empty());
+ }
+}
+
+// We use double-free to detect that each slot can generate as single error.
+// Use-after-free would also be acceptable, but buffer-overflow wouldn't be, as
+// the random left/right alignment means that one right-overflow can disable
+// page protections, and a subsequent left-overflow of a slot that's on the
+// right hand side may not trap.
+TEST_P(BacktraceGuardedPoolAllocator, OneErrorReportPerSlot) {
+ SCOPED_TRACE("");
+ std::vector<void *> Ptrs;
+ for (size_t i = 0; i < GPA.getAllocatorState()->MaxSimultaneousAllocations;
+ ++i) {
+ void *Ptr = AllocateMemory(GPA);
+ ASSERT_NE(Ptr, nullptr);
+ Ptrs.push_back(Ptr);
+ DeallocateMemory(GPA, Ptr);
+ DeallocateMemory(GPA, Ptr);
+ CheckOnlyOneGwpAsanCrash(GetOutputBuffer());
+ ASSERT_NE(std::string::npos, GetOutputBuffer().find("Double Free"));
+ // Ensure the crash from this slot is only reported once.
+ GetOutputBuffer().clear();
+ DeallocateMemory(GPA, Ptr);
+ ASSERT_TRUE(GetOutputBuffer().empty());
+ // Reset the buffer, as we're gonna move to the next allocation.
+ GetOutputBuffer().clear();
+ }
+
+ // All slots should have been used. No further errors should occur.
+ for (size_t i = 0; i < 100; ++i)
+ ASSERT_EQ(AllocateMemory(GPA), nullptr);
+ for (void *Ptr : Ptrs) {
+ DeallocateMemory(GPA, Ptr);
+ TouchMemory(Ptr);
+ }
+ ASSERT_TRUE(GetOutputBuffer().empty());
+}
+
+void singleAllocThrashTask(gwp_asan::GuardedPoolAllocator *GPA,
+ std::atomic<bool> *StartingGun,
+ unsigned NumIterations, unsigned Job, char *Ptr) {
+ while (!*StartingGun) {
+ // Wait for starting gun.
+ }
+
+ for (unsigned i = 0; i < NumIterations; ++i) {
+ switch (Job) {
+ case 0:
+ DeallocateMemory(*GPA, Ptr);
+ break;
+ case 1:
+ DeallocateMemory(*GPA, Ptr + 1);
+ break;
+ case 2:
+ TouchMemory(Ptr);
+ break;
+ case 3:
+ TouchMemory(Ptr - 16);
+ TouchMemory(Ptr + 16);
+ break;
+ default:
+ __builtin_trap();
+ }
+ }
+}
+
+void runInterThreadThrashingSingleAlloc(unsigned NumIterations,
+ gwp_asan::GuardedPoolAllocator *GPA) {
+ std::atomic<bool> StartingGun{false};
+ std::vector<std::thread> Threads;
+ constexpr unsigned kNumThreads = 4;
+ if (std::thread::hardware_concurrency() < kNumThreads) {
+ GTEST_SKIP() << "Not enough threads to run this test";
+ }
+
+ char *Ptr = static_cast<char *>(AllocateMemory(*GPA));
+
+ for (unsigned i = 0; i < kNumThreads; ++i) {
+ Threads.emplace_back(singleAllocThrashTask, GPA, &StartingGun,
+ NumIterations, i, Ptr);
+ }
+
+ StartingGun = true;
+
+ for (auto &T : Threads)
+ T.join();
+}
+
+TEST_P(BacktraceGuardedPoolAllocator, InterThreadThrashingSingleAlloc) {
+ SCOPED_TRACE("");
+ constexpr unsigned kNumIterations = 100000;
+ runInterThreadThrashingSingleAlloc(kNumIterations, &GPA);
+ CheckOnlyOneGwpAsanCrash(GetOutputBuffer());
+}
+
+INSTANTIATE_TEST_SUITE_P(RecoverableTests, BacktraceGuardedPoolAllocator,
+ /* Recoverable */ testing::Values(true));
hwasan_memintrinsics.cpp
hwasan_poisoning.cpp
hwasan_report.cpp
- hwasan_setjmp.S
+ hwasan_setjmp_aarch64.S
+ hwasan_setjmp_riscv64.S
+ hwasan_setjmp_x86_64.S
hwasan_tag_mismatch_aarch64.S
+ hwasan_tag_mismatch_riscv64.S
hwasan_thread.cpp
hwasan_thread_list.cpp
hwasan_type_test.cpp
hwasan_new_delete.cpp
)
+set(HWASAN_RTL_PREINIT_SOURCES
+ hwasan_preinit.cpp
+ )
+
set(HWASAN_RTL_HEADERS
hwasan.h
hwasan_allocator.h
# Prevent clang from generating libc calls.
append_list_if(COMPILER_RT_HAS_FFREESTANDING_FLAG -ffreestanding HWASAN_RTL_CFLAGS)
+# Too many existing bugs, needs cleanup.
+append_list_if(COMPILER_RT_HAS_WNO_FORMAT -Wno-format HWASAN_RTL_CFLAGS)
+
set(HWASAN_DYNAMIC_LINK_FLAGS ${SANITIZER_COMMON_LINK_FLAGS})
if(ANDROID)
-ftls-model=initial-exec HWASAN_DYNAMIC_CFLAGS)
append_list_if(MSVC /DEBUG HWASAN_DYNAMIC_LINK_FLAGS)
-set(HWASAN_DYNAMIC_LIBS ${SANITIZER_CXX_ABI_LIBRARIES} ${SANITIZER_COMMON_LINK_LIBS})
+set(HWASAN_DYNAMIC_LIBS
+ ${COMPILER_RT_UNWINDER_LINK_LIBS}
+ ${SANITIZER_CXX_ABI_LIBRARIES}
+ ${SANITIZER_COMMON_LINK_LIBS})
append_list_if(COMPILER_RT_HAS_LIBDL dl HWASAN_DYNAMIC_LIBS)
append_list_if(COMPILER_RT_HAS_LIBRT rt HWASAN_DYNAMIC_LIBS)
ADDITIONAL_HEADERS ${HWASAN_RTL_HEADERS}
CFLAGS ${HWASAN_DYNAMIC_CFLAGS}
DEFS ${HWASAN_DEFINITIONS})
+add_compiler_rt_object_libraries(RTHwasan_preinit
+ ARCHS ${HWASAN_SUPPORTED_ARCH}
+ SOURCES ${HWASAN_RTL_PREINIT_SOURCES}
+ ADDITIONAL_HEADERS ${HWASAN_RTL_HEADERS}
+ CFLAGS ${HWASAN_RTL_CFLAGS}
+ DEFS ${HWASAN_DEFINITIONS})
file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp "")
add_compiler_rt_object_libraries(RTHwasan_dynamic_version_script_dummy
STATIC
ARCHS ${arch}
OBJECT_LIBS ${hwasan_object_lib}
+ RTHwasan_preinit
RTInterception
RTSanitizerCommon
RTSanitizerCommonLibc
RTSanitizerCommonCoverage
RTSanitizerCommonSymbolizer
+ RTLSanCommon
RTUbsan
CFLAGS ${hwasan_rtl_flags}
PARENT_TARGET hwasan)
RTSanitizerCommonLibc
RTSanitizerCommonCoverage
RTSanitizerCommonSymbolizer
+ RTLSanCommon
RTUbsan
RTUbsan_cxx
# The only purpose of RTHWAsan_dynamic_version_script_dummy is to
endif()
endforeach()
+add_compiler_rt_runtime(clang_rt.hwasan-preinit
+ STATIC
+ ARCHS ${HWASAN_SUPPORTED_ARCH}
+ OBJECT_LIBS RTHwasan_preinit
+ CFLAGS ${HWASAN_RTL_CFLAGS}
+ PARENT_TARGET hwasan)
+
add_compiler_rt_resource_file(hwasan_ignorelist hwasan_ignorelist.txt hwasan)
add_subdirectory("scripts")
#include "hwasan_checks.h"
#include "hwasan_dynamic_shadow.h"
#include "hwasan_globals.h"
+#include "hwasan_mapping.h"
#include "hwasan_poisoning.h"
#include "hwasan_report.h"
#include "hwasan_thread.h"
#include "sanitizer_common/sanitizer_common.h"
#include "sanitizer_common/sanitizer_flag_parser.h"
#include "sanitizer_common/sanitizer_flags.h"
+#include "sanitizer_common/sanitizer_interface_internal.h"
#include "sanitizer_common/sanitizer_libc.h"
#include "sanitizer_common/sanitizer_procmaps.h"
#include "sanitizer_common/sanitizer_stackdepot.h"
cf.clear_shadow_mmap_threshold = 4096 * (SANITIZER_ANDROID ? 2 : 8);
// Sigtrap is used in error reporting.
cf.handle_sigtrap = kHandleSignalExclusive;
+ // FIXME: enable once all false positives have been fixed.
+ cf.detect_leaks = false;
#if SANITIZER_ANDROID
// Let platform handle other signals. It is better at reporting them then we
RegisterHwasanFlags(&parser, f);
RegisterCommonFlags(&parser);
+#if CAN_SANITIZE_LEAKS
+ __lsan::Flags *lf = __lsan::flags();
+ lf->SetDefaults();
+
+ FlagParser lsan_parser;
+ __lsan::RegisterLsanFlags(&lsan_parser, lf);
+ RegisterCommonFlags(&lsan_parser);
+#endif
+
#if HWASAN_CONTAINS_UBSAN
__ubsan::Flags *uf = __ubsan::flags();
uf->SetDefaults();
#endif
parser.ParseStringFromEnv("HWASAN_OPTIONS");
+#if CAN_SANITIZE_LEAKS
+ lsan_parser.ParseStringFromEnv("LSAN_OPTIONS");
+#endif
#if HWASAN_CONTAINS_UBSAN
ubsan_parser.ParseStringFromEnv("UBSAN_OPTIONS");
#endif
if (Verbosity()) ReportUnrecognizedFlags();
if (common_flags()->help) parser.PrintFlagDescriptions();
+ // Flag validation:
+ if (!CAN_SANITIZE_LEAKS && common_flags()->detect_leaks) {
+ Report("%s: detect_leaks is not supported on this platform.\n",
+ SanitizerToolName);
+ Die();
+ }
}
static void CheckUnwind() {
static void HwasanFormatMemoryUsage(InternalScopedString &s) {
HwasanThreadList &thread_list = hwasanThreadList();
auto thread_stats = thread_list.GetThreadStats();
- auto *sds = StackDepotGetStats();
+ auto sds = StackDepotGetStats();
AllocatorStatCounters asc;
GetAllocatorStats(asc);
s.append(
internal_getpid(), GetRSS(), thread_stats.n_live_threads,
thread_stats.total_stack_size,
thread_stats.n_live_threads * thread_list.MemoryUsedPerThread(),
- sds->allocated, sds->n_uniq_ids, asc[AllocatorStatMapped]);
+ sds.allocated, sds.n_uniq_ids, asc[AllocatorStatMapped]);
}
#if SANITIZER_ANDROID
registers_frame);
}
-void HwasanTagMismatch(uptr addr, uptr access_info, uptr *registers_frame,
- size_t outsize) {
+void HwasanTagMismatch(uptr addr, uptr pc, uptr frame, uptr access_info,
+ uptr *registers_frame, size_t outsize) {
__hwasan::AccessInfo ai;
ai.is_store = access_info & 0x10;
ai.is_load = !ai.is_store;
else
ai.size = 1 << (access_info & 0xf);
- HandleTagMismatch(ai, (uptr)__builtin_return_address(0),
- (uptr)__builtin_frame_address(0), nullptr, registers_frame);
- __builtin_unreachable();
+ HandleTagMismatch(ai, pc, frame, nullptr, registers_frame);
}
Thread *GetCurrentThread() {
InitializeSingleGlobal(global);
}
-void __hwasan_init() {
+__attribute__((constructor(0))) void __hwasan_init() {
CHECK(!hwasan_init_is_running);
if (hwasan_inited) return;
hwasan_init_is_running = 1;
DisableCoreDumperIfNecessary();
InitInstrumentation();
- InitLoadedGlobals();
+ if constexpr (!SANITIZER_FUCHSIA) {
+ // Fuchsia's libc provides a hook (__sanitizer_module_loaded) that runs on
+ // the startup path which calls into __hwasan_library_loaded on all
+ // initially loaded modules, so explicitly registering the globals here
+ // isn't needed.
+ InitLoadedGlobals();
+ }
// Needs to be called here because flags()->random_tags might not have been
// initialized when InitInstrumentation() was called.
- GetCurrentThread()->InitRandomState();
+ GetCurrentThread()->EnsureRandomStateInited();
SetPrintfAndReportCallback(AppendToErrorMessageBuffer);
// This may call libc -> needs initialized shadow.
HwasanTSDThreadInit();
HwasanAllocatorInit();
+ HwasanInstallAtForkHandler();
+
+ if (CAN_SANITIZE_LEAKS) {
+ __lsan::InitCommonLsan();
+ InstallAtExitCheckLeaks();
+ }
#if HWASAN_CONTAINS_UBSAN
__ubsan::InitAsPlugin();
#endif
+ if (CAN_SANITIZE_LEAKS) {
+ __lsan::ScopedInterceptorDisabler disabler;
+ Symbolizer::LateInitialize();
+ } else {
+ Symbolizer::LateInitialize();
+ }
+
VPrintf(1, "HWAddressSanitizer init done\n");
hwasan_init_is_running = 0;
uptr shadow_last = MemToShadow(ptr_raw + sz - 1);
Printf("HWASan shadow map for %zx .. %zx (pointer tag %x)\n", ptr_raw,
ptr_raw + sz, GetTagFromPointer((uptr)p));
- for (uptr s = shadow_first; s <= shadow_last; ++s)
- Printf(" %zx: %x\n", ShadowToMem(s), *(tag_t *)s);
+ for (uptr s = shadow_first; s <= shadow_last; ++s) {
+ tag_t mem_tag = *reinterpret_cast<tag_t *>(s);
+ uptr granule_addr = ShadowToMem(s);
+ if (mem_tag && mem_tag < kShadowAlignment)
+ Printf(" %zx: %02x(%02x)\n", granule_addr, mem_tag,
+ *reinterpret_cast<tag_t *>(granule_addr + kShadowAlignment - 1));
+ else
+ Printf(" %zx: %02x\n", granule_addr, mem_tag);
+ }
}
sptr __hwasan_test_shadow(const void *p, uptr sz) {
return t->GenerateRandomTag();
}
+void __hwasan_add_frame_record(u64 frame_record_info) {
+ Thread *t = GetCurrentThread();
+ if (t)
+ t->stack_allocations()->push(frame_record_info);
+}
+
#if !SANITIZER_SUPPORTS_WEAK_HOOKS
extern "C" {
SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
// rest of the mismatch handling code (C++).
void __hwasan_tag_mismatch4(uptr addr, uptr access_info, uptr *registers_frame,
size_t outsize) {
- __hwasan::HwasanTagMismatch(addr, access_info, registers_frame, outsize);
+ __hwasan::HwasanTagMismatch(addr, (uptr)__builtin_return_address(0),
+ (uptr)__builtin_frame_address(0), access_info,
+ registers_frame, outsize);
}
} // extern "C"
void InitializeInterceptors();
void HwasanAllocatorInit();
+void HwasanAllocatorLock();
+void HwasanAllocatorUnlock();
void *hwasan_malloc(uptr size, StackTrace *stack);
void *hwasan_calloc(uptr nmemb, uptr size, StackTrace *stack);
void HwasanOnDeadlySignal(int signo, void *info, void *context);
+void HwasanInstallAtForkHandler();
+
+void InstallAtExitCheckLeaks();
+
void UpdateMemoryUsage();
void AppendToErrorMessageBuffer(const char *buffer);
// This dispatches to HandleTagMismatch but sets up the AccessInfo, program
// counter, and frame pointer.
-void HwasanTagMismatch(uptr addr, uptr access_info, uptr *registers_frame,
- size_t outsize);
+void HwasanTagMismatch(uptr addr, uptr pc, uptr frame, uptr access_info,
+ uptr *registers_frame, size_t outsize);
} // namespace __hwasan
-#define HWASAN_MALLOC_HOOK(ptr, size) \
- do { \
- if (&__sanitizer_malloc_hook) { \
- __sanitizer_malloc_hook(ptr, size); \
- } \
- RunMallocHooks(ptr, size); \
- } while (false)
-#define HWASAN_FREE_HOOK(ptr) \
- do { \
- if (&__sanitizer_free_hook) { \
- __sanitizer_free_hook(ptr); \
- } \
- RunFreeHooks(ptr); \
- } while (false)
-
-#if HWASAN_WITH_INTERCEPTORS && defined(__aarch64__)
+#if HWASAN_WITH_INTERCEPTORS
// For both bionic and glibc __sigset_t is an unsigned long.
typedef unsigned long __hw_sigset_t;
// Setjmp and longjmp implementations are platform specific, and hence the
-// interception code is platform specific too. As yet we've only implemented
-// the interception for AArch64.
-typedef unsigned long long __hw_register_buf[22];
+// interception code is platform specific too.
+# if defined(__aarch64__)
+constexpr size_t kHwRegisterBufSize = 22;
+# elif defined(__x86_64__)
+constexpr size_t kHwRegisterBufSize = 8;
+# elif SANITIZER_RISCV64
+// saving PC, 12 int regs, sp, 12 fp regs
+# ifndef __riscv_float_abi_soft
+constexpr size_t kHwRegisterBufSize = 1 + 12 + 1 + 12;
+# else
+constexpr size_t kHwRegisterBufSize = 1 + 12 + 1;
+# endif
+# endif
+typedef unsigned long long __hw_register_buf[kHwRegisterBufSize];
struct __hw_jmp_buf_struct {
// NOTE: The machine-dependent definition of `__sigsetjmp'
// assume that a `__hw_jmp_buf' begins with a `__hw_register_buf' and that
// `__mask_was_saved' follows it. Do not move these members or add others
// before it.
+ //
+ // We add a __magic field to our struct to catch cases where libc's setjmp
+ // populated the jmp_buf instead of our interceptor.
__hw_register_buf __jmpbuf; // Calling environment.
- int __mask_was_saved; // Saved the signal mask?
+ unsigned __mask_was_saved : 1; // Saved the signal mask?
+ unsigned __magic : 31; // Used to distinguish __hw_jmp_buf from jmp_buf.
__hw_sigset_t __saved_mask; // Saved signal mask.
};
typedef struct __hw_jmp_buf_struct __hw_jmp_buf[1];
typedef struct __hw_jmp_buf_struct __hw_sigjmp_buf[1];
-#endif // HWASAN_WITH_INTERCEPTORS && __aarch64__
+constexpr unsigned kHwJmpBufMagic = 0x248ACE77;
+#endif // HWASAN_WITH_INTERCEPTORS
#define ENSURE_HWASAN_INITED() \
do { \
#include "hwasan.h"
#include "interception/interception.h"
+#include "sanitizer_common/sanitizer_allocator_dlsym.h"
#include "sanitizer_common/sanitizer_allocator_interface.h"
#include "sanitizer_common/sanitizer_tls_get_addr.h"
-using namespace __hwasan;
+#if !SANITIZER_FUCHSIA
-static uptr allocated_for_dlsym;
-static const uptr kDlsymAllocPoolSize = 1024;
-static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize];
+using namespace __hwasan;
-static bool IsInDlsymAllocPool(const void *ptr) {
- uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
- return off < sizeof(alloc_memory_for_dlsym);
-}
+struct DlsymAlloc : public DlSymAllocator<DlsymAlloc> {
+ static bool UseImpl() { return !hwasan_inited; }
+};
-static void *AllocateFromLocalPool(uptr size_in_bytes) {
- uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize;
- void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym];
- allocated_for_dlsym += size_in_words;
- CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize);
- return mem;
-}
+extern "C" {
+SANITIZER_INTERFACE_ATTRIBUTE
int __sanitizer_posix_memalign(void **memptr, uptr alignment, uptr size) {
GET_MALLOC_STACK_TRACE;
CHECK_NE(memptr, 0);
return res;
}
+SANITIZER_INTERFACE_ATTRIBUTE
void *__sanitizer_memalign(uptr alignment, uptr size) {
GET_MALLOC_STACK_TRACE;
return hwasan_memalign(alignment, size, &stack);
}
+SANITIZER_INTERFACE_ATTRIBUTE
void *__sanitizer_aligned_alloc(uptr alignment, uptr size) {
GET_MALLOC_STACK_TRACE;
return hwasan_aligned_alloc(alignment, size, &stack);
}
+SANITIZER_INTERFACE_ATTRIBUTE
void *__sanitizer___libc_memalign(uptr alignment, uptr size) {
GET_MALLOC_STACK_TRACE;
void *ptr = hwasan_memalign(alignment, size, &stack);
return ptr;
}
+SANITIZER_INTERFACE_ATTRIBUTE
void *__sanitizer_valloc(uptr size) {
GET_MALLOC_STACK_TRACE;
return hwasan_valloc(size, &stack);
}
+SANITIZER_INTERFACE_ATTRIBUTE
void *__sanitizer_pvalloc(uptr size) {
GET_MALLOC_STACK_TRACE;
return hwasan_pvalloc(size, &stack);
}
+SANITIZER_INTERFACE_ATTRIBUTE
void __sanitizer_free(void *ptr) {
- GET_MALLOC_STACK_TRACE;
- if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr)))
+ if (!ptr)
return;
+ if (DlsymAlloc::PointerIsMine(ptr))
+ return DlsymAlloc::Free(ptr);
+ GET_MALLOC_STACK_TRACE;
hwasan_free(ptr, &stack);
}
+SANITIZER_INTERFACE_ATTRIBUTE
void __sanitizer_cfree(void *ptr) {
- GET_MALLOC_STACK_TRACE;
- if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr)))
+ if (!ptr)
return;
+ if (DlsymAlloc::PointerIsMine(ptr))
+ return DlsymAlloc::Free(ptr);
+ GET_MALLOC_STACK_TRACE;
hwasan_free(ptr, &stack);
}
+SANITIZER_INTERFACE_ATTRIBUTE
uptr __sanitizer_malloc_usable_size(const void *ptr) {
return __sanitizer_get_allocated_size(ptr);
}
+SANITIZER_INTERFACE_ATTRIBUTE
struct __sanitizer_struct_mallinfo __sanitizer_mallinfo() {
__sanitizer_struct_mallinfo sret;
internal_memset(&sret, 0, sizeof(sret));
return sret;
}
+SANITIZER_INTERFACE_ATTRIBUTE
int __sanitizer_mallopt(int cmd, int value) { return 0; }
+SANITIZER_INTERFACE_ATTRIBUTE
void __sanitizer_malloc_stats(void) {
// FIXME: implement, but don't call REAL(malloc_stats)!
}
+SANITIZER_INTERFACE_ATTRIBUTE
void *__sanitizer_calloc(uptr nmemb, uptr size) {
+ if (DlsymAlloc::Use())
+ return DlsymAlloc::Callocate(nmemb, size);
GET_MALLOC_STACK_TRACE;
- if (UNLIKELY(!hwasan_inited))
- // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
- return AllocateFromLocalPool(nmemb * size);
return hwasan_calloc(nmemb, size, &stack);
}
+SANITIZER_INTERFACE_ATTRIBUTE
void *__sanitizer_realloc(void *ptr, uptr size) {
+ if (DlsymAlloc::Use() || DlsymAlloc::PointerIsMine(ptr))
+ return DlsymAlloc::Realloc(ptr, size);
GET_MALLOC_STACK_TRACE;
- if (UNLIKELY(IsInDlsymAllocPool(ptr))) {
- uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
- uptr copy_size = Min(size, kDlsymAllocPoolSize - offset);
- void *new_ptr;
- if (UNLIKELY(!hwasan_inited)) {
- new_ptr = AllocateFromLocalPool(copy_size);
- } else {
- copy_size = size;
- new_ptr = hwasan_malloc(copy_size, &stack);
- }
- internal_memcpy(new_ptr, ptr, copy_size);
- return new_ptr;
- }
return hwasan_realloc(ptr, size, &stack);
}
+SANITIZER_INTERFACE_ATTRIBUTE
void *__sanitizer_reallocarray(void *ptr, uptr nmemb, uptr size) {
GET_MALLOC_STACK_TRACE;
return hwasan_reallocarray(ptr, nmemb, size, &stack);
}
+SANITIZER_INTERFACE_ATTRIBUTE
void *__sanitizer_malloc(uptr size) {
- GET_MALLOC_STACK_TRACE;
if (UNLIKELY(!hwasan_init_is_running))
ENSURE_HWASAN_INITED();
- if (UNLIKELY(!hwasan_inited))
- // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym.
- return AllocateFromLocalPool(size);
+ if (DlsymAlloc::Use())
+ return DlsymAlloc::Allocate(size);
+ GET_MALLOC_STACK_TRACE;
return hwasan_malloc(size, &stack);
}
+} // extern "C"
+
#if HWASAN_WITH_INTERCEPTORS
# define INTERCEPTOR_ALIAS(RET, FN, ARGS...) \
extern "C" SANITIZER_INTERFACE_ATTRIBUTE RET WRAP(FN)(ARGS) \
INTERCEPTOR_ALIAS(void, malloc_stats, void);
# endif
#endif // #if HWASAN_WITH_INTERCEPTORS
+
+#endif // SANITIZER_FUCHSIA
#include "hwasan_malloc_bisect.h"
#include "hwasan_thread.h"
#include "hwasan_report.h"
+#include "lsan/lsan_common.h"
namespace __hwasan {
static constexpr tag_t kFallbackAllocTag = 0xBB & kTagMask;
static constexpr tag_t kFallbackFreeTag = 0xBC;
-enum RightAlignMode {
- kRightAlignNever,
- kRightAlignSometimes,
- kRightAlignAlways
+enum {
+ // Either just allocated by underlying allocator, but AsanChunk is not yet
+ // ready, or almost returned to undelying allocator and AsanChunk is already
+ // meaningless.
+ CHUNK_INVALID = 0,
+ // The chunk is allocated and not yet freed.
+ CHUNK_ALLOCATED = 1,
};
+
// Initialized in HwasanAllocatorInit, an never changed.
static ALIGNED(16) u8 tail_magic[kShadowAlignment - 1];
bool HwasanChunkView::IsAllocated() const {
- return metadata_ && metadata_->alloc_context_id &&
- metadata_->get_requested_size();
-}
-
-// Aligns the 'addr' right to the granule boundary.
-static uptr AlignRight(uptr addr, uptr requested_size) {
- uptr tail_size = requested_size % kShadowAlignment;
- if (!tail_size) return addr;
- return addr + kShadowAlignment - tail_size;
+ return metadata_ && metadata_->IsAllocated();
}
uptr HwasanChunkView::Beg() const {
- if (metadata_ && metadata_->right_aligned)
- return AlignRight(block_, metadata_->get_requested_size());
return block_;
}
uptr HwasanChunkView::End() const {
return Beg() + UsedSize();
}
uptr HwasanChunkView::UsedSize() const {
- return metadata_->get_requested_size();
+ return metadata_->GetRequestedSize();
}
u32 HwasanChunkView::GetAllocStackId() const {
- return metadata_->alloc_context_id;
+ return metadata_->GetAllocStackId();
}
uptr HwasanChunkView::ActualSize() const {
return allocator.FromPrimary(reinterpret_cast<void *>(block_));
}
+bool HwasanChunkView::AddrIsInside(uptr addr) const {
+ return (addr >= Beg()) && (addr < Beg() + UsedSize());
+}
+
+inline void Metadata::SetAllocated(u32 stack, u64 size) {
+ Thread *t = GetCurrentThread();
+ u64 context = t ? t->unique_id() : kMainTid;
+ context <<= 32;
+ context += stack;
+ requested_size_low = size & ((1ul << 32) - 1);
+ requested_size_high = size >> 32;
+ atomic_store(&alloc_context_id, context, memory_order_relaxed);
+ atomic_store(&chunk_state, CHUNK_ALLOCATED, memory_order_release);
+}
+
+inline void Metadata::SetUnallocated() {
+ atomic_store(&chunk_state, CHUNK_INVALID, memory_order_release);
+ requested_size_low = 0;
+ requested_size_high = 0;
+ atomic_store(&alloc_context_id, 0, memory_order_relaxed);
+}
+
+inline bool Metadata::IsAllocated() const {
+ return atomic_load(&chunk_state, memory_order_relaxed) == CHUNK_ALLOCATED &&
+ GetRequestedSize();
+}
+
+inline u64 Metadata::GetRequestedSize() const {
+ return (static_cast<u64>(requested_size_high) << 32) + requested_size_low;
+}
+
+inline u32 Metadata::GetAllocStackId() const {
+ return atomic_load(&alloc_context_id, memory_order_relaxed);
+}
+
void GetAllocatorStats(AllocatorStatCounters s) {
allocator.GetStats(s);
}
+inline void Metadata::SetLsanTag(__lsan::ChunkTag tag) {
+ lsan_tag = tag;
+}
+
+inline __lsan::ChunkTag Metadata::GetLsanTag() const {
+ return static_cast<__lsan::ChunkTag>(lsan_tag);
+}
+
uptr GetAliasRegionStart() {
#if defined(HWASAN_ALIASING_MODE)
constexpr uptr kAliasRegionOffset = 1ULL << (kTaggableRegionCheckShift - 1);
tail_magic[i] = GetCurrentThread()->GenerateRandomTag();
}
+void HwasanAllocatorLock() { allocator.ForceLock(); }
+
+void HwasanAllocatorUnlock() { allocator.ForceUnlock(); }
+
void AllocatorSwallowThreadLocalCache(AllocatorCache *cache) {
allocator.SwallowCache(cache);
}
}
ReportAllocationSizeTooBig(orig_size, kMaxAllowedMallocSize, stack);
}
+ if (UNLIKELY(IsRssLimitExceeded())) {
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ ReportRssLimitExceeded(stack);
+ }
alignment = Max(alignment, kShadowAlignment);
uptr size = TaggedSize(orig_size);
return nullptr;
ReportOutOfMemory(size, stack);
}
- Metadata *meta =
- reinterpret_cast<Metadata *>(allocator.GetMetaData(allocated));
- meta->set_requested_size(orig_size);
- meta->alloc_context_id = StackDepotPut(*stack);
- meta->right_aligned = false;
if (zeroise) {
internal_memset(allocated, 0, size);
} else if (flags()->max_malloc_fill_size > 0) {
internal_memset(allocated, flags()->malloc_fill_byte, fill_size);
}
if (size != orig_size) {
- internal_memcpy(reinterpret_cast<u8 *>(allocated) + orig_size, tail_magic,
- size - orig_size - 1);
+ u8 *tail = reinterpret_cast<u8 *>(allocated) + orig_size;
+ uptr tail_length = size - orig_size;
+ internal_memcpy(tail, tail_magic, tail_length - 1);
+ // Short granule is excluded from magic tail, so we explicitly untag.
+ tail[tail_length - 1] = 0;
}
void *user_ptr = allocated;
}
}
- HWASAN_MALLOC_HOOK(user_ptr, size);
+ Metadata *meta =
+ reinterpret_cast<Metadata *>(allocator.GetMetaData(allocated));
+#if CAN_SANITIZE_LEAKS
+ meta->SetLsanTag(__lsan::DisabledInThisThread() ? __lsan::kIgnored
+ : __lsan::kDirectlyLeaked);
+#endif
+ meta->SetAllocated(StackDepotPut(*stack), orig_size);
+ RunMallocHooks(user_ptr, size);
return user_ptr;
}
return PossiblyShortTagMatches(mem_tag, tagged_uptr, 1);
}
+static bool CheckInvalidFree(StackTrace *stack, void *untagged_ptr,
+ void *tagged_ptr) {
+ // This function can return true if halt_on_error is false.
+ if (!MemIsApp(reinterpret_cast<uptr>(untagged_ptr)) ||
+ !PointerAndMemoryTagsMatch(tagged_ptr)) {
+ ReportInvalidFree(stack, reinterpret_cast<uptr>(tagged_ptr));
+ return true;
+ }
+ return false;
+}
+
static void HwasanDeallocate(StackTrace *stack, void *tagged_ptr) {
CHECK(tagged_ptr);
- HWASAN_FREE_HOOK(tagged_ptr);
+ RunFreeHooks(tagged_ptr);
- if (!PointerAndMemoryTagsMatch(tagged_ptr))
- ReportInvalidFree(stack, reinterpret_cast<uptr>(tagged_ptr));
+ bool in_taggable_region =
+ InTaggableRegion(reinterpret_cast<uptr>(tagged_ptr));
+ void *untagged_ptr = in_taggable_region ? UntagPtr(tagged_ptr) : tagged_ptr;
+
+ if (CheckInvalidFree(stack, untagged_ptr, tagged_ptr))
+ return;
- void *untagged_ptr = InTaggableRegion(reinterpret_cast<uptr>(tagged_ptr))
- ? UntagPtr(tagged_ptr)
- : tagged_ptr;
void *aligned_ptr = reinterpret_cast<void *>(
RoundDownTo(reinterpret_cast<uptr>(untagged_ptr), kShadowAlignment));
tag_t pointer_tag = GetTagFromPointer(reinterpret_cast<uptr>(tagged_ptr));
Metadata *meta =
reinterpret_cast<Metadata *>(allocator.GetMetaData(aligned_ptr));
- uptr orig_size = meta->get_requested_size();
+ if (!meta) {
+ ReportInvalidFree(stack, reinterpret_cast<uptr>(tagged_ptr));
+ return;
+ }
+ uptr orig_size = meta->GetRequestedSize();
u32 free_context_id = StackDepotPut(*stack);
- u32 alloc_context_id = meta->alloc_context_id;
+ u32 alloc_context_id = meta->GetAllocStackId();
// Check tail magic.
uptr tagged_size = TaggedSize(orig_size);
CHECK_LT(tail_size, kShadowAlignment);
void *tail_beg = reinterpret_cast<void *>(
reinterpret_cast<uptr>(aligned_ptr) + orig_size);
- if (tail_size && internal_memcmp(tail_beg, tail_magic, tail_size))
+ tag_t short_granule_memtag = *(reinterpret_cast<tag_t *>(
+ reinterpret_cast<uptr>(tail_beg) + tail_size));
+ if (tail_size &&
+ (internal_memcmp(tail_beg, tail_magic, tail_size) ||
+ (in_taggable_region && pointer_tag != short_granule_memtag)))
ReportTailOverwritten(stack, reinterpret_cast<uptr>(tagged_ptr),
orig_size, tail_magic);
}
- meta->set_requested_size(0);
- meta->alloc_context_id = 0;
+ // TODO(kstoimenov): consider meta->SetUnallocated(free_context_id).
+ meta->SetUnallocated();
// This memory will not be reused by anyone else, so we are free to keep it
// poisoned.
Thread *t = GetCurrentThread();
Min(TaggedSize(orig_size), (uptr)flags()->max_free_fill_size);
internal_memset(aligned_ptr, flags()->free_fill_byte, fill_size);
}
- if (InTaggableRegion(reinterpret_cast<uptr>(tagged_ptr)) &&
- flags()->tag_in_free && malloc_bisect(stack, 0) &&
+ if (in_taggable_region && flags()->tag_in_free && malloc_bisect(stack, 0) &&
atomic_load_relaxed(&hwasan_allocator_tagging_enabled)) {
// Always store full 8-bit tags on free to maximize UAF detection.
tag_t tag;
static void *HwasanReallocate(StackTrace *stack, void *tagged_ptr_old,
uptr new_size, uptr alignment) {
- if (!PointerAndMemoryTagsMatch(tagged_ptr_old))
- ReportInvalidFree(stack, reinterpret_cast<uptr>(tagged_ptr_old));
-
+ void *untagged_ptr_old =
+ InTaggableRegion(reinterpret_cast<uptr>(tagged_ptr_old))
+ ? UntagPtr(tagged_ptr_old)
+ : tagged_ptr_old;
+ if (CheckInvalidFree(stack, untagged_ptr_old, tagged_ptr_old))
+ return nullptr;
void *tagged_ptr_new =
HwasanAllocate(stack, new_size, alignment, false /*zeroise*/);
if (tagged_ptr_old && tagged_ptr_new) {
- void *untagged_ptr_old = UntagPtr(tagged_ptr_old);
Metadata *meta =
reinterpret_cast<Metadata *>(allocator.GetMetaData(untagged_ptr_old));
internal_memcpy(
UntagPtr(tagged_ptr_new), untagged_ptr_old,
- Min(new_size, static_cast<uptr>(meta->get_requested_size())));
+ Min(new_size, static_cast<uptr>(meta->GetRequestedSize())));
HwasanDeallocate(stack, tagged_ptr_old);
}
return tagged_ptr_new;
}
HwasanChunkView FindHeapChunkByAddress(uptr address) {
+ if (!allocator.PointerIsMine(reinterpret_cast<void *>(address)))
+ return HwasanChunkView();
void *block = allocator.GetBlockBegin(reinterpret_cast<void*>(address));
if (!block)
return HwasanChunkView();
if (!untagged_ptr) return 0;
const void *beg = allocator.GetBlockBegin(untagged_ptr);
Metadata *b = (Metadata *)allocator.GetMetaData(untagged_ptr);
- if (b->right_aligned) {
- if (beg != reinterpret_cast<void *>(RoundDownTo(
- reinterpret_cast<uptr>(untagged_ptr), kShadowAlignment)))
- return 0;
- } else {
- if (beg != untagged_ptr) return 0;
- }
- return b->get_requested_size();
+ if (beg != untagged_ptr) return 0;
+ return b->GetRequestedSize();
}
void *hwasan_malloc(uptr size, StackTrace *stack) {
} // namespace __hwasan
+// --- Implementation of LSan-specific functions --- {{{1
+namespace __lsan {
+
+void LockAllocator() {
+ __hwasan::HwasanAllocatorLock();
+}
+
+void UnlockAllocator() {
+ __hwasan::HwasanAllocatorUnlock();
+}
+
+void GetAllocatorGlobalRange(uptr *begin, uptr *end) {
+ *begin = (uptr)&__hwasan::allocator;
+ *end = *begin + sizeof(__hwasan::allocator);
+}
+
+uptr PointsIntoChunk(void *p) {
+ p = __hwasan::InTaggableRegion(reinterpret_cast<uptr>(p)) ? UntagPtr(p) : p;
+ uptr addr = reinterpret_cast<uptr>(p);
+ uptr chunk =
+ reinterpret_cast<uptr>(__hwasan::allocator.GetBlockBeginFastLocked(p));
+ if (!chunk)
+ return 0;
+ __hwasan::Metadata *metadata = reinterpret_cast<__hwasan::Metadata *>(
+ __hwasan::allocator.GetMetaData(reinterpret_cast<void *>(chunk)));
+ if (!metadata || !metadata->IsAllocated())
+ return 0;
+ if (addr < chunk + metadata->GetRequestedSize())
+ return chunk;
+ if (IsSpecialCaseOfOperatorNew0(chunk, metadata->GetRequestedSize(), addr))
+ return chunk;
+ return 0;
+}
+
+uptr GetUserBegin(uptr chunk) {
+ if (__hwasan::InTaggableRegion(chunk))
+ CHECK_EQ(UntagAddr(chunk), chunk);
+ void *block = __hwasan::allocator.GetBlockBeginFastLocked(
+ reinterpret_cast<void *>(chunk));
+ if (!block)
+ return 0;
+ __hwasan::Metadata *metadata = reinterpret_cast<__hwasan::Metadata *>(
+ __hwasan::allocator.GetMetaData(block));
+ if (!metadata || !metadata->IsAllocated())
+ return 0;
+
+ return reinterpret_cast<uptr>(block);
+}
+
+LsanMetadata::LsanMetadata(uptr chunk) {
+ if (__hwasan::InTaggableRegion(chunk))
+ CHECK_EQ(UntagAddr(chunk), chunk);
+ metadata_ =
+ chunk ? __hwasan::allocator.GetMetaData(reinterpret_cast<void *>(chunk))
+ : nullptr;
+}
+
+bool LsanMetadata::allocated() const {
+ if (!metadata_)
+ return false;
+ __hwasan::Metadata *m = reinterpret_cast<__hwasan::Metadata *>(metadata_);
+ return m->IsAllocated();
+}
+
+ChunkTag LsanMetadata::tag() const {
+ __hwasan::Metadata *m = reinterpret_cast<__hwasan::Metadata *>(metadata_);
+ return m->GetLsanTag();
+}
+
+void LsanMetadata::set_tag(ChunkTag value) {
+ __hwasan::Metadata *m = reinterpret_cast<__hwasan::Metadata *>(metadata_);
+ m->SetLsanTag(value);
+}
+
+uptr LsanMetadata::requested_size() const {
+ __hwasan::Metadata *m = reinterpret_cast<__hwasan::Metadata *>(metadata_);
+ return m->GetRequestedSize();
+}
+
+u32 LsanMetadata::stack_trace_id() const {
+ __hwasan::Metadata *m = reinterpret_cast<__hwasan::Metadata *>(metadata_);
+ return m->GetAllocStackId();
+}
+
+void ForEachChunk(ForEachChunkCallback callback, void *arg) {
+ __hwasan::allocator.ForEachChunk(callback, arg);
+}
+
+IgnoreObjectResult IgnoreObjectLocked(const void *p) {
+ p = __hwasan::InTaggableRegion(reinterpret_cast<uptr>(p)) ? UntagPtr(p) : p;
+ uptr addr = reinterpret_cast<uptr>(p);
+ uptr chunk =
+ reinterpret_cast<uptr>(__hwasan::allocator.GetBlockBeginFastLocked(p));
+ if (!chunk)
+ return kIgnoreObjectInvalid;
+ __hwasan::Metadata *metadata = reinterpret_cast<__hwasan::Metadata *>(
+ __hwasan::allocator.GetMetaData(reinterpret_cast<void *>(chunk)));
+ if (!metadata || !metadata->IsAllocated())
+ return kIgnoreObjectInvalid;
+ if (addr >= chunk + metadata->GetRequestedSize())
+ return kIgnoreObjectInvalid;
+ if (metadata->GetLsanTag() == kIgnored)
+ return kIgnoreObjectAlreadyIgnored;
+
+ metadata->SetLsanTag(kIgnored);
+ return kIgnoreObjectSuccess;
+}
+
+} // namespace __lsan
+
using namespace __hwasan;
void __hwasan_enable_allocator_tagging() {
#include "hwasan_interface_internal.h"
#include "hwasan_mapping.h"
#include "hwasan_poisoning.h"
+#include "lsan/lsan_common.h"
#include "sanitizer_common/sanitizer_allocator.h"
#include "sanitizer_common/sanitizer_allocator_checks.h"
#include "sanitizer_common/sanitizer_allocator_interface.h"
#include "sanitizer_common/sanitizer_common.h"
#include "sanitizer_common/sanitizer_ring_buffer.h"
-#if !defined(__aarch64__) && !defined(__x86_64__)
-#error Unsupported platform
+#if !defined(__aarch64__) && !defined(__x86_64__) && !(SANITIZER_RISCV64)
+# error Unsupported platform
#endif
namespace __hwasan {
struct Metadata {
+ private:
+ atomic_uint64_t alloc_context_id;
u32 requested_size_low;
- u32 requested_size_high : 31;
- u32 right_aligned : 1;
- u32 alloc_context_id;
- u64 get_requested_size() {
- return (static_cast<u64>(requested_size_high) << 32) + requested_size_low;
- }
- void set_requested_size(u64 size) {
- requested_size_low = size & ((1ul << 32) - 1);
- requested_size_high = size >> 32;
- }
+ u16 requested_size_high;
+ atomic_uint8_t chunk_state;
+ u8 lsan_tag;
+
+ public:
+ inline void SetAllocated(u32 stack, u64 size);
+ inline void SetUnallocated();
+
+ inline bool IsAllocated() const;
+ inline u64 GetRequestedSize() const;
+ inline u32 GetAllocStackId() const;
+ inline void SetLsanTag(__lsan::ChunkTag tag);
+ inline __lsan::ChunkTag GetLsanTag() const;
};
+static_assert(sizeof(Metadata) == 16);
struct HwasanMapUnmapCallback {
void OnMap(uptr p, uptr size) const { UpdateMemoryUsage(); }
uptr ActualSize() const; // Size allocated by the allocator.
u32 GetAllocStackId() const;
bool FromSmallHeap() const;
+ bool AddrIsInside(uptr addr) const;
+
private:
+ friend class __lsan::LsanMetadata;
uptr block_;
Metadata *const metadata_;
};
"int3\n"
"nopl %c0(%%rax)\n" ::"n"(0x40 + X),
"D"(p));
+#elif SANITIZER_RISCV64
+ // Put pointer into x10
+ // addiw contains immediate of 0x40 + X, where 0x40 is magic number and X
+ // encodes access size
+ register uptr x10 asm("x10") = p;
+ asm volatile(
+ "ebreak\n"
+ "addiw x0, x0, %1\n" ::"r"(x10),
+ "I"(0x40 + X));
#else
// FIXME: not always sigill.
__builtin_trap();
"int3\n"
"nopl %c0(%%rax)\n" ::"n"(0x40 + X),
"D"(p), "S"(size));
+#elif SANITIZER_RISCV64
+ // Put access size into x11
+ register uptr x10 asm("x10") = p;
+ register uptr x11 asm("x11") = size;
+ asm volatile(
+ "ebreak\n"
+ "addiw x0, x0, %2\n" ::"r"(x10),
+ "r"(x11), "I"(0x40 + X));
#else
__builtin_trap();
#endif
return false;
if ((ptr & (kShadowAlignment - 1)) + sz > mem_tag)
return false;
-#ifndef __aarch64__
+#if !defined(__aarch64__) && !(SANITIZER_RISCV64)
ptr = UntagAddr(ptr);
#endif
return *(u8 *)(ptr | (kShadowAlignment - 1)) == ptr_tag;
// is statically linked and the sanitizer runtime and the program are linked
// against different unwinders. The _Unwind_Context data structure is opaque so
// it may be incompatible between unwinders.
-typedef _Unwind_Word GetGRFn(_Unwind_Context* context, int index);
-typedef _Unwind_Word GetCFAFn(_Unwind_Context* context);
+typedef uintptr_t GetGRFn(_Unwind_Context* context, int index);
+typedef uintptr_t GetCFAFn(_Unwind_Context* context);
extern "C" SANITIZER_INTERFACE_ATTRIBUTE _Unwind_Reason_Code
__hwasan_personality_wrapper(int version, _Unwind_Action actions,
uptr fp = get_gr(context, 6); // rbp
#elif defined(__aarch64__)
uptr fp = get_gr(context, 29); // x29
+#elif SANITIZER_RISCV64
+ uptr fp = get_gr(context, 8); // x8
#else
#error Unsupported architecture
#endif
HWASAN_FLAG(bool, free_checks_tail_magic, 1,
"If set, free() will check the magic values "
- "to the right of the allocated object "
+ "after the allocated object "
"if the allocation size is not a divident of the granule size")
HWASAN_FLAG(
int, max_free_fill_size, 0,
#include "sanitizer_common/sanitizer_fuchsia.h"
#if SANITIZER_FUCHSIA
+#include <zircon/features.h>
+#include <zircon/syscalls.h>
+
#include "hwasan.h"
#include "hwasan_interface_internal.h"
#include "hwasan_report.h"
static void ThreadStartHook(void *hook, thrd_t self) {
Thread *thread = static_cast<Thread *>(hook);
FinishThreadInitialization(thread);
- thread->InitRandomState();
+ thread->EnsureRandomStateInited();
}
// This is the function that sets up the stack ring buffer and enables us to use
// function is unneeded.
void InstallAtExitHandler() {}
-// TODO(fxbug.dev/81499): Once we finalize the tagged pointer ABI in zircon, we should come back
-// here and implement the appropriate check that TBI is enabled.
-void InitializeOsSupport() {}
+void HwasanInstallAtForkHandler() {}
+
+void InstallAtExitCheckLeaks() {}
+
+void InitializeOsSupport() {
+#ifdef __aarch64__
+ uint32_t features = 0;
+ CHECK_EQ(zx_system_get_features(ZX_FEATURE_KIND_ADDRESS_TAGGING, &features),
+ ZX_OK);
+ if (!(features & ZX_ARM64_FEATURE_ADDRESS_TAGGING_TBI) &&
+ flags()->fail_without_syscall_abi) {
+ Printf(
+ "FATAL: HWAddressSanitizer requires "
+ "ZX_ARM64_FEATURE_ADDRESS_TAGGING_TBI.\n");
+ Die();
+ }
+#endif
+}
} // namespace __hwasan
+namespace __lsan {
+
+bool UseExitcodeOnLeak() { return __hwasan::flags()->halt_on_error; }
+
+} // namespace __lsan
+
extern "C" {
void *__sanitizer_before_thread_create_hook(thrd_t thread, bool detached,
__hwasan::ThreadExitHook(hook, self);
}
+void __sanitizer_module_loaded(const struct dl_phdr_info *info, size_t) {
+ __hwasan_library_loaded(info->dlpi_addr, info->dlpi_phdr, info->dlpi_phnum);
+}
+
} // extern "C"
#endif // SANITIZER_FUCHSIA
return res;
}
+INTERCEPTOR(int, pthread_join, void *t, void **arg) {
+ return REAL(pthread_join)(t, arg);
+}
+
+DEFINE_REAL_PTHREAD_FUNCTIONS
+
DEFINE_REAL(int, vfork)
DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(int, vfork)
-#endif // HWASAN_WITH_INTERCEPTORS
-#if HWASAN_WITH_INTERCEPTORS && defined(__aarch64__)
// Get and/or change the set of blocked signals.
extern "C" int sigprocmask(int __how, const __hw_sigset_t *__restrict __set,
__hw_sigset_t *__restrict __oset);
#define SIG_BLOCK 0
#define SIG_SETMASK 2
extern "C" int __sigjmp_save(__hw_sigjmp_buf env, int savemask) {
+ env[0].__magic = kHwJmpBufMagic;
env[0].__mask_was_saved =
(savemask && sigprocmask(SIG_BLOCK, (__hw_sigset_t *)0,
&env[0].__saved_mask) == 0);
static void __attribute__((always_inline))
InternalLongjmp(__hw_register_buf env, int retval) {
+# if defined(__aarch64__)
+ constexpr size_t kSpIndex = 13;
+# elif defined(__x86_64__)
+ constexpr size_t kSpIndex = 6;
+# elif SANITIZER_RISCV64
+ constexpr size_t kSpIndex = 13;
+# endif
+
// Clear all memory tags on the stack between here and where we're going.
- unsigned long long stack_pointer = env[13];
+ unsigned long long stack_pointer = env[kSpIndex];
// The stack pointer should never be tagged, so we don't need to clear the
// tag for this function call.
__hwasan_handle_longjmp((void *)stack_pointer);
// Must implement this ourselves, since we don't know the order of registers
// in different libc implementations and many implementations mangle the
// stack pointer so we can't use it without knowing the demangling scheme.
+# if defined(__aarch64__)
register long int retval_tmp asm("x1") = retval;
register void *env_address asm("x0") = &env[0];
asm volatile("ldp x19, x20, [%0, #0<<3];"
"br x30;"
: "+r"(env_address)
: "r"(retval_tmp));
+# elif defined(__x86_64__)
+ register long int retval_tmp asm("%rsi") = retval;
+ register void *env_address asm("%rdi") = &env[0];
+ asm volatile(
+ // Restore registers.
+ "mov (0*8)(%0),%%rbx;"
+ "mov (1*8)(%0),%%rbp;"
+ "mov (2*8)(%0),%%r12;"
+ "mov (3*8)(%0),%%r13;"
+ "mov (4*8)(%0),%%r14;"
+ "mov (5*8)(%0),%%r15;"
+ "mov (6*8)(%0),%%rsp;"
+ "mov (7*8)(%0),%%rdx;"
+ // Return 1 if retval is 0.
+ "mov $1,%%rax;"
+ "test %1,%1;"
+ "cmovnz %1,%%rax;"
+ "jmp *%%rdx;" ::"r"(env_address),
+ "r"(retval_tmp));
+# elif SANITIZER_RISCV64
+ register long int retval_tmp asm("x11") = retval;
+ register void *env_address asm("x10") = &env[0];
+ asm volatile(
+ "ld ra, 0<<3(%0);"
+ "ld s0, 1<<3(%0);"
+ "ld s1, 2<<3(%0);"
+ "ld s2, 3<<3(%0);"
+ "ld s3, 4<<3(%0);"
+ "ld s4, 5<<3(%0);"
+ "ld s5, 6<<3(%0);"
+ "ld s6, 7<<3(%0);"
+ "ld s7, 8<<3(%0);"
+ "ld s8, 9<<3(%0);"
+ "ld s9, 10<<3(%0);"
+ "ld s10, 11<<3(%0);"
+ "ld s11, 12<<3(%0);"
+# if __riscv_float_abi_double
+ "fld fs0, 14<<3(%0);"
+ "fld fs1, 15<<3(%0);"
+ "fld fs2, 16<<3(%0);"
+ "fld fs3, 17<<3(%0);"
+ "fld fs4, 18<<3(%0);"
+ "fld fs5, 19<<3(%0);"
+ "fld fs6, 20<<3(%0);"
+ "fld fs7, 21<<3(%0);"
+ "fld fs8, 22<<3(%0);"
+ "fld fs9, 23<<3(%0);"
+ "fld fs10, 24<<3(%0);"
+ "fld fs11, 25<<3(%0);"
+# elif __riscv_float_abi_soft
+# else
+# error "Unsupported case"
+# endif
+ "ld a4, 13<<3(%0);"
+ "mv sp, a4;"
+ // Return the value requested to return through arguments.
+ // This should be in x11 given what we requested above.
+ "seqz a0, %1;"
+ "add a0, a0, %1;"
+ "ret;"
+ : "+r"(env_address)
+ : "r"(retval_tmp));
+# endif
}
INTERCEPTOR(void, siglongjmp, __hw_sigjmp_buf env, int val) {
+ if (env[0].__magic != kHwJmpBufMagic) {
+ Printf(
+ "WARNING: Unexpected bad jmp_buf. Either setjmp was not called or "
+ "there is a bug in HWASan.\n");
+ return REAL(siglongjmp)(env, val);
+ }
+
if (env[0].__mask_was_saved)
// Restore the saved signal mask.
(void)sigprocmask(SIG_SETMASK, &env[0].__saved_mask,
// _setjmp on start_thread. Hence we have to intercept the longjmp on
// pthread_exit so the __hw_jmp_buf order matches.
INTERCEPTOR(void, __libc_longjmp, __hw_jmp_buf env, int val) {
+ if (env[0].__magic != kHwJmpBufMagic)
+ return REAL(__libc_longjmp)(env, val);
InternalLongjmp(env[0].__jmpbuf, val);
}
INTERCEPTOR(void, longjmp, __hw_jmp_buf env, int val) {
+ if (env[0].__magic != kHwJmpBufMagic) {
+ Printf(
+ "WARNING: Unexpected bad jmp_buf. Either setjmp was not called or "
+ "there is a bug in HWASan.\n");
+ return REAL(longjmp)(env, val);
+ }
InternalLongjmp(env[0].__jmpbuf, val);
}
#undef SIG_BLOCK
#undef SIG_SETMASK
-#endif // HWASAN_WITH_INTERCEPTORS && __aarch64__
-
-static void BeforeFork() {
- StackDepotLockAll();
-}
-
-static void AfterFork() {
- StackDepotUnlockAll();
-}
-
-INTERCEPTOR(int, fork, void) {
- ENSURE_HWASAN_INITED();
- BeforeFork();
- int pid = REAL(fork)();
- AfterFork();
- return pid;
-}
+# endif // HWASAN_WITH_INTERCEPTORS
namespace __hwasan {
int OnExit() {
+ if (CAN_SANITIZE_LEAKS && common_flags()->detect_leaks &&
+ __lsan::HasReportedLeaks()) {
+ return common_flags()->exitcode;
+ }
// FIXME: ask frontend whether we need to return failure.
return 0;
}
static int inited = 0;
CHECK_EQ(inited, 0);
- INTERCEPT_FUNCTION(fork);
-
#if HWASAN_WITH_INTERCEPTORS
#if defined(__linux__)
+ INTERCEPT_FUNCTION(__libc_longjmp);
+ INTERCEPT_FUNCTION(longjmp);
+ INTERCEPT_FUNCTION(siglongjmp);
INTERCEPT_FUNCTION(vfork);
#endif // __linux__
INTERCEPT_FUNCTION(pthread_create);
-#endif
+ INTERCEPT_FUNCTION(pthread_join);
+# endif
inited = 1;
}
SANITIZER_INTERFACE_ATTRIBUTE
void __hwasan_print_memory_usage();
+// The compiler will generate this when
+// `-hwasan-record-stack-history-with-calls` is added as a flag, which will add
+// frame record information to the stack ring buffer. This is an alternative to
+// the compiler emitting instructions in the prologue for doing the same thing
+// by accessing the ring buffer directly.
SANITIZER_INTERFACE_ATTRIBUTE
-int __sanitizer_posix_memalign(void **memptr, uptr alignment, uptr size);
-
-SANITIZER_INTERFACE_ATTRIBUTE
-void * __sanitizer_memalign(uptr alignment, uptr size);
-
-SANITIZER_INTERFACE_ATTRIBUTE
-void * __sanitizer_aligned_alloc(uptr alignment, uptr size);
-
-SANITIZER_INTERFACE_ATTRIBUTE
-void * __sanitizer___libc_memalign(uptr alignment, uptr size);
-
-SANITIZER_INTERFACE_ATTRIBUTE
-void * __sanitizer_valloc(uptr size);
-
-SANITIZER_INTERFACE_ATTRIBUTE
-void * __sanitizer_pvalloc(uptr size);
-
-SANITIZER_INTERFACE_ATTRIBUTE
-void __sanitizer_free(void *ptr);
-
-SANITIZER_INTERFACE_ATTRIBUTE
-void __sanitizer_cfree(void *ptr);
-
-SANITIZER_INTERFACE_ATTRIBUTE
-uptr __sanitizer_malloc_usable_size(const void *ptr);
-
-SANITIZER_INTERFACE_ATTRIBUTE
-__hwasan::__sanitizer_struct_mallinfo __sanitizer_mallinfo();
-
-SANITIZER_INTERFACE_ATTRIBUTE
-int __sanitizer_mallopt(int cmd, int value);
-
-SANITIZER_INTERFACE_ATTRIBUTE
-void __sanitizer_malloc_stats(void);
-
-SANITIZER_INTERFACE_ATTRIBUTE
-void * __sanitizer_calloc(uptr nmemb, uptr size);
-
-SANITIZER_INTERFACE_ATTRIBUTE
-void * __sanitizer_realloc(void *ptr, uptr size);
-
-SANITIZER_INTERFACE_ATTRIBUTE
-void * __sanitizer_reallocarray(void *ptr, uptr nmemb, uptr size);
-
-SANITIZER_INTERFACE_ATTRIBUTE
-void * __sanitizer_malloc(uptr size);
+void __hwasan_add_frame_record(u64 frame_record_info);
SANITIZER_INTERFACE_ATTRIBUTE
void *__hwasan_memcpy(void *dst, const void *src, uptr size);
#include "sanitizer_common/sanitizer_platform.h"
#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
-#include "hwasan.h"
-#include "hwasan_dynamic_shadow.h"
-#include "hwasan_interface_internal.h"
-#include "hwasan_mapping.h"
-#include "hwasan_report.h"
-#include "hwasan_thread.h"
-#include "hwasan_thread_list.h"
-
-#include <dlfcn.h>
-#include <elf.h>
-#include <link.h>
-#include <pthread.h>
-#include <signal.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <sys/resource.h>
-#include <sys/time.h>
-#include <unistd.h>
-#include <unwind.h>
-#include <sys/prctl.h>
-#include <errno.h>
-
-#include "sanitizer_common/sanitizer_common.h"
-#include "sanitizer_common/sanitizer_procmaps.h"
+# include <dlfcn.h>
+# include <elf.h>
+# include <errno.h>
+# include <link.h>
+# include <pthread.h>
+# include <signal.h>
+# include <stdio.h>
+# include <stdlib.h>
+# include <sys/prctl.h>
+# include <sys/resource.h>
+# include <sys/time.h>
+# include <unistd.h>
+# include <unwind.h>
+
+# include "hwasan.h"
+# include "hwasan_dynamic_shadow.h"
+# include "hwasan_interface_internal.h"
+# include "hwasan_mapping.h"
+# include "hwasan_report.h"
+# include "hwasan_thread.h"
+# include "hwasan_thread_list.h"
+# include "sanitizer_common/sanitizer_common.h"
+# include "sanitizer_common/sanitizer_procmaps.h"
+# include "sanitizer_common/sanitizer_stackdepot.h"
// Configurations of HWASAN_WITH_INTERCEPTORS and SANITIZER_ANDROID.
//
// Tested with check-hwasan on x86_64-linux.
// HWASAN_WITH_INTERCEPTORS=ON, SANITIZER_ANDROID=ON
// Tested with check-hwasan on aarch64-linux-android.
-#if !SANITIZER_ANDROID
+# if !SANITIZER_ANDROID
SANITIZER_INTERFACE_ATTRIBUTE
THREADLOCAL uptr __hwasan_tls;
-#endif
+# endif
namespace __hwasan {
FindDynamicShadowStart(shadow_size_bytes);
}
+static void MaybeDieIfNoTaggingAbi(const char *message) {
+ if (!flags()->fail_without_syscall_abi)
+ return;
+ Printf("FATAL: %s\n", message);
+ Die();
+}
+
+# define PR_SET_TAGGED_ADDR_CTRL 55
+# define PR_GET_TAGGED_ADDR_CTRL 56
+# define PR_TAGGED_ADDR_ENABLE (1UL << 0)
+# define ARCH_GET_UNTAG_MASK 0x4001
+# define ARCH_ENABLE_TAGGED_ADDR 0x4002
+# define ARCH_GET_MAX_TAG_BITS 0x4003
+
+static bool CanUseTaggingAbi() {
+# if defined(__x86_64__)
+ unsigned long num_bits = 0;
+ // Check for x86 LAM support. This API is based on a currently unsubmitted
+ // patch to the Linux kernel (as of August 2022) and is thus subject to
+ // change. The patch is here:
+ // https://lore.kernel.org/all/20220815041803.17954-1-kirill.shutemov@linux.intel.com/
+ //
+ // arch_prctl(ARCH_GET_MAX_TAG_BITS, &bits) returns the maximum number of tag
+ // bits the user can request, or zero if LAM is not supported by the hardware.
+ if (internal_iserror(internal_arch_prctl(ARCH_GET_MAX_TAG_BITS,
+ reinterpret_cast<uptr>(&num_bits))))
+ return false;
+ // The platform must provide enough bits for HWASan tags.
+ if (num_bits < kTagBits)
+ return false;
+ return true;
+# else
+ // Check for ARM TBI support.
+ return !internal_iserror(internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0));
+# endif // __x86_64__
+}
+
+static bool EnableTaggingAbi() {
+# if defined(__x86_64__)
+ // Enable x86 LAM tagging for the process.
+ //
+ // arch_prctl(ARCH_ENABLE_TAGGED_ADDR, bits) enables tagging if the number of
+ // tag bits requested by the user does not exceed that provided by the system.
+ // arch_prctl(ARCH_GET_UNTAG_MASK, &mask) returns the mask of significant
+ // address bits. It is ~0ULL if either LAM is disabled for the process or LAM
+ // is not supported by the hardware.
+ if (internal_iserror(internal_arch_prctl(ARCH_ENABLE_TAGGED_ADDR, kTagBits)))
+ return false;
+ unsigned long mask = 0;
+ // Make sure the tag bits are where we expect them to be.
+ if (internal_iserror(internal_arch_prctl(ARCH_GET_UNTAG_MASK,
+ reinterpret_cast<uptr>(&mask))))
+ return false;
+ // @mask has ones for non-tag bits, whereas @kAddressTagMask has ones for tag
+ // bits. Therefore these masks must not overlap.
+ if (mask & kAddressTagMask)
+ return false;
+ return true;
+# else
+ // Enable ARM TBI tagging for the process. If for some reason tagging is not
+ // supported, prctl(PR_SET_TAGGED_ADDR_CTRL, PR_TAGGED_ADDR_ENABLE) returns
+ // -EINVAL.
+ if (internal_iserror(internal_prctl(PR_SET_TAGGED_ADDR_CTRL,
+ PR_TAGGED_ADDR_ENABLE, 0, 0, 0)))
+ return false;
+ // Ensure that TBI is enabled.
+ if (internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0) !=
+ PR_TAGGED_ADDR_ENABLE)
+ return false;
+ return true;
+# endif // __x86_64__
+}
+
void InitializeOsSupport() {
-#define PR_SET_TAGGED_ADDR_CTRL 55
-#define PR_GET_TAGGED_ADDR_CTRL 56
-#define PR_TAGGED_ADDR_ENABLE (1UL << 0)
// Check we're running on a kernel that can use the tagged address ABI.
- int local_errno = 0;
- if (internal_iserror(internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0),
- &local_errno) &&
- local_errno == EINVAL) {
+ bool has_abi = CanUseTaggingAbi();
+
+ if (!has_abi) {
# if SANITIZER_ANDROID || defined(HWASAN_ALIASING_MODE)
// Some older Android kernels have the tagged pointer ABI on
// unconditionally, and hence don't have the tagged-addr prctl while still
// case.
return;
# else
- if (flags()->fail_without_syscall_abi) {
- Printf(
- "FATAL: "
- "HWAddressSanitizer requires a kernel with tagged address ABI.\n");
- Die();
- }
+ MaybeDieIfNoTaggingAbi(
+ "HWAddressSanitizer requires a kernel with tagged address ABI.");
# endif
}
- // Turn on the tagged address ABI.
- if ((internal_iserror(internal_prctl(PR_SET_TAGGED_ADDR_CTRL,
- PR_TAGGED_ADDR_ENABLE, 0, 0, 0)) ||
- !internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0))) {
-# if defined(__x86_64__) && !defined(HWASAN_ALIASING_MODE)
- // Try the new prctl API for Intel LAM. The API is based on a currently
- // unsubmitted patch to the Linux kernel (as of May 2021) and is thus
- // subject to change. Patch is here:
- // https://lore.kernel.org/linux-mm/20210205151631.43511-12-kirill.shutemov@linux.intel.com/
- int tag_bits = kTagBits;
- int tag_shift = kAddressTagShift;
- if (!internal_iserror(
- internal_prctl(PR_SET_TAGGED_ADDR_CTRL, PR_TAGGED_ADDR_ENABLE,
- reinterpret_cast<unsigned long>(&tag_bits),
- reinterpret_cast<unsigned long>(&tag_shift), 0))) {
- CHECK_EQ(tag_bits, kTagBits);
- CHECK_EQ(tag_shift, kAddressTagShift);
- return;
- }
-# endif // defined(__x86_64__) && !defined(HWASAN_ALIASING_MODE)
- if (flags()->fail_without_syscall_abi) {
- Printf(
- "FATAL: HWAddressSanitizer failed to enable tagged address syscall "
- "ABI.\nSuggest check `sysctl abi.tagged_addr_disabled` "
- "configuration.\n");
- Die();
- }
- }
-#undef PR_SET_TAGGED_ADDR_CTRL
-#undef PR_GET_TAGGED_ADDR_CTRL
-#undef PR_TAGGED_ADDR_ENABLE
+ if (EnableTaggingAbi())
+ return;
+
+# if SANITIZER_ANDROID
+ MaybeDieIfNoTaggingAbi(
+ "HWAddressSanitizer failed to enable tagged address syscall ABI.\n"
+ "Check the `sysctl abi.tagged_addr_disabled` configuration.");
+# else
+ MaybeDieIfNoTaggingAbi(
+ "HWAddressSanitizer failed to enable tagged address syscall ABI.\n");
+# endif
}
bool InitShadow() {
CHECK(GetTagFromPointer(p) == 0);
# endif
- return p >= kHighMemStart || (p >= kLowMemStart && p <= kLowMemEnd);
+ return (p >= kHighMemStart && p <= kHighMemEnd) ||
+ (p >= kLowMemStart && p <= kLowMemEnd);
}
-void InstallAtExitHandler() {
- atexit(HwasanAtExit);
-}
+void InstallAtExitHandler() { atexit(HwasanAtExit); }
// ---------------------- TSD ---------------- {{{1
extern "C" void __hwasan_thread_enter() {
- hwasanThreadList().CreateCurrentThread()->InitRandomState();
+ hwasanThreadList().CreateCurrentThread()->EnsureRandomStateInited();
}
extern "C" void __hwasan_thread_exit() {
hwasanThreadList().ReleaseThread(t);
}
-#if HWASAN_WITH_INTERCEPTORS
+# if HWASAN_WITH_INTERCEPTORS
static pthread_key_t tsd_key;
static bool tsd_key_inited = false;
tsd_key_inited = true;
CHECK_EQ(0, pthread_key_create(&tsd_key, HwasanTSDDtor));
}
-#else
+# else
void HwasanTSDInit() {}
void HwasanTSDThreadInit() {}
-#endif
+# endif
-#if SANITIZER_ANDROID
-uptr *GetCurrentThreadLongPtr() {
- return (uptr *)get_android_tls_ptr();
-}
-#else
-uptr *GetCurrentThreadLongPtr() {
- return &__hwasan_tls;
-}
-#endif
+# if SANITIZER_ANDROID
+uptr *GetCurrentThreadLongPtr() { return (uptr *)get_android_tls_ptr(); }
+# else
+uptr *GetCurrentThreadLongPtr() { return &__hwasan_tls; }
+# endif
-#if SANITIZER_ANDROID
+# if SANITIZER_ANDROID
void AndroidTestTlsSlot() {
uptr kMagicValue = 0x010203040A0B0C0D;
uptr *tls_ptr = GetCurrentThreadLongPtr();
}
*tls_ptr = old_value;
}
-#else
+# else
void AndroidTestTlsSlot() {}
-#endif
+# endif
static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) {
// Access type is passed in a platform dependent way (see below) and encoded
// recoverable. Valid values of Y are 0 to 4, which are interpreted as
// log2(access_size), and 0xF, which means that access size is passed via
// platform dependent register (see below).
-#if defined(__aarch64__)
+# if defined(__aarch64__)
// Access type is encoded in BRK immediate as 0x900 + 0xXY. For Y == 0xF,
// access size is stored in X1 register. Access address is always in X0
// register.
uptr pc = (uptr)info->si_addr;
const unsigned code = ((*(u32 *)pc) >> 5) & 0xffff;
if ((code & 0xff00) != 0x900)
- return AccessInfo{}; // Not ours.
+ return AccessInfo{}; // Not ours.
const bool is_store = code & 0x10;
const bool recover = code & 0x20;
const uptr addr = uc->uc_mcontext.regs[0];
const unsigned size_log = code & 0xf;
if (size_log > 4 && size_log != 0xf)
- return AccessInfo{}; // Not ours.
+ return AccessInfo{}; // Not ours.
const uptr size = size_log == 0xf ? uc->uc_mcontext.regs[1] : 1U << size_log;
-#elif defined(__x86_64__)
+# elif defined(__x86_64__)
// Access type is encoded in the instruction following INT3 as
// NOP DWORD ptr [EAX + 0x40 + 0xXY]. For Y == 0xF, access size is stored in
// RSI register. Access address is always in RDI register.
uptr pc = (uptr)uc->uc_mcontext.gregs[REG_RIP];
- uint8_t *nop = (uint8_t*)pc;
- if (*nop != 0x0f || *(nop + 1) != 0x1f || *(nop + 2) != 0x40 ||
+ uint8_t *nop = (uint8_t *)pc;
+ if (*nop != 0x0f || *(nop + 1) != 0x1f || *(nop + 2) != 0x40 ||
*(nop + 3) < 0x40)
- return AccessInfo{}; // Not ours.
+ return AccessInfo{}; // Not ours.
const unsigned code = *(nop + 3);
const bool is_store = code & 0x10;
const uptr addr = uc->uc_mcontext.gregs[REG_RDI];
const unsigned size_log = code & 0xf;
if (size_log > 4 && size_log != 0xf)
- return AccessInfo{}; // Not ours.
+ return AccessInfo{}; // Not ours.
const uptr size =
size_log == 0xf ? uc->uc_mcontext.gregs[REG_RSI] : 1U << size_log;
-#else
-# error Unsupported architecture
-#endif
+# elif SANITIZER_RISCV64
+ // Access type is encoded in the instruction following EBREAK as
+ // ADDI x0, x0, [0x40 + 0xXY]. For Y == 0xF, access size is stored in
+ // X11 register. Access address is always in X10 register.
+ uptr pc = (uptr)uc->uc_mcontext.__gregs[REG_PC];
+ uint8_t byte1 = *((u8 *)(pc + 0));
+ uint8_t byte2 = *((u8 *)(pc + 1));
+ uint8_t byte3 = *((u8 *)(pc + 2));
+ uint8_t byte4 = *((u8 *)(pc + 3));
+ uint32_t ebreak = (byte1 | (byte2 << 8) | (byte3 << 16) | (byte4 << 24));
+ bool isFaultShort = false;
+ bool isEbreak = (ebreak == 0x100073);
+ bool isShortEbreak = false;
+# if defined(__riscv_compressed)
+ isFaultShort = ((ebreak & 0x3) != 0x3);
+ isShortEbreak = ((ebreak & 0xffff) == 0x9002);
+# endif
+ // faulted insn is not ebreak, not our case
+ if (!(isEbreak || isShortEbreak))
+ return AccessInfo{};
+ // advance pc to point after ebreak and reconstruct addi instruction
+ pc += isFaultShort ? 2 : 4;
+ byte1 = *((u8 *)(pc + 0));
+ byte2 = *((u8 *)(pc + 1));
+ byte3 = *((u8 *)(pc + 2));
+ byte4 = *((u8 *)(pc + 3));
+ // reconstruct instruction
+ uint32_t instr = (byte1 | (byte2 << 8) | (byte3 << 16) | (byte4 << 24));
+ // check if this is really 32 bit instruction
+ // code is encoded in top 12 bits, since instruction is supposed to be with
+ // imm
+ const unsigned code = (instr >> 20) & 0xffff;
+ const uptr addr = uc->uc_mcontext.__gregs[10];
+ const bool is_store = code & 0x10;
+ const bool recover = code & 0x20;
+ const unsigned size_log = code & 0xf;
+ if (size_log > 4 && size_log != 0xf)
+ return AccessInfo{}; // Not our case
+ const uptr size =
+ size_log == 0xf ? uc->uc_mcontext.__gregs[11] : 1U << size_log;
+
+# else
+# error Unsupported architecture
+# endif
return AccessInfo{addr, size, is_store, !is_store, recover};
}
SignalContext sig{info, uc};
HandleTagMismatch(ai, StackTrace::GetNextInstructionPc(sig.pc), sig.bp, uc);
-#if defined(__aarch64__)
+# if defined(__aarch64__)
uc->uc_mcontext.pc += 4;
-#elif defined(__x86_64__)
-#else
-# error Unsupported architecture
-#endif
+# elif defined(__x86_64__)
+# elif SANITIZER_RISCV64
+ // pc points to EBREAK which is 2 bytes long
+ uint8_t *exception_source = (uint8_t *)(uc->uc_mcontext.__gregs[REG_PC]);
+ uint8_t byte1 = (uint8_t)(*(exception_source + 0));
+ uint8_t byte2 = (uint8_t)(*(exception_source + 1));
+ uint8_t byte3 = (uint8_t)(*(exception_source + 2));
+ uint8_t byte4 = (uint8_t)(*(exception_source + 3));
+ uint32_t faulted = (byte1 | (byte2 << 8) | (byte3 << 16) | (byte4 << 24));
+ bool isFaultShort = false;
+# if defined(__riscv_compressed)
+ isFaultShort = ((faulted & 0x3) != 0x3);
+# endif
+ uc->uc_mcontext.__gregs[REG_PC] += isFaultShort ? 2 : 4;
+# else
+# error Unsupported architecture
+# endif
return true;
}
void HwasanOnDeadlySignal(int signo, void *info, void *context) {
// Probably a tag mismatch.
if (signo == SIGTRAP)
- if (HwasanOnSIGTRAP(signo, (siginfo_t *)info, (ucontext_t*)context))
+ if (HwasanOnSIGTRAP(signo, (siginfo_t *)info, (ucontext_t *)context))
return;
HandleDeadlySignal(info, context, GetTid(), &OnStackUnwind, nullptr);
return AddTagToPointer(p, tag);
}
-} // namespace __hwasan
+void HwasanInstallAtForkHandler() {
+ auto before = []() {
+ HwasanAllocatorLock();
+ StackDepotLockAll();
+ };
+ auto after = []() {
+ StackDepotUnlockAll();
+ HwasanAllocatorUnlock();
+ };
+ pthread_atfork(before, after, after);
+}
+
+void InstallAtExitCheckLeaks() {
+ if (CAN_SANITIZE_LEAKS) {
+ if (common_flags()->detect_leaks && common_flags()->leak_check_at_exit) {
+ if (flags()->halt_on_error)
+ Atexit(__lsan::DoLeakCheck);
+ else
+ Atexit(__lsan::DoRecoverableLeakCheckVoid);
+ }
+ }
+}
+
+} // namespace __hwasan
-#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
+#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
reinterpret_cast<uptr>(to), size);
CheckAddressSized<ErrorAction::Recover, AccessType::Load>(
reinterpret_cast<uptr>(from), size);
- return memmove(UntagPtr(to), UntagPtr(from), size);
+ return memmove(to, from, size);
}
#if HWASAN_REPLACE_OPERATORS_NEW_AND_DELETE
// TODO(alekseys): throw std::bad_alloc instead of dying on OOM.
-#define OPERATOR_NEW_BODY(nothrow) \
- GET_MALLOC_STACK_TRACE; \
- void *res = hwasan_malloc(size, &stack);\
- if (!nothrow && UNLIKELY(!res)) ReportOutOfMemory(size, &stack);\
- return res
-#define OPERATOR_NEW_ALIGN_BODY(nothrow) \
- GET_MALLOC_STACK_TRACE; \
- void *res = hwasan_aligned_alloc(static_cast<uptr>(align), size, &stack); \
- if (!nothrow && UNLIKELY(!res)) \
- ReportOutOfMemory(size, &stack); \
- return res
-
-#define OPERATOR_DELETE_BODY \
- GET_MALLOC_STACK_TRACE; \
- if (ptr) hwasan_free(ptr, &stack)
+# define OPERATOR_NEW_BODY(nothrow) \
+ GET_MALLOC_STACK_TRACE; \
+ void *res = hwasan_malloc(size, &stack); \
+ if (!nothrow && UNLIKELY(!res)) \
+ ReportOutOfMemory(size, &stack); \
+ return res
+# define OPERATOR_NEW_ALIGN_BODY(nothrow) \
+ GET_MALLOC_STACK_TRACE; \
+ void *res = hwasan_memalign(static_cast<uptr>(align), size, &stack); \
+ if (!nothrow && UNLIKELY(!res)) \
+ ReportOutOfMemory(size, &stack); \
+ return res
+
+# define OPERATOR_DELETE_BODY \
+ GET_MALLOC_STACK_TRACE; \
+ if (ptr) \
+ hwasan_free(ptr, &stack)
#elif defined(__ANDROID__)
// since we previously released a runtime that intercepted these functions,
// removing the interceptors would break ABI. Therefore we simply forward to
// malloc and free.
-#define OPERATOR_NEW_BODY(nothrow) return malloc(size)
-#define OPERATOR_DELETE_BODY free(ptr)
+# define OPERATOR_NEW_BODY(nothrow) return malloc(size)
+# define OPERATOR_DELETE_BODY free(ptr)
#endif
// Fake std::nothrow_t to avoid including <new>.
namespace std {
- struct nothrow_t {};
+struct nothrow_t {};
} // namespace std
-
-
-INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-void *operator new(size_t size) { OPERATOR_NEW_BODY(false /*nothrow*/); }
-INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-void *operator new[](size_t size) { OPERATOR_NEW_BODY(false /*nothrow*/); }
-INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-void *operator new(size_t size, std::nothrow_t const&) {
+INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new(size_t size) {
+ OPERATOR_NEW_BODY(false /*nothrow*/);
+}
+INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new[](
+ size_t size) {
+ OPERATOR_NEW_BODY(false /*nothrow*/);
+}
+INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new(
+ size_t size, std::nothrow_t const &) {
OPERATOR_NEW_BODY(true /*nothrow*/);
}
-INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-void *operator new[](size_t size, std::nothrow_t const&) {
+INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new[](
+ size_t size, std::nothrow_t const &) {
OPERATOR_NEW_BODY(true /*nothrow*/);
}
-INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete(void *ptr)
- NOEXCEPT {
+INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete(
+ void *ptr) NOEXCEPT {
OPERATOR_DELETE_BODY;
}
INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[](
}
} // namespace __hwasan
+
+// --- Implementation of LSan-specific functions --- {{{1
+namespace __lsan {
+bool WordIsPoisoned(uptr addr) {
+ // Fixme: implement actual tag checking.
+ return false;
+}
+} // namespace __lsan
--- /dev/null
+//===-- hwasan_preinit.cpp ------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer, an address sanity checker.
+//
+// Call __hwasan_init at the very early stage of process startup.
+//===----------------------------------------------------------------------===//
+#include "hwasan_interface_internal.h"
+#include "sanitizer_common/sanitizer_internal_defs.h"
+
+#if SANITIZER_CAN_USE_PREINIT_ARRAY
+// The symbol is called __local_hwasan_preinit, because it's not intended to
+// be exported.
+// This code linked into the main executable when -fsanitize=hwaddress is in
+// the link flags. It can only use exported interface functions.
+__attribute__((section(".preinit_array"), used)) static void (
+ *__local_hwasan_preinit)(void) = __hwasan_init;
+#endif
class ScopedReport {
public:
ScopedReport(bool fatal = false) : error_message_(1), fatal(fatal) {
- BlockingMutexLock lock(&error_message_lock_);
+ Lock lock(&error_message_lock_);
error_message_ptr_ = fatal ? &error_message_ : nullptr;
++hwasan_report_count;
}
~ScopedReport() {
void (*report_cb)(const char *);
{
- BlockingMutexLock lock(&error_message_lock_);
+ Lock lock(&error_message_lock_);
report_cb = error_report_callback_;
error_message_ptr_ = nullptr;
}
}
static void MaybeAppendToErrorMessage(const char *msg) {
- BlockingMutexLock lock(&error_message_lock_);
+ Lock lock(&error_message_lock_);
if (!error_message_ptr_)
return;
uptr len = internal_strlen(msg);
}
static void SetErrorReportCallback(void (*callback)(const char *)) {
- BlockingMutexLock lock(&error_message_lock_);
+ Lock lock(&error_message_lock_);
error_report_callback_ = callback;
}
bool fatal;
static InternalMmapVector<char> *error_message_ptr_;
- static BlockingMutex error_message_lock_;
+ static Mutex error_message_lock_;
static void (*error_report_callback_)(const char *);
};
InternalMmapVector<char> *ScopedReport::error_message_ptr_;
-BlockingMutex ScopedReport::error_message_lock_;
+Mutex ScopedReport::error_message_lock_;
void (*ScopedReport::error_report_callback_)(const char *);
// If there is an active ScopedReport, append to its error message.
return res;
}
+static void MaybePrintAndroidHelpUrl() {
+#if SANITIZER_ANDROID
+ Printf(
+ "Learn more about HWASan reports: "
+ "https://source.android.com/docs/security/test/memory-safety/"
+ "hwasan-reports\n");
+#endif
+}
+
// A RAII object that holds a copy of the current thread stack ring buffer.
// The actual stack buffer may change while we are iterating over it (for
// example, Printf may call syslog() which can itself be built with hwasan).
whence = "inside";
} else if (candidate == left) {
offset = untagged_addr - chunk.End();
- whence = "to the right of";
+ whence = "after";
} else {
offset = chunk.Beg() - untagged_addr;
- whence = "to the left of";
+ whence = "before";
}
Printf("%s", d.Error());
Printf("\nCause: heap-buffer-overflow\n");
Printf("%s", d.Default());
Printf("%s", d.Location());
- Printf("%p is located %zd bytes %s %zd-byte region [%p,%p)\n",
+ Printf("%p is located %zd bytes %s a %zd-byte region [%p,%p)\n",
untagged_addr, offset, whence, chunk.UsedSize(), chunk.Beg(),
chunk.End());
Printf("%s", d.Allocation());
Printf("%s", d.Location());
if (sym->SymbolizeData(mem, &info) && info.start) {
Printf(
- "%p is located %zd bytes to the %s of %zd-byte global variable "
+ "%p is located %zd bytes %s a %zd-byte global variable "
"%s [%p,%p) in %s\n",
untagged_addr,
candidate == left ? untagged_addr - (info.start + info.size)
: info.start - untagged_addr,
- candidate == left ? "right" : "left", info.size, info.name,
+ candidate == left ? "after" : "before", info.size, info.name,
info.start, info.start + info.size, module_name);
} else {
uptr size = GetGlobalSizeFromDescriptor(mem);
if (size == 0)
// We couldn't find the size of the global from the descriptors.
- Printf("%p is located to the %s of a global variable in (%s+0x%x)\n",
- untagged_addr, candidate == left ? "right" : "left", module_name,
- module_address);
+ Printf(
+ "%p is located %s a global variable in "
+ "\n #0 0x%x (%s+0x%x)\n",
+ untagged_addr, candidate == left ? "after" : "before", mem,
+ module_name, module_address);
else
Printf(
- "%p is located to the %s of a %zd-byte global variable in "
- "(%s+0x%x)\n",
- untagged_addr, candidate == left ? "right" : "left", size,
+ "%p is located %s a %zd-byte global variable in "
+ "\n #0 0x%x (%s+0x%x)\n",
+ untagged_addr, candidate == left ? "after" : "before", size, mem,
module_name, module_address);
}
Printf("%s", d.Default());
int num_descriptions_printed = 0;
uptr untagged_addr = UntagAddr(tagged_addr);
+ if (MemIsShadow(untagged_addr)) {
+ Printf("%s%p is HWAsan shadow memory.\n%s", d.Location(), untagged_addr,
+ d.Default());
+ return;
+ }
+
// Print some very basic information about the address, if it's a heap.
HwasanChunkView chunk = FindHeapChunkByAddress(untagged_addr);
if (uptr beg = chunk.Beg()) {
Printf("%s", d.Error());
Printf("\nCause: use-after-free\n");
Printf("%s", d.Location());
- Printf("%p is located %zd bytes inside of %zd-byte region [%p,%p)\n",
+ Printf("%p is located %zd bytes inside a %zd-byte region [%p,%p)\n",
untagged_addr, untagged_addr - UntagAddr(har.tagged_addr),
har.requested_size, UntagAddr(har.tagged_addr),
UntagAddr(har.tagged_addr) + har.requested_size);
InternalScopedString s;
for (tag_t *row = beg_row; row < end_row; row += row_len) {
s.append("%s", row == center_row_beg ? "=>" : " ");
- s.append("%p:", row);
+ s.append("%p:", (void *)ShadowToMem(reinterpret_cast<uptr>(row)));
for (uptr i = 0; i < row_len; i++) {
s.append("%s", row + i == tag_ptr ? "[" : " ");
print_tag(s, &row[i]);
"description of short granule tags\n");
}
+uptr GetTopPc(StackTrace *stack) {
+ return stack->size ? StackTrace::GetPreviousInstructionPc(stack->trace[0])
+ : 0;
+}
+
void ReportInvalidFree(StackTrace *stack, uptr tagged_addr) {
ScopedReport R(flags()->halt_on_error);
uptr untagged_addr = UntagAddr(tagged_addr);
tag_t ptr_tag = GetTagFromPointer(tagged_addr);
- tag_t *tag_ptr = reinterpret_cast<tag_t*>(MemToShadow(untagged_addr));
- tag_t mem_tag = *tag_ptr;
+ tag_t *tag_ptr = nullptr;
+ tag_t mem_tag = 0;
+ if (MemIsApp(untagged_addr)) {
+ tag_ptr = reinterpret_cast<tag_t *>(MemToShadow(untagged_addr));
+ if (MemIsShadow(reinterpret_cast<uptr>(tag_ptr)))
+ mem_tag = *tag_ptr;
+ else
+ tag_ptr = nullptr;
+ }
Decorator d;
Printf("%s", d.Error());
- uptr pc = stack->size ? stack->trace[0] : 0;
+ uptr pc = GetTopPc(stack);
const char *bug_type = "invalid-free";
- Report("ERROR: %s: %s on address %p at pc %p\n", SanitizerToolName, bug_type,
- untagged_addr, pc);
+ const Thread *thread = GetCurrentThread();
+ if (thread) {
+ Report("ERROR: %s: %s on address %p at pc %p on thread T%zd\n",
+ SanitizerToolName, bug_type, untagged_addr, pc, thread->unique_id());
+ } else {
+ Report("ERROR: %s: %s on address %p at pc %p on unknown thread\n",
+ SanitizerToolName, bug_type, untagged_addr, pc);
+ }
Printf("%s", d.Access());
- Printf("tags: %02x/%02x (ptr/mem)\n", ptr_tag, mem_tag);
+ if (tag_ptr)
+ Printf("tags: %02x/%02x (ptr/mem)\n", ptr_tag, mem_tag);
Printf("%s", d.Default());
stack->Print();
PrintAddressDescription(tagged_addr, 0, nullptr);
- PrintTagsAroundAddr(tag_ptr);
+ if (tag_ptr)
+ PrintTagsAroundAddr(tag_ptr);
+ MaybePrintAndroidHelpUrl();
ReportErrorSummary(bug_type, stack);
}
void ReportTailOverwritten(StackTrace *stack, uptr tagged_addr, uptr orig_size,
const u8 *expected) {
uptr tail_size = kShadowAlignment - (orig_size % kShadowAlignment);
+ u8 actual_expected[kShadowAlignment];
+ internal_memcpy(actual_expected, expected, tail_size);
+ tag_t ptr_tag = GetTagFromPointer(tagged_addr);
+ // Short granule is stashed in the last byte of the magic string. To avoid
+ // confusion, make the expected magic string contain the short granule tag.
+ if (orig_size % kShadowAlignment != 0) {
+ actual_expected[tail_size - 1] = ptr_tag;
+ }
+
ScopedReport R(flags()->halt_on_error);
Decorator d;
uptr untagged_addr = UntagAddr(tagged_addr);
s.append("Expected: ");
for (uptr i = 0; i < kShadowAlignment - tail_size; i++)
s.append(".. ");
- for (uptr i = 0; i < tail_size; i++)
- s.append("%02x ", expected[i]);
+ for (uptr i = 0; i < tail_size; i++) s.append("%02x ", actual_expected[i]);
s.append("\n");
s.append(" ");
for (uptr i = 0; i < kShadowAlignment - tail_size; i++)
s.append(" ");
for (uptr i = 0; i < tail_size; i++)
- s.append("%s ", expected[i] != tail[i] ? "^^" : " ");
+ s.append("%s ", actual_expected[i] != tail[i] ? "^^" : " ");
s.append("\nThis error occurs when a buffer overflow overwrites memory\n"
- "to the right of a heap object, but within the %zd-byte granule, e.g.\n"
+ "after a heap object, but within the %zd-byte granule, e.g.\n"
" char *x = new char[20];\n"
" x[25] = 42;\n"
"%s does not detect such bugs in uninstrumented code at the time of write,"
tag_t *tag_ptr = reinterpret_cast<tag_t*>(MemToShadow(untagged_addr));
PrintTagsAroundAddr(tag_ptr);
+ MaybePrintAndroidHelpUrl();
ReportErrorSummary(bug_type, stack);
}
GetCurrentThread()->stack_allocations());
Decorator d;
- Printf("%s", d.Error());
uptr untagged_addr = UntagAddr(tagged_addr);
// TODO: when possible, try to print heap-use-after-free, etc.
const char *bug_type = "tag-mismatch";
- uptr pc = stack->size ? stack->trace[0] : 0;
+ uptr pc = GetTopPc(stack);
+ Printf("%s", d.Error());
Report("ERROR: %s: %s on address %p at pc %p\n", SanitizerToolName, bug_type,
untagged_addr, pc);
tag_t mem_tag = *tag_ptr;
Printf("%s", d.Access());
- Printf("%s of size %zu at %p tags: %02x/%02x (ptr/mem) in thread T%zd\n",
- is_store ? "WRITE" : "READ", access_size, untagged_addr, ptr_tag,
- mem_tag, t->unique_id());
+ if (mem_tag && mem_tag < kShadowAlignment) {
+ tag_t *granule_ptr = reinterpret_cast<tag_t *>((untagged_addr + offset) &
+ ~(kShadowAlignment - 1));
+ // If offset is 0, (untagged_addr + offset) is not aligned to granules.
+ // This is the offset of the leftmost accessed byte within the bad granule.
+ u8 in_granule_offset = (untagged_addr + offset) & (kShadowAlignment - 1);
+ tag_t short_tag = granule_ptr[kShadowAlignment - 1];
+ // The first mismatch was a short granule that matched the ptr_tag.
+ if (short_tag == ptr_tag) {
+ // If the access starts after the end of the short granule, then the first
+ // bad byte is the first byte of the access; otherwise it is the first
+ // byte past the end of the short granule
+ if (mem_tag > in_granule_offset) {
+ offset += mem_tag - in_granule_offset;
+ }
+ }
+ Printf(
+ "%s of size %zu at %p tags: %02x/%02x(%02x) (ptr/mem) in thread T%zd\n",
+ is_store ? "WRITE" : "READ", access_size, untagged_addr, ptr_tag,
+ mem_tag, short_tag, t->unique_id());
+ } else {
+ Printf("%s of size %zu at %p tags: %02x/%02x (ptr/mem) in thread T%zd\n",
+ is_store ? "WRITE" : "READ", access_size, untagged_addr, ptr_tag,
+ mem_tag, t->unique_id());
+ }
if (offset != 0)
- Printf("Invalid access starting at offset [%zu, %zu)\n", offset,
- Min(access_size, static_cast<uptr>(offset) + (1 << kShadowScale)));
+ Printf("Invalid access starting at offset %zu\n", offset);
Printf("%s", d.Default());
stack->Print();
if (registers_frame)
ReportRegisters(registers_frame, pc);
+ MaybePrintAndroidHelpUrl();
ReportErrorSummary(bug_type, stack);
}
// See the frame breakdown defined in __hwasan_tag_mismatch (from
-// hwasan_tag_mismatch_aarch64.S).
+// hwasan_tag_mismatch_{aarch64,riscv64}.S).
void ReportRegisters(uptr *frame, uptr pc) {
Printf("Registers where the failure occurred (pc %p):\n", pc);
// reduce the amount of logcat error messages printed. Each Printf() will
// result in a new logcat line, irrespective of whether a newline is present,
// and so we wish to reduce the number of Printf() calls we have to make.
+#if defined(__aarch64__)
Printf(" x0 %016llx x1 %016llx x2 %016llx x3 %016llx\n",
frame[0], frame[1], frame[2], frame[3]);
+#elif SANITIZER_RISCV64
+ Printf(" sp %016llx x1 %016llx x2 %016llx x3 %016llx\n",
+ reinterpret_cast<u8 *>(frame) + 256, frame[1], frame[2], frame[3]);
+#endif
Printf(" x4 %016llx x5 %016llx x6 %016llx x7 %016llx\n",
frame[4], frame[5], frame[6], frame[7]);
Printf(" x8 %016llx x9 %016llx x10 %016llx x11 %016llx\n",
frame[24], frame[25], frame[26], frame[27]);
// hwasan_check* reduces the stack pointer by 256, then __hwasan_tag_mismatch
// passes it to this function.
+#if defined(__aarch64__)
Printf(" x28 %016llx x29 %016llx x30 %016llx sp %016llx\n", frame[28],
frame[29], frame[30], reinterpret_cast<u8 *>(frame) + 256);
+#elif SANITIZER_RISCV64
+ Printf(" x28 %016llx x29 %016llx x30 %016llx x31 %016llx\n", frame[28],
+ frame[29], frame[30], frame[31]);
+#else
+#endif
}
} // namespace __hwasan
--- /dev/null
+//===-- hwasan_setjmp_aarch64.S -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+// HWAddressSanitizer runtime.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_asm.h"
+#include "builtins/assembly.h"
+
+#if HWASAN_WITH_INTERCEPTORS && defined(__aarch64__)
+#include "sanitizer_common/sanitizer_platform.h"
+
+// We want to save the context of the calling function.
+// That requires
+// 1) No modification of the link register by this function.
+// 2) No modification of the stack pointer by this function.
+// 3) (no modification of any other saved register, but that's not really going
+// to occur, and hence isn't as much of a worry).
+//
+// There's essentially no way to ensure that the compiler will not modify the
+// stack pointer when compiling a C function.
+// Hence we have to write this function in assembly.
+
+.section .text
+.file "hwasan_setjmp_aarch64.S"
+
+.global __interceptor_setjmp
+ASM_TYPE_FUNCTION(__interceptor_setjmp)
+__interceptor_setjmp:
+ CFI_STARTPROC
+ BTI_C
+ mov x1, #0
+ b __interceptor_sigsetjmp
+ CFI_ENDPROC
+ASM_SIZE(__interceptor_setjmp)
+
+#if SANITIZER_ANDROID
+// Bionic also defines a function `setjmp` that calls `sigsetjmp` saving the
+// current signal.
+.global __interceptor_setjmp_bionic
+ASM_TYPE_FUNCTION(__interceptor_setjmp_bionic)
+__interceptor_setjmp_bionic:
+ CFI_STARTPROC
+ BTI_C
+ mov x1, #1
+ b __interceptor_sigsetjmp
+ CFI_ENDPROC
+ASM_SIZE(__interceptor_setjmp_bionic)
+#endif
+
+.global __interceptor_sigsetjmp
+ASM_TYPE_FUNCTION(__interceptor_sigsetjmp)
+__interceptor_sigsetjmp:
+ CFI_STARTPROC
+ BTI_C
+ stp x19, x20, [x0, #0<<3]
+ stp x21, x22, [x0, #2<<3]
+ stp x23, x24, [x0, #4<<3]
+ stp x25, x26, [x0, #6<<3]
+ stp x27, x28, [x0, #8<<3]
+ stp x29, x30, [x0, #10<<3]
+ stp d8, d9, [x0, #14<<3]
+ stp d10, d11, [x0, #16<<3]
+ stp d12, d13, [x0, #18<<3]
+ stp d14, d15, [x0, #20<<3]
+ mov x2, sp
+ str x2, [x0, #13<<3]
+ // We always have the second argument to __sigjmp_save (savemask) set, since
+ // the _setjmp function above has set it for us as `false`.
+ // This function is defined in hwasan_interceptors.cc
+ b __sigjmp_save
+ CFI_ENDPROC
+ASM_SIZE(__interceptor_sigsetjmp)
+
+
+.macro WEAK_ALIAS first second
+ .weak \second
+ .equ \second\(), \first
+.endm
+
+#if SANITIZER_ANDROID
+WEAK_ALIAS __interceptor_sigsetjmp, sigsetjmp
+WEAK_ALIAS __interceptor_setjmp_bionic, setjmp
+#else
+WEAK_ALIAS __interceptor_sigsetjmp, __sigsetjmp
+#endif
+
+WEAK_ALIAS __interceptor_setjmp, _setjmp
+#endif
+
+// We do not need executable stack.
+NO_EXEC_STACK_DIRECTIVE
+
+GNU_PROPERTY_BTI_PAC
--- /dev/null
+//===-- hwasan_setjmp_riscv64.S -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+// setjmp interceptor for risc-v.
+// HWAddressSanitizer runtime.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_asm.h"
+#include "builtins/assembly.h"
+
+#if HWASAN_WITH_INTERCEPTORS && defined(__riscv) && (__riscv_xlen == 64)
+#include "sanitizer_common/sanitizer_platform.h"
+
+// We want to save the context of the calling function.
+// That requires
+// 1) No modification of the link register by this function.
+// 2) No modification of the stack pointer by this function.
+// 3) (no modification of any other saved register, but that's not really going
+// to occur, and hence isn't as much of a worry).
+//
+// There's essentially no way to ensure that the compiler will not modify the
+// stack pointer when compiling a C function.
+// Hence we have to write this function in assembly.
+
+.section .text
+.file "hwasan_setjmp_riscv64.S"
+
+.global __interceptor_setjmp
+ASM_TYPE_FUNCTION(__interceptor_setjmp)
+__interceptor_setjmp:
+ CFI_STARTPROC
+ addi x11, x0, 0
+ tail __interceptor_sigsetjmp
+ CFI_ENDPROC
+ASM_SIZE(__interceptor_setjmp)
+
+.global __interceptor_sigsetjmp
+ASM_TYPE_FUNCTION(__interceptor_sigsetjmp)
+__interceptor_sigsetjmp:
+ CFI_STARTPROC
+ sd ra, 0<<3(x10)
+ sd s0, 1<<3(x10)
+ sd s1, 2<<3(x10)
+ sd s2, 3<<3(x10)
+ sd s3, 4<<3(x10)
+ sd s4, 5<<3(x10)
+ sd s5, 6<<3(x10)
+ sd s6, 7<<3(x10)
+ sd s7, 8<<3(x10)
+ sd s8, 9<<3(x10)
+ sd s9, 10<<3(x10)
+ sd s10, 11<<3(x10)
+ sd s11, 12<<3(x10)
+ sd sp, 13<<3(x10)
+#if __riscv_float_abi_double
+ fsd fs0, 14<<3(x10)
+ fsd fs1, 15<<3(x10)
+ fsd fs2, 16<<3(x10)
+ fsd fs3, 17<<3(x10)
+ fsd fs4, 18<<3(x10)
+ fsd fs5, 19<<3(x10)
+ fsd fs6, 20<<3(x10)
+ fsd fs7, 21<<3(x10)
+ fsd fs8, 22<<3(x10)
+ fsd fs9, 23<<3(x10)
+ fsd fs10, 24<<3(x10)
+ fsd fs11, 25<<3(x10)
+#elif __riscv_float_abi_soft
+#else
+# error "Unsupported case"
+#endif
+ // We always have the second argument to __sigjmp_save (savemask) set, since
+ // the _setjmp function above has set it for us as `false`.
+ // This function is defined in hwasan_interceptors.cc
+ tail __sigjmp_save
+ CFI_ENDPROC
+ASM_SIZE(__interceptor_sigsetjmp)
+
+
+.macro WEAK_ALIAS first second
+ .weak \second
+ .equ \second\(), \first
+.endm
+
+WEAK_ALIAS __interceptor_sigsetjmp, __sigsetjmp
+
+WEAK_ALIAS __interceptor_setjmp, _setjmp
+#endif
+
+// We do not need executable stack.
+NO_EXEC_STACK_DIRECTIVE
--- /dev/null
+//===-- hwasan_setjmp_x86_64.S --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// setjmp interceptor for x86_64.
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_asm.h"
+
+#if HWASAN_WITH_INTERCEPTORS && defined(__x86_64__)
+#include "sanitizer_common/sanitizer_platform.h"
+
+// We want to save the context of the calling function.
+// That requires
+// 1) No modification of the return address by this function.
+// 2) No modification of the stack pointer by this function.
+// 3) (no modification of any other saved register, but that's not really going
+// to occur, and hence isn't as much of a worry).
+//
+// There's essentially no way to ensure that the compiler will not modify the
+// stack pointer when compiling a C function.
+// Hence we have to write this function in assembly.
+//
+// TODO: Handle Intel CET.
+
+.section .text
+.file "hwasan_setjmp_x86_64.S"
+
+.global __interceptor_setjmp
+ASM_TYPE_FUNCTION(__interceptor_setjmp)
+__interceptor_setjmp:
+ CFI_STARTPROC
+ _CET_ENDBR
+ xorl %esi, %esi
+ jmp __interceptor_sigsetjmp
+ CFI_ENDPROC
+ASM_SIZE(__interceptor_setjmp)
+
+.global __interceptor_sigsetjmp
+ASM_TYPE_FUNCTION(__interceptor_sigsetjmp)
+__interceptor_sigsetjmp:
+ CFI_STARTPROC
+ _CET_ENDBR
+
+ // Save callee save registers.
+ mov %rbx, (0*8)(%rdi)
+ mov %rbp, (1*8)(%rdi)
+ mov %r12, (2*8)(%rdi)
+ mov %r13, (3*8)(%rdi)
+ mov %r14, (4*8)(%rdi)
+ mov %r15, (5*8)(%rdi)
+
+ // Save SP as it was in caller's frame.
+ lea 8(%rsp), %rdx
+ mov %rdx, (6*8)(%rdi)
+
+ // Save return address.
+ mov (%rsp), %rax
+ mov %rax, (7*8)(%rdi)
+
+ jmp __sigjmp_save
+
+ CFI_ENDPROC
+ASM_SIZE(__interceptor_sigsetjmp)
+
+
+.macro WEAK_ALIAS first second
+ .weak \second
+ .equ \second\(), \first
+.endm
+
+WEAK_ALIAS __interceptor_sigsetjmp, __sigsetjmp
+WEAK_ALIAS __interceptor_setjmp, _setjmp
+#endif
+
+// We do not need executable stack.
+NO_EXEC_STACK_DIRECTIVE
--- /dev/null
+#include "sanitizer_common/sanitizer_asm.h"
+
+// The content of this file is RISCV64-only:
+#if defined(__riscv) && (__riscv_xlen == 64)
+
+// The responsibility of the HWASan entry point in compiler-rt is to primarily
+// readjust the stack from the callee and save the current register values to
+// the stack.
+// This entry point function should be called from a __hwasan_check_* symbol.
+// These are generated during a lowering pass in the backend, and are found in
+// RISCVAsmPrinter::EmitHwasanMemaccessSymbols(). Please look there for
+// further information.
+// The __hwasan_check_* caller of this function should have expanded the stack
+// and saved the previous values of x10(arg0), x11(arg1), x1(ra), and x8(fp).
+// This function will "consume" these saved values and treats it as part of its
+// own stack frame. In this sense, the __hwasan_check_* callee and this function
+// "share" a stack frame. This allows us to omit having unwinding information
+// (.cfi_*) present in every __hwasan_check_* function, therefore reducing binary size.
+// This is particularly important as hwasan_check_* instances are duplicated in every
+// translation unit where HWASan is enabled.
+// This function calls HwasanTagMismatch to step back into the C++ code that
+// completes the stack unwinding and error printing. This function is is not
+// permitted to return.
+
+
+// | ... |
+// | ... |
+// | Previous stack frames... |
+// +=================================+
+// | ... |
+// | |
+// | Stack frame space for x12 - x31.|
+// | |
+// | ... |
+// +---------------------------------+ <-- [SP + 96]
+// | Saved x11(arg1), as |
+// | __hwasan_check_* clobbers it. |
+// +---------------------------------+ <-- [SP + 88]
+// | Saved x10(arg0), as |
+// | __hwasan_check_* clobbers it. |
+// +---------------------------------+ <-- [SP + 80]
+// | |
+// | Stack frame space for x9. |
+// +---------------------------------+ <-- [SP + 72]
+// | |
+// | Saved x8(fp), as |
+// | __hwasan_check_* clobbers it. |
+// +---------------------------------+ <-- [SP + 64]
+// | ... |
+// | |
+// | Stack frame space for x2 - x7. |
+// | |
+// | ... |
+// +---------------------------------+ <-- [SP + 16]
+// | Return address (x1) for caller |
+// | of __hwasan_check_*. |
+// +---------------------------------+ <-- [SP + 8]
+// | Reserved place for x0, possibly |
+// | junk, since we don't save it. |
+// +---------------------------------+ <-- [x2 / SP]
+
+// This function takes two arguments:
+// * x10/a0: The data address.
+// * x11/a1: The encoded access info for the failing access.
+
+.section .text
+.file "hwasan_tag_mismatch_riscv64.S"
+
+.global __hwasan_tag_mismatch_v2
+ASM_TYPE_FUNCTION(__hwasan_tag_mismatch_v2)
+__hwasan_tag_mismatch_v2:
+ CFI_STARTPROC
+
+ // Set the CFA to be the return address for caller of __hwasan_check_*. Note
+ // that we do not emit CFI predicates to describe the contents of this stack
+ // frame, as this proxy entry point should never be debugged. The contents
+ // are static and are handled by the unwinder after calling
+ // __hwasan_tag_mismatch. The frame pointer is already correctly setup
+ // by __hwasan_check_*.
+ addi fp, sp, 256
+ CFI_DEF_CFA(fp, 0)
+ CFI_OFFSET(ra, -248)
+ CFI_OFFSET(fp, -192)
+
+ // Save the rest of the registers into the preallocated space left by
+ // __hwasan_check.
+ sd x31, 248(sp)
+ sd x30, 240(sp)
+ sd x29, 232(sp)
+ sd x28, 224(sp)
+ sd x27, 216(sp)
+ sd x26, 208(sp)
+ sd x25, 200(sp)
+ sd x24, 192(sp)
+ sd x23, 184(sp)
+ sd x22, 176(sp)
+ sd x21, 168(sp)
+ sd x20, 160(sp)
+ sd x19, 152(sp)
+ sd x18, 144(sp)
+ sd x17, 136(sp)
+ sd x16, 128(sp)
+ sd x15, 120(sp)
+ sd x14, 112(sp)
+ sd x13, 104(sp)
+ sd x12, 96(sp)
+ // sd x11, 88(sp) ; already saved
+ // sd x10, 80(sp) ; already saved
+ sd x9, 72(sp)
+ // sd x8, 64(sp) ; already saved
+ sd x7, 56(sp)
+ sd x6, 48(sp)
+ sd x5, 40(sp)
+ sd x4, 32(sp)
+ sd x3, 24(sp)
+ sd x2, 16(sp)
+ // sd x1, 8(sp) ; already saved
+ // sd x0, 0(sp) ; don't store zero register
+
+ // Pass the address of the frame to __hwasan_tag_mismatch4, so that it can
+ // extract the saved registers from this frame without having to worry about
+ // finding this frame.
+ mv x12, sp
+
+ call __hwasan_tag_mismatch4
+ CFI_ENDPROC
+ASM_SIZE(__hwasan_tag_mismatch_v2)
+
+#endif // defined(__riscv) && (__riscv_xlen == 64)
+
+// We do not need executable stack.
+NO_EXEC_STACK_DIRECTIVE
+#include "hwasan_thread.h"
+
#include "hwasan.h"
+#include "hwasan_interface_internal.h"
#include "hwasan_mapping.h"
-#include "hwasan_thread.h"
#include "hwasan_poisoning.h"
-#include "hwasan_interface_internal.h"
-
+#include "hwasan_thread_list.h"
+#include "sanitizer_common/sanitizer_atomic.h"
#include "sanitizer_common/sanitizer_file.h"
#include "sanitizer_common/sanitizer_placement_new.h"
#include "sanitizer_common/sanitizer_tls_get_addr.h"
-
namespace __hwasan {
static u32 RandomSeed() {
void Thread::InitRandomState() {
random_state_ = flags()->random_tags ? RandomSeed() : unique_id_;
+ random_state_inited_ = true;
// Push a random number of zeros onto the ring buffer so that the first stack
// tag base will be random.
CHECK_EQ(0, stack_top_);
CHECK_EQ(0, stack_bottom_);
- static u64 unique_id;
- unique_id_ = unique_id++;
+ static atomic_uint64_t unique_id;
+ unique_id_ = atomic_fetch_add(&unique_id, 1, memory_order_relaxed);
+
if (auto sz = flags()->heap_history_size)
heap_allocations_ = HeapAllocationsRingBuffer::New(sz);
- InitStackAndTls(state);
#if !SANITIZER_FUCHSIA
// Do not initialize the stack ring buffer just yet on Fuchsia. Threads will
// be initialized before we enter the thread itself, so we will instead call
// this later.
InitStackRingBuffer(stack_buffer_start, stack_buffer_size);
#endif
+ InitStackAndTls(state);
+ dtls_ = DTLS_Get();
}
void Thread::InitStackRingBuffer(uptr stack_buffer_start,
}
void Thread::Print(const char *Prefix) {
- Printf("%sT%zd %p stack: [%p,%p) sz: %zd tls: [%p,%p)\n", Prefix,
- unique_id_, this, stack_bottom(), stack_top(),
- stack_top() - stack_bottom(),
- tls_begin(), tls_end());
+ Printf("%sT%zd %p stack: [%p,%p) sz: %zd tls: [%p,%p)\n", Prefix, unique_id_,
+ (void *)this, stack_bottom(), stack_top(),
+ stack_top() - stack_bottom(), tls_begin(), tls_end());
}
static u32 xorshift(u32 state) {
// Generate a (pseudo-)random non-zero tag.
tag_t Thread::GenerateRandomTag(uptr num_bits) {
DCHECK_GT(num_bits, 0);
- if (tagging_disabled_) return 0;
+ if (tagging_disabled_)
+ return 0;
tag_t tag;
const uptr tag_mask = (1ULL << num_bits) - 1;
do {
if (flags()->random_tags) {
- if (!random_buffer_)
+ if (!random_buffer_) {
+ EnsureRandomStateInited();
random_buffer_ = random_state_ = xorshift(random_state_);
+ }
CHECK(random_buffer_);
tag = random_buffer_ & tag_mask;
random_buffer_ >>= num_bits;
} else {
+ EnsureRandomStateInited();
random_state_ += 1;
tag = random_state_ & tag_mask;
}
}
} // namespace __hwasan
+
+// --- Implementation of LSan-specific functions --- {{{1
+namespace __lsan {
+
+static __hwasan::HwasanThreadList *GetHwasanThreadListLocked() {
+ auto &tl = __hwasan::hwasanThreadList();
+ tl.CheckLocked();
+ return &tl;
+}
+
+static __hwasan::Thread *GetThreadByOsIDLocked(tid_t os_id) {
+ return GetHwasanThreadListLocked()->FindThreadLocked(
+ [os_id](__hwasan::Thread *t) { return t->os_id() == os_id; });
+}
+
+void LockThreadRegistry() { __hwasan::hwasanThreadList().Lock(); }
+
+void UnlockThreadRegistry() { __hwasan::hwasanThreadList().Unlock(); }
+
+void EnsureMainThreadIDIsCorrect() {
+ auto *t = __hwasan::GetCurrentThread();
+ if (t && (t->IsMainThread()))
+ t->set_os_id(GetTid());
+}
+
+bool GetThreadRangesLocked(tid_t os_id, uptr *stack_begin, uptr *stack_end,
+ uptr *tls_begin, uptr *tls_end, uptr *cache_begin,
+ uptr *cache_end, DTLS **dtls) {
+ auto *t = GetThreadByOsIDLocked(os_id);
+ if (!t)
+ return false;
+ *stack_begin = t->stack_bottom();
+ *stack_end = t->stack_top();
+ *tls_begin = t->tls_begin();
+ *tls_end = t->tls_end();
+ // Fixme: is this correct for HWASan.
+ *cache_begin = 0;
+ *cache_end = 0;
+ *dtls = t->dtls();
+ return true;
+}
+
+void GetAllThreadAllocatorCachesLocked(InternalMmapVector<uptr> *caches) {}
+
+void GetThreadExtraStackRangesLocked(tid_t os_id,
+ InternalMmapVector<Range> *ranges) {}
+void GetThreadExtraStackRangesLocked(InternalMmapVector<Range> *ranges) {}
+
+void GetAdditionalThreadContextPtrsLocked(InternalMmapVector<uptr> *ptrs) {}
+void GetRunningThreadsLocked(InternalMmapVector<tid_t> *threads) {}
+
+} // namespace __lsan
void Init(uptr stack_buffer_start, uptr stack_buffer_size,
const InitState *state = nullptr);
- void InitRandomState();
+
void InitStackAndTls(const InitState *state = nullptr);
// Must be called from the thread itself.
void InitStackRingBuffer(uptr stack_buffer_start, uptr stack_buffer_size);
+ inline void EnsureRandomStateInited() {
+ if (UNLIKELY(!random_state_inited_))
+ InitRandomState();
+ }
+
void Destroy();
uptr stack_top() { return stack_top_; }
uptr stack_size() { return stack_top() - stack_bottom(); }
uptr tls_begin() { return tls_begin_; }
uptr tls_end() { return tls_end_; }
+ DTLS *dtls() { return dtls_; }
bool IsMainThread() { return unique_id_ == 0; }
bool AddrIsInStack(uptr addr) {
void DisableTagging() { tagging_disabled_++; }
void EnableTagging() { tagging_disabled_--; }
- u64 unique_id() const { return unique_id_; }
+ u32 unique_id() const { return unique_id_; }
void Announce() {
if (announced_) return;
announced_ = true;
Print("Thread: ");
}
+ tid_t os_id() const { return os_id_; }
+ void set_os_id(tid_t os_id) { os_id_ = os_id; }
+
uptr &vfork_spill() { return vfork_spill_; }
private:
// via mmap() and *must* be valid in zero-initialized state.
void ClearShadowForThreadStackAndTLS();
void Print(const char *prefix);
+ void InitRandomState();
uptr vfork_spill_;
uptr stack_top_;
uptr stack_bottom_;
uptr tls_begin_;
uptr tls_end_;
+ DTLS *dtls_;
u32 random_state_;
u32 random_buffer_;
HeapAllocationsRingBuffer *heap_allocations_;
StackAllocationsRingBuffer *stack_allocations_;
- u64 unique_id_; // counting from zero.
+ u32 unique_id_; // counting from zero.
+
+ tid_t os_id_;
u32 tagging_disabled_; // if non-zero, malloc uses zero tag in this thread.
bool announced_;
+ bool random_state_inited_; // Whether InitRandomState() has been called.
+
friend struct ThreadListHead;
};
uptr total_stack_size;
};
-class HwasanThreadList {
+class SANITIZER_MUTEX HwasanThreadList {
public:
HwasanThreadList(uptr storage, uptr size)
: free_space_(storage), free_space_end_(storage + size) {
RoundUpTo(ring_buffer_size_ + sizeof(Thread), ring_buffer_size_ * 2);
}
- Thread *CreateCurrentThread(const Thread::InitState *state = nullptr) {
+ Thread *CreateCurrentThread(const Thread::InitState *state = nullptr)
+ SANITIZER_EXCLUDES(free_list_mutex_, live_list_mutex_) {
Thread *t = nullptr;
{
SpinMutexLock l(&free_list_mutex_);
ReleaseMemoryPagesToOS(start, start + thread_alloc_size_);
}
- void RemoveThreadFromLiveList(Thread *t) {
+ void RemoveThreadFromLiveList(Thread *t)
+ SANITIZER_EXCLUDES(live_list_mutex_) {
SpinMutexLock l(&live_list_mutex_);
for (Thread *&t2 : live_list_)
if (t2 == t) {
CHECK(0 && "thread not found in live list");
}
- void ReleaseThread(Thread *t) {
+ void ReleaseThread(Thread *t) SANITIZER_EXCLUDES(free_list_mutex_) {
RemoveThreadStats(t);
t->Destroy();
DontNeedThread(t);
}
template <class CB>
- void VisitAllLiveThreads(CB cb) {
+ void VisitAllLiveThreads(CB cb) SANITIZER_EXCLUDES(live_list_mutex_) {
SpinMutexLock l(&live_list_mutex_);
for (Thread *t : live_list_) cb(t);
}
- void AddThreadStats(Thread *t) {
+ template <class CB>
+ Thread *FindThreadLocked(CB cb) SANITIZER_CHECK_LOCKED(stats_mutex_) {
+ CheckLocked();
+ for (Thread *t : live_list_)
+ if (cb(t))
+ return t;
+ return nullptr;
+ }
+
+ void AddThreadStats(Thread *t) SANITIZER_EXCLUDES(stats_mutex_) {
SpinMutexLock l(&stats_mutex_);
stats_.n_live_threads++;
stats_.total_stack_size += t->stack_size();
}
- void RemoveThreadStats(Thread *t) {
+ void RemoveThreadStats(Thread *t) SANITIZER_EXCLUDES(stats_mutex_) {
SpinMutexLock l(&stats_mutex_);
stats_.n_live_threads--;
stats_.total_stack_size -= t->stack_size();
}
- ThreadStats GetThreadStats() {
+ ThreadStats GetThreadStats() SANITIZER_EXCLUDES(stats_mutex_) {
SpinMutexLock l(&stats_mutex_);
return stats_;
}
uptr GetRingBufferSize() const { return ring_buffer_size_; }
+ void Lock() SANITIZER_ACQUIRE(live_list_mutex_) { live_list_mutex_.Lock(); }
+ void CheckLocked() const SANITIZER_CHECK_LOCKED(live_list_mutex_) {
+ live_list_mutex_.CheckLocked();
+ }
+ void Unlock() SANITIZER_RELEASE(live_list_mutex_) {
+ live_list_mutex_.Unlock();
+ }
+
private:
Thread *AllocThread() {
SpinMutexLock l(&free_space_mutex_);
uptr thread_alloc_size_;
SpinMutex free_list_mutex_;
- InternalMmapVector<Thread *> free_list_;
+ InternalMmapVector<Thread *> free_list_
+ SANITIZER_GUARDED_BY(free_list_mutex_);
SpinMutex live_list_mutex_;
- InternalMmapVector<Thread *> live_list_;
+ InternalMmapVector<Thread *> live_list_
+ SANITIZER_GUARDED_BY(live_list_mutex_);
- ThreadStats stats_;
SpinMutex stats_mutex_;
+ ThreadStats stats_ SANITIZER_GUARDED_BY(stats_mutex_);
};
void InitThreadList(uptr storage, uptr size);
#define CHECK_TYPE_SIZE_FITS(TYPE) \
COMPILER_CHECK(sizeof(__hw_##TYPE) <= sizeof(TYPE))
-#if HWASAN_WITH_INTERCEPTORS && defined(__aarch64__)
+#if HWASAN_WITH_INTERCEPTORS
CHECK_TYPE_SIZE_FITS(jmp_buf);
CHECK_TYPE_SIZE_FITS(sigjmp_buf);
#endif
-#!/usr/bin/env python
+#!/usr/bin/env python3
#===- lib/hwasan/scripts/hwasan_symbolize ----------------------------------===#
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# HWAddressSanitizer offline symbolization script.
#
#===------------------------------------------------------------------------===#
+
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import argparse
import glob
+import html
+import json
+import mmap
import os
import re
-import sys
-import string
+import struct
import subprocess
-import argparse
+import sys
-last_access_address = None
-last_access_tag = None
+if sys.version_info.major < 3:
+ # Simulate Python 3.x behaviour of defaulting to UTF-8 for print. This is
+ # important in case any symbols are non-ASCII.
+ import codecs
+ sys.stdout = codecs.getwriter("utf-8")(sys.stdout)
+
+# Below, a parser for a subset of ELF. It only supports 64 bit, little-endian,
+# and only parses what is necessary to find the build ids. It uses a memoryview
+# into an mmap to avoid copying.
+Ehdr_size = 64
+e_shnum_offset = 60
+e_shoff_offset = 40
+
+Shdr_size = 64
+sh_type_offset = 4
+sh_offset_offset = 24
+sh_size_offset = 32
+SHT_NOTE = 7
+
+Nhdr_size = 12
+NT_GNU_BUILD_ID = 3
+
+def align_up(size, alignment):
+ return (size + alignment - 1) & ~(alignment - 1)
+
+def handle_Nhdr(mv, sh_size):
+ offset = 0
+ while offset < sh_size:
+ n_namesz, n_descsz, n_type = struct.unpack_from('<III', buffer=mv,
+ offset=offset)
+ if (n_type == NT_GNU_BUILD_ID and n_namesz == 4 and
+ mv[offset + Nhdr_size: offset + Nhdr_size + 4] == b"GNU\x00"):
+ value = mv[offset + Nhdr_size + 4: offset + Nhdr_size + 4 + n_descsz]
+ return value.hex()
+ offset += Nhdr_size + align_up(n_namesz, 4) + align_up(n_descsz, 4)
+ return None
+
+def handle_Shdr(mv):
+ sh_type, = struct.unpack_from('<I', buffer=mv, offset=sh_type_offset)
+ if sh_type != SHT_NOTE:
+ return None, None
+ sh_offset, = struct.unpack_from('<Q', buffer=mv, offset=sh_offset_offset)
+ sh_size, = struct.unpack_from('<Q', buffer=mv, offset=sh_size_offset)
+ return sh_offset, sh_size
+
+def handle_elf(mv):
+ # \x02 is ELFCLASS64, \x01 is ELFDATA2LSB. HWASan currently only works on
+ # 64-bit little endian platforms (x86_64 and ARM64). If this changes, we will
+ # have to extend the parsing code.
+ if mv[:6] != b'\x7fELF\x02\x01':
+ return None
+ e_shnum, = struct.unpack_from('<H', buffer=mv, offset=e_shnum_offset)
+ e_shoff, = struct.unpack_from('<Q', buffer=mv, offset=e_shoff_offset)
+ for i in range(0, e_shnum):
+ start = e_shoff + i * Shdr_size
+ sh_offset, sh_size = handle_Shdr(mv[start: start + Shdr_size])
+ if sh_offset is None:
+ continue
+ note_hdr = mv[sh_offset: sh_offset + sh_size]
+ result = handle_Nhdr(note_hdr, sh_size)
+ if result is not None:
+ return result
+
+def get_buildid(filename):
+ with open(filename, "r") as fd:
+ if os.fstat(fd.fileno()).st_size < Ehdr_size:
+ return None
+ with mmap.mmap(fd.fileno(), 0, access=mmap.ACCESS_READ) as m:
+ with memoryview(m) as mv:
+ return handle_elf(mv)
class Symbolizer:
def __init__(self, path, binary_prefixes, paths_to_cut):
self.__paths_to_cut = paths_to_cut
self.__log = False
self.__warnings = set()
+ self.__index = {}
+ self.__link_prefixes = []
+ self.__html = False
+ self.__last_access_address = None
+ self.__last_access_tag = None
+
+ def enable_html(self, enable):
+ self.__html = enable
def enable_logging(self, enable):
self.__log = enable
+ def maybe_escape(self, text):
+ if self.__html:
+ # We need to manually use for leading spaces, html.escape does
+ # not do that, and HTML ignores them.
+ spaces = 0
+ for i, c in enumerate(text):
+ spaces = i
+ if c != ' ':
+ break
+ text = text[spaces:]
+ return spaces * ' ' + html.escape(text)
+ return text
+
+ def print(self, line, escape=True):
+ if escape:
+ line = self.maybe_escape(line)
+ if self.__html:
+ line += '<br/>'
+ print(line)
+
+ def read_linkify(self, filename):
+ with open(filename, 'r') as fd:
+ data = json.load(fd)
+ self.__link_prefixes = [(e["prefix"], e["link"]) for e in data]
+
def __open_pipe(self):
if not self.__pipe:
- self.__pipe = subprocess.Popen([self.__path, "-inlining", "-functions"],
- stdin=subprocess.PIPE, stdout=subprocess.PIPE)
-
- class __EOF:
+ opt = {}
+ if sys.version_info.major > 2:
+ opt['encoding'] = 'utf-8'
+ self.__pipe = subprocess.Popen([self.__path, "--inlining", "--functions"],
+ stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+ **opt)
+
+ class __EOF(Exception):
pass
def __write(self, s):
- print >>self.__pipe.stdin, s
+ print(s, file=self.__pipe.stdin)
+ self.__pipe.stdin.flush()
if self.__log:
- print >>sys.stderr, ("#>> |%s|" % (s,))
+ print("#>> |%s|" % (s,), file=sys.stderr)
def __read(self):
s = self.__pipe.stdout.readline().rstrip()
if self.__log:
- print >>sys.stderr, ("# << |%s|" % (s,))
+ print("# << |%s|" % (s,), file=sys.stderr)
if s == '':
raise Symbolizer.__EOF
return s
file_name = re.sub(".*crtstuff.c:0", "???:0", file_name)
return file_name
- def __process_binary_name(self, name):
+ def __process_binary_name(self, name, buildid):
if name.startswith('/'):
name = name[1:]
+ if buildid is not None and buildid in self.__index:
+ return self.__index[buildid]
+
for p in self.__binary_prefixes:
full_path = os.path.join(p, name)
if os.path.exists(full_path):
return full_path
+ apex_prefix = "apex/com.android."
+ if name.startswith(apex_prefix):
+ full_path = os.path.join(p, "apex/com.google.android." + name[len(apex_prefix):])
+ if os.path.exists(full_path):
+ return full_path
# Try stripping extra path components as the last resort.
for p in self.__binary_prefixes:
full_path = os.path.join(p, os.path.basename(name))
if os.path.exists(full_path):
return full_path
if name not in self.__warnings:
- print >>sys.stderr, "Could not find symbols for", name
+ print("Could not find symbols for", name, file=sys.stderr)
self.__warnings.add(name)
return None
- def iter_locals(self, binary, addr):
+ def iter_locals(self, binary, addr, buildid):
self.__open_pipe()
p = self.__pipe
- binary = self.__process_binary_name(binary)
+ binary = self.__process_binary_name(binary, buildid)
if not binary:
return
self.__write("FRAME %s %s" % (binary, addr))
except Symbolizer.__EOF:
pass
- def iter_call_stack(self, binary, addr):
+ def iter_call_stack(self, binary, buildid, addr):
self.__open_pipe()
p = self.__pipe
- binary = self.__process_binary_name(binary)
+ binary = self.__process_binary_name(binary, buildid)
if not binary:
return
self.__write("CODE %s %s" % (binary, addr))
except Symbolizer.__EOF:
pass
-def symbolize_line(line, symbolizer_path):
- #0 0x7f6e35cf2e45 (/blah/foo.so+0x11fe45)
- match = re.match(r'^(.*?)#([0-9]+)( *)(0x[0-9a-f]*) *\((.*)\+(0x[0-9a-f]+)\)', line, re.UNICODE)
- if match:
- frameno = match.group(2)
- binary = match.group(5)
- addr = int(match.group(6), 16)
-
- frames = list(symbolizer.iter_call_stack(binary, addr))
-
- if len(frames) > 0:
- print "%s#%s%s%s in %s" % (match.group(1).encode('utf-8'), match.group(2).encode('utf-8'),
- match.group(3).encode('utf-8'), frames[0][0], frames[0][1])
- for i in range(1, len(frames)):
- space1 = ' ' * match.end(1)
- space2 = ' ' * (match.start(4) - match.end(1) - 2)
- print "%s->%s%s in %s" % (space1, space2, frames[i][0], frames[i][1])
+ def maybe_linkify(self, file_line):
+ if not self.__html or not self.__link_prefixes:
+ return file_line
+ filename, line_col = file_line.split(':', 1)
+ if not line_col:
+ line = '0' # simplify the link generation
else:
- print line.rstrip().encode('utf-8')
- else:
- print line.rstrip().encode('utf-8')
-
-def save_access_address(line):
- global last_access_address, last_access_tag
- match = re.match(r'^(.*?)HWAddressSanitizer: tag-mismatch on address (0x[0-9a-f]+) ', line, re.UNICODE)
- if match:
- last_access_address = int(match.group(2), 16)
- match = re.match(r'^(.*?) of size [0-9]+ at 0x[0-9a-f]* tags: ([0-9a-f]+)/[0-9a-f]+ \(ptr/mem\)', line, re.UNICODE)
- if match:
- last_access_tag = int(match.group(2), 16)
-
-def process_stack_history(line, symbolizer, ignore_tags=False):
- if last_access_address is None or last_access_tag is None:
- return
- if re.match(r'Previously allocated frames:', line, re.UNICODE):
- return True
- pc_mask = (1 << 48) - 1
- fp_mask = (1 << 20) - 1
- # record_addr:0x1234ABCD record:0x1234ABCD (/path/to/binary+0x1234ABCD)
- match = re.match(r'^(.*?)record_addr:(0x[0-9a-f]+) +record:(0x[0-9a-f]+) +\((.*)\+(0x[0-9a-f]+)\)', line, re.UNICODE)
- if match:
- record_addr = int(match.group(2), 16)
- record = int(match.group(3), 16)
- binary = match.group(4)
- addr = int(match.group(5), 16)
- base_tag = (record_addr >> 3) & 0xFF
- fp = (record >> 48) << 4
- pc = record & pc_mask
-
- for local in symbolizer.iter_locals(binary, addr):
- frame_offset = local[3]
- size = local[4]
- if frame_offset is None or size is None:
- continue
- obj_offset = (last_access_address - fp - frame_offset) & fp_mask
- if obj_offset >= size:
- continue
- tag_offset = local[5]
- if not ignore_tags and (tag_offset is None or base_tag ^ tag_offset != last_access_tag):
- continue
- print ''
- print 'Potentially referenced stack object:'
- print ' %d bytes inside variable "%s" in stack frame of function "%s"' % (obj_offset, local[2], local[0])
- print ' at %s' % (local[1],)
- return True
- return False
-
-parser = argparse.ArgumentParser()
-parser.add_argument('-d', action='store_true')
-parser.add_argument('-v', action='store_true')
-parser.add_argument('--ignore-tags', action='store_true')
-parser.add_argument('--symbols', action='append')
-parser.add_argument('--source', action='append')
-parser.add_argument('--symbolizer')
-parser.add_argument('args', nargs=argparse.REMAINDER)
-args = parser.parse_args()
-
-# Unstripped binaries location.
-binary_prefixes = args.symbols or []
-if not binary_prefixes:
- if 'ANDROID_PRODUCT_OUT' in os.environ:
- product_out = os.path.join(os.environ['ANDROID_PRODUCT_OUT'], 'symbols')
- binary_prefixes.append(product_out)
-
-for p in binary_prefixes:
- if not os.path.isdir(p):
- print >>sys.stderr, "Symbols path does not exist or is not a directory:", p
- sys.exit(1)
-
-# Source location.
-paths_to_cut = args.source or []
-if not paths_to_cut:
- paths_to_cut.append(os.getcwd() + '/')
- if 'ANDROID_BUILD_TOP' in os.environ:
- paths_to_cut.append(os.environ['ANDROID_BUILD_TOP'] + '/')
-
-# llvm-symbolizer binary.
-# 1. --symbolizer flag
-# 2. environment variable
-# 3. unsuffixed binary in the current directory
-# 4. if inside Android platform, prebuilt binary at a known path
-# 5. first "llvm-symbolizer", then "llvm-symbolizer-$VER" with the
-# highest available version in $PATH
-symbolizer_path = args.symbolizer
-if not symbolizer_path:
- if 'LLVM_SYMBOLIZER_PATH' in os.environ:
- symbolizer_path = os.environ['LLVM_SYMBOLIZER_PATH']
- elif 'HWASAN_SYMBOLIZER_PATH' in os.environ:
- symbolizer_path = os.environ['HWASAN_SYMBOLIZER_PATH']
-
-if not symbolizer_path:
- s = os.path.join(os.path.dirname(sys.argv[0]), 'llvm-symbolizer')
- if os.path.exists(s):
- symbolizer_path = s
-
-if not symbolizer_path:
- if 'ANDROID_BUILD_TOP' in os.environ:
- s = os.path.join(os.environ['ANDROID_BUILD_TOP'], 'prebuilts/clang/host/linux-x86/llvm-binutils-stable/llvm-symbolizer')
- if os.path.exists(s):
- symbolizer_path = s
+ line = line_col.split(':')[0]
+ longest_prefix = max((
+ (prefix, link) for prefix, link in self.__link_prefixes
+ if filename.startswith(prefix)),
+ key=lambda x: len(x[0]), default=None)
+ if longest_prefix is None:
+ return file_line
+ else:
+ prefix, link = longest_prefix
+ return '<a href="{}">{}</a>'.format(
+ html.escape(link.format(file=filename[len(prefix):], line=line,
+ file_line=file_line, prefix=prefix)), file_line)
-if not symbolizer_path:
- for path in os.environ["PATH"].split(os.pathsep):
- p = os.path.join(path, 'llvm-symbolizer')
- if os.path.exists(p):
- symbolizer_path = p
- break
+ def build_index(self):
+ for p in self.__binary_prefixes:
+ for dname, _, fnames in os.walk(p):
+ for fn in fnames:
+ filename = os.path.join(dname, fn)
+ try:
+ bid = get_buildid(filename)
+ except FileNotFoundError:
+ continue
+ except Exception as e:
+ print("Failed to parse {}: {}".format(filename, e), file=sys.stderr)
+ continue
+ if bid is not None:
+ self.__index[bid] = filename
+
+ def symbolize_line(self, line):
+ #0 0x7f6e35cf2e45 (/blah/foo.so+0x11fe45) (BuildId: 4abce4cd41ea5c2f34753297b7e774d9)
+ match = re.match(r'^(.*?)#([0-9]+)( *)(0x[0-9a-f]*) *\((.*)\+(0x[0-9a-f]+)\)'
+ r'(?:\s*\(BuildId: ([0-9a-f]+)\))?', line, re.UNICODE)
+ if match:
+ frameno = match.group(2)
+ binary = match.group(5)
+ addr = int(match.group(6), 16)
+ buildid = match.group(7)
+
+ frames = list(self.iter_call_stack(binary, buildid, addr))
+
+ if len(frames) > 0:
+ self.print(
+ self.maybe_escape(
+ "%s#%s%s%s in " % (match.group(1), match.group(2), match.group(3),
+ frames[0][0])
+ ) + self.maybe_linkify(frames[0][1]),
+ escape=False)
+ for i in range(1, len(frames)):
+ space1 = ' ' * match.end(1)
+ space2 = ' ' * (match.start(4) - match.end(1) - 2)
+ self.print(
+ self.maybe_escape("%s->%s%s in " % (space1, space2, frames[i][0]))
+ + self.maybe_linkify(frames[i][1]), escape=False)
+ else:
+ self.print(line.rstrip())
+ else:
+ self.print(line.rstrip())
+
+ def save_access_address(self, line):
+ match = re.match(r'^(.*?)HWAddressSanitizer: tag-mismatch on address (0x[0-9a-f]+) ', line, re.UNICODE)
+ if match:
+ self.__last_access_address = int(match.group(2), 16)
+ match = re.match(r'^(.*?) of size [0-9]+ at 0x[0-9a-f]* tags: ([0-9a-f]+)/[0-9a-f]+(\([0-9a-f]+\))? \(ptr/mem\)', line, re.UNICODE)
+ if match:
+ self.__last_access_tag = int(match.group(2), 16)
+
+ def process_stack_history(self, line, ignore_tags=False):
+ if self.__last_access_address is None or self.__last_access_tag is None:
+ return
+ if re.match(r'Previously allocated frames:', line, re.UNICODE):
+ return True
+ pc_mask = (1 << 48) - 1
+ fp_mask = (1 << 20) - 1
+ # record_addr:0x1234ABCD record:0x1234ABCD (/path/to/binary+0x1234ABCD) (BuildId: 4abce4cd41ea5c2f34753297b7e774d9)
+ match = re.match(r'^(.*?)record_addr:(0x[0-9a-f]+) +record:(0x[0-9a-f]+) +\((.*)\+(0x[0-9a-f]+)\)'
+ r'(?:\s*\(BuildId: ([0-9a-f]+)\))?', line, re.UNICODE)
+ if match:
+ record_addr = int(match.group(2), 16)
+ record = int(match.group(3), 16)
+ binary = match.group(4)
+ addr = int(match.group(5), 16)
+ buildid = match.group(6)
+ base_tag = (record_addr >> 3) & 0xFF
+ fp = (record >> 48) << 4
+ pc = record & pc_mask
+
+ for local in self.iter_locals(binary, addr, buildid):
+ frame_offset = local[3]
+ size = local[4]
+ if frame_offset is None or size is None:
+ continue
+ obj_offset = (self.__last_access_address - fp - frame_offset) & fp_mask
+ if obj_offset >= size:
+ continue
+ tag_offset = local[5]
+ if not ignore_tags and (tag_offset is None or base_tag ^ tag_offset != self.__last_access_tag):
+ continue
+ self.print('')
+ self.print('Potentially referenced stack object:')
+ self.print(' %d bytes inside a variable "%s" in stack frame of function "%s"' % (obj_offset, local[2], local[0]))
+ self.print(' at %s' % (local[1],))
+ return True
+ return False
def extract_version(s):
idx = s.rfind('-')
x = float(s[idx + 1:])
return x
-if not symbolizer_path:
- for path in os.environ["PATH"].split(os.pathsep):
- candidates = glob.glob(os.path.join(path, 'llvm-symbolizer-*'))
- if len(candidates) > 0:
- candidates.sort(key = extract_version, reverse = True)
- symbolizer_path = candidates[0]
- break
-
-if not os.path.exists(symbolizer_path):
- print >>sys.stderr, "Symbolizer path does not exist:", symbolizer_path
- sys.exit(1)
-
-if args.v:
- print "Looking for symbols in:"
- for s in binary_prefixes:
- print " %s" % (s,)
- print "Stripping source path prefixes:"
- for s in paths_to_cut:
- print " %s" % (s,)
- print "Using llvm-symbolizer binary in:\n %s" % (symbolizer_path,)
- print
-
-symbolizer = Symbolizer(symbolizer_path, binary_prefixes, paths_to_cut)
-symbolizer.enable_logging(args.d)
-
-for line in sys.stdin:
- line = line.decode('utf-8')
- save_access_address(line)
- if process_stack_history(line, symbolizer, ignore_tags=args.ignore_tags):
- continue
- symbolize_line(line, symbolizer_path)
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('-d', action='store_true')
+ parser.add_argument('-v', action='store_true')
+ parser.add_argument('--ignore-tags', action='store_true')
+ parser.add_argument('--symbols', action='append')
+ parser.add_argument('--source', action='append')
+ parser.add_argument('--index', action='store_true')
+ parser.add_argument('--symbolizer')
+ parser.add_argument('--linkify', type=str)
+ parser.add_argument('--html', action='store_true')
+ parser.add_argument('args', nargs=argparse.REMAINDER)
+ args = parser.parse_args()
+
+ # Unstripped binaries location.
+ binary_prefixes = args.symbols or []
+ if not binary_prefixes:
+ if 'ANDROID_PRODUCT_OUT' in os.environ:
+ product_out = os.path.join(os.environ['ANDROID_PRODUCT_OUT'], 'symbols')
+ binary_prefixes.append(product_out)
+ binary_prefixes.append('/')
+
+ for p in binary_prefixes:
+ if not os.path.isdir(p):
+ print("Symbols path does not exist or is not a directory:", p, file=sys.stderr)
+ sys.exit(1)
+
+ # Source location.
+ paths_to_cut = args.source or []
+ if not paths_to_cut:
+ paths_to_cut.append(os.getcwd() + '/')
+ if 'ANDROID_BUILD_TOP' in os.environ:
+ paths_to_cut.append(os.environ['ANDROID_BUILD_TOP'] + '/')
+
+ # llvm-symbolizer binary.
+ # 1. --symbolizer flag
+ # 2. environment variable
+ # 3. unsuffixed binary in the current directory
+ # 4. if inside Android platform, prebuilt binary at a known path
+ # 5. first "llvm-symbolizer", then "llvm-symbolizer-$VER" with the
+ # highest available version in $PATH
+ symbolizer_path = args.symbolizer
+ if not symbolizer_path:
+ if 'LLVM_SYMBOLIZER_PATH' in os.environ:
+ symbolizer_path = os.environ['LLVM_SYMBOLIZER_PATH']
+ elif 'HWASAN_SYMBOLIZER_PATH' in os.environ:
+ symbolizer_path = os.environ['HWASAN_SYMBOLIZER_PATH']
+
+ if not symbolizer_path:
+ s = os.path.join(os.path.dirname(sys.argv[0]), 'llvm-symbolizer')
+ if os.path.exists(s):
+ symbolizer_path = s
+
+ if not symbolizer_path:
+ if 'ANDROID_BUILD_TOP' in os.environ:
+ s = os.path.join(os.environ['ANDROID_BUILD_TOP'], 'prebuilts/clang/host/linux-x86/llvm-binutils-stable/llvm-symbolizer')
+ if os.path.exists(s):
+ symbolizer_path = s
+
+ if not symbolizer_path:
+ for path in os.environ["PATH"].split(os.pathsep):
+ p = os.path.join(path, 'llvm-symbolizer')
+ if os.path.exists(p):
+ symbolizer_path = p
+ break
+
+ if not symbolizer_path:
+ for path in os.environ["PATH"].split(os.pathsep):
+ candidates = glob.glob(os.path.join(path, 'llvm-symbolizer-*'))
+ if len(candidates) > 0:
+ candidates.sort(key = extract_version, reverse = True)
+ symbolizer_path = candidates[0]
+ break
+
+ if not os.path.exists(symbolizer_path):
+ print("Symbolizer path does not exist:", symbolizer_path, file=sys.stderr)
+ sys.exit(1)
+
+ if args.v:
+ print("Looking for symbols in:")
+ for s in binary_prefixes:
+ print(" %s" % (s,))
+ print("Stripping source path prefixes:")
+ for s in paths_to_cut:
+ print(" %s" % (s,))
+ print("Using llvm-symbolizer binary in:\n %s" % (symbolizer_path,))
+ print()
+
+ symbolizer = Symbolizer(symbolizer_path, binary_prefixes, paths_to_cut)
+ symbolizer.enable_html(args.html)
+ symbolizer.enable_logging(args.d)
+ if args.index:
+ symbolizer.build_index()
+
+ if args.linkify:
+ if not args.html:
+ print('Need --html to --linkify', file=sys.stderr)
+ sys.exit(1)
+ symbolizer.read_linkify(args.linkify)
+
+ for line in sys.stdin:
+ if sys.version_info.major < 3:
+ line = line.decode('utf-8')
+ symbolizer.save_access_address(line)
+ if symbolizer.process_stack_history(line, ignore_tags=args.ignore_tags):
+ continue
+ symbolizer.symbolize_line(line)
+
+
+if __name__ == '__main__':
+ main()
set(INTERCEPTION_CFLAGS ${SANITIZER_COMMON_CFLAGS})
append_rtti_flag(OFF INTERCEPTION_CFLAGS)
+# Silence warnings in system headers with MSVC.
+if(NOT CLANG_CL)
+ append_list_if(COMPILER_RT_HAS_EXTERNAL_FLAG "/experimental:external /external:W0 /external:anglebrackets" INTERCEPTION_CFLAGS)
+endif()
+
add_compiler_rt_object_libraries(RTInterception
OS ${SANITIZER_COMMON_SUPPORTED_OS}
ARCHS ${SANITIZER_COMMON_SUPPORTED_ARCH}
#include "interception.h"
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
-#endif // SANITIZER_MAC
+#endif // SANITIZER_APPLE
// Mac-specific interception methods.
//===----------------------------------------------------------------------===//
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
#if !defined(INCLUDED_FROM_INTERCEPTION_LIB)
# error "interception_mac.h should be included from interception.h only"
#define INTERCEPT_FUNCTION_VER_MAC(func, symver)
#endif // INTERCEPTION_MAC_H
-#endif // SANITIZER_MAC
+#endif // SANITIZER_APPLE
#include "interception.h"
-#if SANITIZER_LINUX || SANITIZER_MAC
+#if SANITIZER_LINUX || SANITIZER_APPLE
#include <sys/types.h>
#include <stddef.h>
COMPILER_CHECK(sizeof(::PTRDIFF_T) == sizeof(ptrdiff_t));
COMPILER_CHECK(sizeof(::INTMAX_T) == sizeof(intmax_t));
-#if !SANITIZER_MAC
+# if SANITIZER_GLIBC || SANITIZER_ANDROID
COMPILER_CHECK(sizeof(::OFF64_T) == sizeof(off64_t));
-#endif
+# endif
// The following are the cases when pread (and friends) is used instead of
// pread64. In those cases we need OFF_T to match off_t. We don't care about the
// tramp: jmp QWORD [addr]
// addr: .bytes <hook>
//
-// Note: <real> is equilavent to <label>.
+// Note: <real> is equivalent to <label>.
//
// 3) HotPatch
//
return allocated_space;
}
+// The following prologues cannot be patched because of the short jump
+// jumping to the patching region.
+
+#if SANITIZER_WINDOWS64
+// ntdll!wcslen in Win11
+// 488bc1 mov rax,rcx
+// 0fb710 movzx edx,word ptr [rax]
+// 4883c002 add rax,2
+// 6685d2 test dx,dx
+// 75f4 jne -12
+static const u8 kPrologueWithShortJump1[] = {
+ 0x48, 0x8b, 0xc1, 0x0f, 0xb7, 0x10, 0x48, 0x83,
+ 0xc0, 0x02, 0x66, 0x85, 0xd2, 0x75, 0xf4,
+};
+
+// ntdll!strrchr in Win11
+// 4c8bc1 mov r8,rcx
+// 8a01 mov al,byte ptr [rcx]
+// 48ffc1 inc rcx
+// 84c0 test al,al
+// 75f7 jne -9
+static const u8 kPrologueWithShortJump2[] = {
+ 0x4c, 0x8b, 0xc1, 0x8a, 0x01, 0x48, 0xff, 0xc1,
+ 0x84, 0xc0, 0x75, 0xf7,
+};
+#endif
+
// Returns 0 on error.
static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
+#if SANITIZER_WINDOWS64
+ if (memcmp((u8*)address, kPrologueWithShortJump1,
+ sizeof(kPrologueWithShortJump1)) == 0 ||
+ memcmp((u8*)address, kPrologueWithShortJump2,
+ sizeof(kPrologueWithShortJump2)) == 0) {
+ return 0;
+ }
+#endif
+
switch (*(u64*)address) {
case 0x90909090909006EB: // stub: jmp over 6 x nop.
return 8;
case 0xA1: // A1 XX XX XX XX XX XX XX XX :
// movabs eax, dword ptr ds:[XXXXXXXX]
return 9;
+
+ case 0x83:
+ const u8 next_byte = *(u8*)(address + 1);
+ const u8 mod = next_byte >> 6;
+ const u8 rm = next_byte & 7;
+ if (mod == 1 && rm == 4)
+ return 5; // 83 ModR/M SIB Disp8 Imm8
+ // add|or|adc|sbb|and|sub|xor|cmp [r+disp8], imm8
}
switch (*(u16*)address) {
case 0x5641: // push r14
case 0x5741: // push r15
case 0x9066: // Two-byte NOP
+ case 0xc084: // test al, al
+ case 0x018a: // mov al, byte ptr [rcx]
return 2;
case 0x058B: // 8B 05 XX XX XX XX : mov eax, dword ptr [XX XX XX XX]
case 0xd12b48: // 48 2b d1 : sub rdx, rcx
case 0x07c1f6: // f6 c1 07 : test cl, 0x7
case 0xc98548: // 48 85 C9 : test rcx, rcx
+ case 0xd28548: // 48 85 d2 : test rdx, rdx
case 0xc0854d: // 4d 85 c0 : test r8, r8
case 0xc2b60f: // 0f b6 c2 : movzx eax, dl
case 0xc03345: // 45 33 c0 : xor r8d, r8d
case 0xca2b48: // 48 2b ca : sub rcx, rdx
case 0x10b70f: // 0f b7 10 : movzx edx, WORD PTR [rax]
case 0xc00b4d: // 3d 0b c0 : or r8, r8
+ case 0xc08b41: // 41 8b c0 : mov eax, r8d
case 0xd18b48: // 48 8b d1 : mov rdx, rcx
case 0xdc8b4c: // 4c 8b dc : mov r11, rsp
case 0xd18b4c: // 4c 8b d1 : mov r10, rcx
case 0x246c8948: // 48 89 6C 24 XX : mov QWORD ptr [rsp + XX], rbp
case 0x245c8948: // 48 89 5c 24 XX : mov QWORD PTR [rsp + XX], rbx
case 0x24748948: // 48 89 74 24 XX : mov QWORD PTR [rsp + XX], rsi
+ case 0x247c8948: // 48 89 7c 24 XX : mov QWORD PTR [rsp + XX], rdi
case 0x244C8948: // 48 89 4C 24 XX : mov QWORD PTR [rsp + XX], rcx
case 0x24548948: // 48 89 54 24 XX : mov QWORD PTR [rsp + XX], rdx
case 0x244c894c: // 4c 89 4c 24 XX : mov QWORD PTR [rsp + XX], r9
return false;
if (orig_old_func) {
- uptr relative_offset = *(u32*)(old_func + 1);
+ sptr relative_offset = *(s32 *)(old_func + 1);
uptr absolute_target = old_func + relative_offset + kJumpInstructionLength;
*orig_old_func = absolute_target;
}
} // namespace __interception
-#endif // SANITIZER_MAC
+#endif // SANITIZER_APPLE
set(INTERCEPTION_TEST_CFLAGS_COMMON
${COMPILER_RT_UNITTEST_CFLAGS}
${COMPILER_RT_GTEST_CFLAGS}
+ ${SANITIZER_TEST_CXX_CFLAGS}
-I${COMPILER_RT_SOURCE_DIR}/include
-I${COMPILER_RT_SOURCE_DIR}/lib
-I${COMPILER_RT_SOURCE_DIR}/lib/interception
-Werror=sign-compare)
set(INTERCEPTION_TEST_LINK_FLAGS_COMMON
- ${COMPILER_RT_UNITTEST_LINK_FLAGS})
+ ${COMPILER_RT_UNITTEST_LINK_FLAGS}
+ ${COMPILER_RT_UNWINDER_LINK_LIBS}
+ ${SANITIZER_TEST_CXX_LIBRARIES})
# -gline-tables-only must be enough for these tests, so use it if possible.
if(COMPILER_RT_TEST_COMPILER_ID MATCHES "Clang")
RUNTIME ${INTERCEPTION_COMMON_LIB}
SOURCES ${INTERCEPTION_UNITTESTS} ${COMPILER_RT_GTEST_SOURCE}
COMPILE_DEPS ${INTERCEPTION_TEST_HEADERS}
- DEPS gtest
+ DEPS llvm_gtest
CFLAGS ${INTERCEPTION_TEST_CFLAGS_COMMON}
LINK_FLAGS ${INTERCEPTION_TEST_LINK_FLAGS_COMMON})
endmacro()
0xC3, // ret
};
-#else
+const u8 kIdentityCodeWithJumpBackwards[] = {
+ 0x89, 0xC8, // mov eax, ecx
+ 0xC3, // ret
+ 0xE9, 0xF8, 0xFF, 0xFF,
+ 0xFF, // jmp - 8
+ 0xCC, 0xCC, 0xCC, 0xCC,
+};
+const u8 kIdentityCodeWithJumpBackwardsOffset = 3;
+
+# else
const u8 kIdentityCodeWithPrologue[] = {
0x55, // push ebp
0xC3, // ret
};
-#endif
+const u8 kIdentityCodeWithJumpBackwards[] = {
+ 0x8B, 0x44, 0x24, 0x04, // mov eax,dword ptr [esp + 4]
+ 0xC3, // ret
+ 0xE9, 0xF6, 0xFF, 0xFF,
+ 0xFF, // jmp - 10
+ 0xCC, 0xCC, 0xCC, 0xCC,
+};
+const u8 kIdentityCodeWithJumpBackwardsOffset = 5;
+
+# endif
const u8 kPatchableCode1[] = {
0xB8, 0x4B, 0x00, 0x00, 0x00, // mov eax,4B
0x90, 0x90, 0x90, 0x90,
};
+const u8 kUnpatchableCode7[] = {
+ 0x33, 0xc0, // xor eax,eax
+ 0x48, 0x85, 0xd2, // test rdx,rdx
+ 0x74, 0x10, // je +16 (unpatchable)
+};
+
+const u8 kUnpatchableCode8[] = {
+ 0x48, 0x8b, 0xc1, // mov rax,rcx
+ 0x0f, 0xb7, 0x10, // movzx edx,word ptr [rax]
+ 0x48, 0x83, 0xc0, 0x02, // add rax,2
+ 0x66, 0x85, 0xd2, // test dx,dx
+ 0x75, 0xf4, // jne -12 (unpatchable)
+};
+
+const u8 kUnpatchableCode9[] = {
+ 0x4c, 0x8b, 0xc1, // mov r8,rcx
+ 0x8a, 0x01, // mov al,byte ptr [rcx]
+ 0x48, 0xff, 0xc1, // inc rcx
+ 0x84, 0xc0, // test al,al
+ 0x75, 0xf7, // jne -9 (unpatchable)
+};
+
const u8 kPatchableCode6[] = {
0x48, 0x89, 0x54, 0x24, 0xBB, // mov QWORD PTR [rsp + 0xBB], rdx
0x33, 0xC9, // xor ecx,ecx
0xC3, // ret
};
+const u8 kPatchableCode9[] = {
+ 0x8a, 0x01, // al,byte ptr [rcx]
+ 0x45, 0x33, 0xc0, // xor r8d,r8d
+ 0x84, 0xc0, // test al,al
+};
+
+const u8 kPatchableCode10[] = {
+ 0x45, 0x33, 0xc0, // xor r8d,r8d
+ 0x41, 0x8b, 0xc0, // mov eax,r8d
+ 0x48, 0x85, 0xd2, // test rdx,rdx
+};
+
+const u8 kPatchableCode11[] = {
+ 0x48, 0x83, 0xec, 0x38, // sub rsp,38h
+ 0x83, 0x64, 0x24, 0x28, 0x00, // and dword ptr [rsp+28h],0
+};
+
// A buffer holding the dynamically generated code under test.
u8* ActiveCode;
const size_t ActiveCodeLength = 4096;
EXPECT_NE(DbgPrint_adddress, isdigit_address);
}
-template<class T>
+template <class T>
static void TestIdentityFunctionPatching(
- const T &code,
- TestOverrideFunction override,
- FunctionPrefixKind prefix_kind = FunctionPrefixNone) {
+ const T &code, TestOverrideFunction override,
+ FunctionPrefixKind prefix_kind = FunctionPrefixNone,
+ int function_start_offset = 0) {
uptr identity_address;
LoadActiveCode(code, &identity_address, prefix_kind);
+ identity_address += function_start_offset;
IdentityFunction identity = (IdentityFunction)identity_address;
// Validate behavior before dynamic patching.
TestOnlyReleaseTrampolineRegions();
}
-#if !SANITIZER_WINDOWS64
+# if !SANITIZER_WINDOWS64
TEST(Interception, OverrideFunctionWithDetour) {
TestOverrideFunction override = OverrideFunctionWithDetour;
FunctionPrefixKind prefix = FunctionPrefixDetour;
TEST(Interception, OverrideFunctionWithRedirectJump) {
TestOverrideFunction override = OverrideFunctionWithRedirectJump;
TestIdentityFunctionPatching(kIdentityCodeWithJump, override);
+ TestIdentityFunctionPatching(kIdentityCodeWithJumpBackwards, override,
+ FunctionPrefixNone,
+ kIdentityCodeWithJumpBackwardsOffset);
}
TEST(Interception, OverrideFunctionWithHotPatch) {
EXPECT_TRUE(TestFunctionPatching(kPatchableCode2, override, prefix));
#if SANITIZER_WINDOWS64
EXPECT_FALSE(TestFunctionPatching(kPatchableCode3, override, prefix));
+ EXPECT_TRUE(TestFunctionPatching(kPatchableCode9, override, prefix));
+ EXPECT_TRUE(TestFunctionPatching(kPatchableCode10, override, prefix));
+ EXPECT_TRUE(TestFunctionPatching(kPatchableCode11, override, prefix));
+ EXPECT_FALSE(TestFunctionPatching(kUnpatchableCode7, override, prefix));
+ EXPECT_FALSE(TestFunctionPatching(kUnpatchableCode8, override, prefix));
+ EXPECT_FALSE(TestFunctionPatching(kUnpatchableCode9, override, prefix));
#else
EXPECT_TRUE(TestFunctionPatching(kPatchableCode3, override, prefix));
#endif
set(LSAN_CFLAGS ${SANITIZER_COMMON_CFLAGS})
append_rtti_flag(OFF LSAN_CFLAGS)
+# Too many existing bugs, needs cleanup.
+append_list_if(COMPILER_RT_HAS_WNO_FORMAT -Wno-format LSAN_CFLAGS)
+
set(LSAN_COMMON_SOURCES
lsan_common.cpp
lsan_common_fuchsia.cpp
add_compiler_rt_runtime(clang_rt.lsan
SHARED
- OS ${SANITIZER_COMMON_SUPPORTED_OS}
+ OS ${LSAN_SUPPORTED_OS}
ARCHS ${LSAN_SUPPORTED_ARCH}
SOURCES ${LSAN_SOURCES}
ADDITIONAL_HEADERS ${LSAN_HEADERS}
#include "lsan.h"
-#include "sanitizer_common/sanitizer_flags.h"
-#include "sanitizer_common/sanitizer_flag_parser.h"
#include "lsan_allocator.h"
#include "lsan_common.h"
#include "lsan_thread.h"
+#include "sanitizer_common/sanitizer_flag_parser.h"
+#include "sanitizer_common/sanitizer_flags.h"
+#include "sanitizer_common/sanitizer_interface_internal.h"
bool lsan_inited;
bool lsan_init_is_running;
InitializeThreadRegistry();
InstallDeadlySignalHandlers(LsanOnDeadlySignal);
InitializeMainThread();
-
- if (common_flags()->detect_leaks && common_flags()->leak_check_at_exit)
- Atexit(DoLeakCheck);
+ InstallAtExitCheckLeaks();
InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir);
#include "lsan_thread.h"
#if SANITIZER_POSIX
-#include "lsan_posix.h"
+# include "lsan_posix.h"
#elif SANITIZER_FUCHSIA
-#include "lsan_fuchsia.h"
+# include "lsan_fuchsia.h"
#endif
#include "sanitizer_common/sanitizer_flags.h"
#include "sanitizer_common/sanitizer_stacktrace.h"
-#define GET_STACK_TRACE(max_size, fast) \
- __sanitizer::BufferedStackTrace stack; \
- stack.Unwind(StackTrace::GetCurrentPc(), \
- GET_CURRENT_FRAME(), nullptr, fast, max_size);
+#define GET_STACK_TRACE(max_size, fast) \
+ __sanitizer::BufferedStackTrace stack; \
+ stack.Unwind(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), nullptr, fast, \
+ max_size);
#define GET_STACK_TRACE_FATAL \
GET_STACK_TRACE(kStackTraceMax, common_flags()->fast_unwind_on_fatal)
void InitializeInterceptors();
void ReplaceSystemMalloc();
void LsanOnDeadlySignal(int signo, void *siginfo, void *context);
-
-#define ENSURE_LSAN_INITED do { \
- CHECK(!lsan_init_is_running); \
- if (!lsan_inited) \
- __lsan_init(); \
-} while (0)
+void InstallAtExitCheckLeaks();
+
+#define ENSURE_LSAN_INITED \
+ do { \
+ CHECK(!lsan_init_is_running); \
+ if (!lsan_inited) \
+ __lsan_init(); \
+ } while (0)
} // namespace __lsan
namespace __lsan {
#if defined(__i386__) || defined(__arm__)
-static const uptr kMaxAllowedMallocSize = 1UL << 30;
+static const uptr kMaxAllowedMallocSize = 1ULL << 30;
#elif defined(__mips64) || defined(__aarch64__)
-static const uptr kMaxAllowedMallocSize = 4UL << 30;
+static const uptr kMaxAllowedMallocSize = 4ULL << 30;
#else
-static const uptr kMaxAllowedMallocSize = 8UL << 30;
+static const uptr kMaxAllowedMallocSize = 8ULL << 30;
#endif
static Allocator allocator;
size = 1;
if (size > max_malloc_size)
return ReportAllocationSizeTooBig(size, stack);
+ if (UNLIKELY(IsRssLimitExceeded())) {
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ ReportRssLimitExceeded(&stack);
+ }
void *p = allocator.Allocate(GetAllocatorCache(), size, alignment);
if (UNLIKELY(!p)) {
SetAllocatorOutOfMemory();
if (cleared && allocator.FromPrimary(p))
memset(p, 0, size);
RegisterAllocation(stack, p, size);
- if (&__sanitizer_malloc_hook) __sanitizer_malloc_hook(p, size);
RunMallocHooks(p, size);
return p;
}
}
void Deallocate(void *p) {
- if (&__sanitizer_free_hook) __sanitizer_free_hook(p);
RunFreeHooks(p);
RegisterDeallocation(p);
allocator.Deallocate(GetAllocatorCache(), p);
}
uptr GetMallocUsableSize(const void *p) {
+ if (!p)
+ return 0;
ChunkMetadata *m = Metadata(p);
if (!m) return 0;
return m->requested_size;
}
}
-void GetAdditionalThreadContextPtrs(ThreadContextBase *tctx, void *ptrs) {
+void GetAdditionalThreadContextPtrsLocked(InternalMmapVector<uptr> *ptrs) {
// This function can be used to treat memory reachable from `tctx` as live.
// This is useful for threads that have been created but not yet started.
return GetMallocUsableSize(p);
}
-#if !SANITIZER_SUPPORTS_WEAK_HOOKS
-// Provide default (no-op) implementation of malloc hooks.
-SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-void __sanitizer_malloc_hook(void *ptr, uptr size) {
- (void)ptr;
- (void)size;
-}
-SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-void __sanitizer_free_hook(void *ptr) {
- (void)ptr;
-}
-#endif
} // extern "C"
u32 stack_trace_id;
};
-#if defined(__mips64) || defined(__aarch64__) || defined(__i386__) || \
- defined(__arm__) || SANITIZER_RISCV64
+#if !SANITIZER_CAN_USE_ALLOCATOR64
template <typename AddressSpaceViewTy>
struct AP32 {
static const uptr kSpaceBeg = 0;
template <typename AddressSpaceView>
using PrimaryAllocatorASVT = SizeClassAllocator32<AP32<AddressSpaceView>>;
using PrimaryAllocator = PrimaryAllocatorASVT<LocalAddressSpaceView>;
-#elif defined(__x86_64__) || defined(__powerpc64__) || defined(__s390x__)
-# if SANITIZER_FUCHSIA
+#else
+# if SANITIZER_FUCHSIA || defined(__powerpc64__)
const uptr kAllocatorSpace = ~(uptr)0;
const uptr kAllocatorSize = 0x40000000000ULL; // 4T.
-# elif defined(__powerpc64__)
-const uptr kAllocatorSpace = 0xa0000000000ULL;
-const uptr kAllocatorSize = 0x20000000000ULL; // 2T.
#elif defined(__s390x__)
const uptr kAllocatorSpace = 0x40000000000ULL;
const uptr kAllocatorSize = 0x40000000000ULL; // 4T.
#include "sanitizer_common/sanitizer_tls_get_addr.h"
#if CAN_SANITIZE_LEAKS
+
+# if SANITIZER_APPLE
+// https://github.com/apple-oss-distributions/objc4/blob/8701d5672d3fd3cd817aeb84db1077aafe1a1604/runtime/objc-runtime-new.h#L127
+# if SANITIZER_IOS && !SANITIZER_IOSSIM
+# define OBJC_DATA_MASK 0x0000007ffffffff8UL
+# else
+# define OBJC_DATA_MASK 0x00007ffffffffff8UL
+# endif
+// https://github.com/apple-oss-distributions/objc4/blob/8701d5672d3fd3cd817aeb84db1077aafe1a1604/runtime/objc-runtime-new.h#L139
+# define OBJC_FAST_IS_RW 0x8000000000000000UL
+# endif
+
namespace __lsan {
// This mutex is used to prevent races between DoLeakCheck and IgnoreObject, and
// also to protect the global list of root regions.
-BlockingMutex global_mutex(LINKER_INITIALIZED);
+Mutex global_mutex;
Flags lsan_flags;
-
void DisableCounterUnderflow() {
if (common_flags()->detect_leaks) {
Report("Unmatched call to __lsan_enable().\n");
}
void Flags::SetDefaults() {
-#define LSAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue;
-#include "lsan_flags.inc"
-#undef LSAN_FLAG
+# define LSAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue;
+# include "lsan_flags.inc"
+# undef LSAN_FLAG
}
void RegisterLsanFlags(FlagParser *parser, Flags *f) {
-#define LSAN_FLAG(Type, Name, DefaultValue, Description) \
- RegisterFlag(parser, #Name, Description, &f->Name);
-#include "lsan_flags.inc"
-#undef LSAN_FLAG
+# define LSAN_FLAG(Type, Name, DefaultValue, Description) \
+ RegisterFlag(parser, #Name, Description, &f->Name);
+# include "lsan_flags.inc"
+# undef LSAN_FLAG
}
-#define LOG_POINTERS(...) \
- do { \
- if (flags()->log_pointers) Report(__VA_ARGS__); \
- } while (0)
+# define LOG_POINTERS(...) \
+ do { \
+ if (flags()->log_pointers) \
+ Report(__VA_ARGS__); \
+ } while (0)
-#define LOG_THREADS(...) \
- do { \
- if (flags()->log_threads) Report(__VA_ARGS__); \
- } while (0)
+# define LOG_THREADS(...) \
+ do { \
+ if (flags()->log_threads) \
+ Report(__VA_ARGS__); \
+ } while (0)
class LeakSuppressionContext {
bool parsed = false;
SuppressionContext context;
bool suppressed_stacks_sorted = true;
InternalMmapVector<u32> suppressed_stacks;
+ const LoadedModule *suppress_module = nullptr;
- Suppression *GetSuppressionForAddr(uptr addr);
void LazyInit();
+ Suppression *GetSuppressionForAddr(uptr addr);
+ bool SuppressInvalid(const StackTrace &stack);
+ bool SuppressByRule(const StackTrace &stack, uptr hit_count, uptr total_size);
public:
LeakSuppressionContext(const char *supprression_types[],
int suppression_types_num)
: context(supprression_types, suppression_types_num) {}
- Suppression *GetSuppressionForStack(u32 stack_trace_id);
+ bool Suppress(u32 stack_trace_id, uptr hit_count, uptr total_size);
const InternalMmapVector<u32> &GetSortedSuppressedStacks() {
if (!suppressed_stacks_sorted) {
ALIGNED(64) static char suppression_placeholder[sizeof(LeakSuppressionContext)];
static LeakSuppressionContext *suppression_ctx = nullptr;
static const char kSuppressionLeak[] = "leak";
-static const char *kSuppressionTypes[] = { kSuppressionLeak };
+static const char *kSuppressionTypes[] = {kSuppressionLeak};
static const char kStdSuppressions[] =
-#if SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT
+# if SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT
// For more details refer to the SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT
// definition.
"leak:*pthread_exit*\n"
-#endif // SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT
-#if SANITIZER_MAC
+# endif // SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT
+# if SANITIZER_APPLE
// For Darwin and os_log/os_trace: https://reviews.llvm.org/D35173
"leak:*_os_trace*\n"
-#endif
+# endif
// TLS leak in some glibc versions, described in
// https://sourceware.org/bugzilla/show_bug.cgi?id=12650.
"leak:*tls_get_addr*\n";
if (&__lsan_default_suppressions)
context.Parse(__lsan_default_suppressions());
context.Parse(kStdSuppressions);
+ if (flags()->use_tls && flags()->use_ld_allocations)
+ suppress_module = GetLinker();
+ }
+}
+
+Suppression *LeakSuppressionContext::GetSuppressionForAddr(uptr addr) {
+ Suppression *s = nullptr;
+
+ // Suppress by module name.
+ const char *module_name = Symbolizer::GetOrInit()->GetModuleNameForPc(addr);
+ if (!module_name)
+ module_name = "<unknown module>";
+ if (context.Match(module_name, kSuppressionLeak, &s))
+ return s;
+
+ // Suppress by file or function name.
+ SymbolizedStack *frames = Symbolizer::GetOrInit()->SymbolizePC(addr);
+ for (SymbolizedStack *cur = frames; cur; cur = cur->next) {
+ if (context.Match(cur->info.function, kSuppressionLeak, &s) ||
+ context.Match(cur->info.file, kSuppressionLeak, &s)) {
+ break;
+ }
}
+ frames->ClearAll();
+ return s;
+}
+
+static uptr GetCallerPC(const StackTrace &stack) {
+ // The top frame is our malloc/calloc/etc. The next frame is the caller.
+ if (stack.size >= 2)
+ return stack.trace[1];
+ return 0;
+}
+
+# if SANITIZER_APPLE
+// Objective-C class data pointers are stored with flags in the low bits, so
+// they need to be transformed back into something that looks like a pointer.
+static inline void *MaybeTransformPointer(void *p) {
+ uptr ptr = reinterpret_cast<uptr>(p);
+ if ((ptr & OBJC_FAST_IS_RW) == OBJC_FAST_IS_RW)
+ ptr &= OBJC_DATA_MASK;
+ return reinterpret_cast<void *>(ptr);
+}
+# endif
+
+// On Linux, treats all chunks allocated from ld-linux.so as reachable, which
+// covers dynamically allocated TLS blocks, internal dynamic loader's loaded
+// modules accounting etc.
+// Dynamic TLS blocks contain the TLS variables of dynamically loaded modules.
+// They are allocated with a __libc_memalign() call in allocate_and_init()
+// (elf/dl-tls.c). Glibc won't tell us the address ranges occupied by those
+// blocks, but we can make sure they come from our own allocator by intercepting
+// __libc_memalign(). On top of that, there is no easy way to reach them. Their
+// addresses are stored in a dynamically allocated array (the DTV) which is
+// referenced from the static TLS. Unfortunately, we can't just rely on the DTV
+// being reachable from the static TLS, and the dynamic TLS being reachable from
+// the DTV. This is because the initial DTV is allocated before our interception
+// mechanism kicks in, and thus we don't recognize it as allocated memory. We
+// can't special-case it either, since we don't know its size.
+// Our solution is to include in the root set all allocations made from
+// ld-linux.so (which is where allocate_and_init() is implemented). This is
+// guaranteed to include all dynamic TLS blocks (and possibly other allocations
+// which we don't care about).
+// On all other platforms, this simply checks to ensure that the caller pc is
+// valid before reporting chunks as leaked.
+bool LeakSuppressionContext::SuppressInvalid(const StackTrace &stack) {
+ uptr caller_pc = GetCallerPC(stack);
+ // If caller_pc is unknown, this chunk may be allocated in a coroutine. Mark
+ // it as reachable, as we can't properly report its allocation stack anyway.
+ return !caller_pc ||
+ (suppress_module && suppress_module->containsAddress(caller_pc));
+}
+
+bool LeakSuppressionContext::SuppressByRule(const StackTrace &stack,
+ uptr hit_count, uptr total_size) {
+ for (uptr i = 0; i < stack.size; i++) {
+ Suppression *s = GetSuppressionForAddr(
+ StackTrace::GetPreviousInstructionPc(stack.trace[i]));
+ if (s) {
+ s->weight += total_size;
+ atomic_fetch_add(&s->hit_count, hit_count, memory_order_relaxed);
+ return true;
+ }
+ }
+ return false;
+}
+
+bool LeakSuppressionContext::Suppress(u32 stack_trace_id, uptr hit_count,
+ uptr total_size) {
+ LazyInit();
+ StackTrace stack = StackDepotGet(stack_trace_id);
+ if (!SuppressInvalid(stack) && !SuppressByRule(stack, hit_count, total_size))
+ return false;
+ suppressed_stacks_sorted = false;
+ suppressed_stacks.push_back(stack_trace_id);
+ return true;
}
static LeakSuppressionContext *GetSuppressionContext() {
return suppression_ctx;
}
-static InternalMmapVector<RootRegion> *root_regions;
+static InternalMmapVectorNoCtor<RootRegion> root_regions;
-InternalMmapVector<RootRegion> const *GetRootRegions() { return root_regions; }
-
-void InitializeRootRegions() {
- CHECK(!root_regions);
- ALIGNED(64) static char placeholder[sizeof(InternalMmapVector<RootRegion>)];
- root_regions = new (placeholder) InternalMmapVector<RootRegion>();
+InternalMmapVectorNoCtor<RootRegion> const *GetRootRegions() {
+ return &root_regions;
}
void InitCommonLsan() {
- InitializeRootRegions();
if (common_flags()->detect_leaks) {
// Initialization which can fail or print warnings should only be done if
// LSan is actually enabled.
}
}
-class Decorator: public __sanitizer::SanitizerCommonDecorator {
+class Decorator : public __sanitizer::SanitizerCommonDecorator {
public:
- Decorator() : SanitizerCommonDecorator() { }
+ Decorator() : SanitizerCommonDecorator() {}
const char *Error() { return Red(); }
const char *Leak() { return Blue(); }
};
-static inline bool CanBeAHeapPointer(uptr p) {
+static inline bool MaybeUserPointer(uptr p) {
// Since our heap is located in mmap-ed memory, we can assume a sensible lower
// bound on heap addresses.
const uptr kMinAddress = 4 * 4096;
- if (p < kMinAddress) return false;
-#if defined(__x86_64__)
+ if (p < kMinAddress)
+ return false;
+# if defined(__x86_64__)
// Accept only canonical form user-space addresses.
return ((p >> 47) == 0);
-#elif defined(__mips64)
+# elif defined(__mips64)
return ((p >> 40) == 0);
-#elif defined(__aarch64__)
- unsigned runtimeVMA =
- (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1);
- return ((p >> runtimeVMA) == 0);
-#else
+# elif defined(__aarch64__)
+ // Accept up to 48 bit VMA.
+ return ((p >> 48) == 0);
+# elif defined(__loongarch_lp64)
+ // Allow 47-bit user-space VMA at current.
+ return ((p >> 47) == 0);
+# else
return true;
-#endif
+# endif
}
// Scans the memory range, looking for byte patterns that point into allocator
// (|tag| = kReachable) and finding indirectly leaked chunks
// (|tag| = kIndirectlyLeaked). In the second case, there's no flood fill,
// so |frontier| = 0.
-void ScanRangeForPointers(uptr begin, uptr end,
- Frontier *frontier,
+void ScanRangeForPointers(uptr begin, uptr end, Frontier *frontier,
const char *region_type, ChunkTag tag) {
CHECK(tag == kReachable || tag == kIndirectlyLeaked);
const uptr alignment = flags()->pointer_alignment();
- LOG_POINTERS("Scanning %s range %p-%p.\n", region_type, begin, end);
+ LOG_POINTERS("Scanning %s range %p-%p.\n", region_type, (void *)begin,
+ (void *)end);
uptr pp = begin;
if (pp % alignment)
pp = pp + alignment - pp % alignment;
for (; pp + sizeof(void *) <= end; pp += alignment) {
void *p = *reinterpret_cast<void **>(pp);
- if (!CanBeAHeapPointer(reinterpret_cast<uptr>(p))) continue;
+# if SANITIZER_APPLE
+ p = MaybeTransformPointer(p);
+# endif
+ if (!MaybeUserPointer(reinterpret_cast<uptr>(p)))
+ continue;
uptr chunk = PointsIntoChunk(p);
- if (!chunk) continue;
+ if (!chunk)
+ continue;
// Pointers to self don't count. This matters when tag == kIndirectlyLeaked.
- if (chunk == begin) continue;
+ if (chunk == begin)
+ continue;
LsanMetadata m(chunk);
- if (m.tag() == kReachable || m.tag() == kIgnored) continue;
+ if (m.tag() == kReachable || m.tag() == kIgnored)
+ continue;
// Do this check relatively late so we can log only the interesting cases.
if (!flags()->use_poisoned && WordIsPoisoned(pp)) {
LOG_POINTERS(
"%p is poisoned: ignoring %p pointing into chunk %p-%p of size "
"%zu.\n",
- pp, p, chunk, chunk + m.requested_size(), m.requested_size());
+ (void *)pp, p, (void *)chunk, (void *)(chunk + m.requested_size()),
+ m.requested_size());
continue;
}
m.set_tag(tag);
- LOG_POINTERS("%p: found %p pointing into chunk %p-%p of size %zu.\n", pp, p,
- chunk, chunk + m.requested_size(), m.requested_size());
+ LOG_POINTERS("%p: found %p pointing into chunk %p-%p of size %zu.\n",
+ (void *)pp, p, (void *)chunk,
+ (void *)(chunk + m.requested_size()), m.requested_size());
if (frontier)
frontier->push_back(chunk);
}
}
}
-void ForEachExtraStackRangeCb(uptr begin, uptr end, void* arg) {
- Frontier *frontier = reinterpret_cast<Frontier *>(arg);
- ScanRangeForPointers(begin, end, frontier, "FAKE STACK", kReachable);
+void ScanExtraStackRanges(const InternalMmapVector<Range> &ranges,
+ Frontier *frontier) {
+ for (uptr i = 0; i < ranges.size(); i++) {
+ ScanRangeForPointers(ranges[i].begin, ranges[i].end, frontier, "FAKE STACK",
+ kReachable);
+ }
}
-#if SANITIZER_FUCHSIA
+# if SANITIZER_FUCHSIA
// Fuchsia handles all threads together with its own callback.
-static void ProcessThreads(SuspendedThreadsList const &, Frontier *) {}
+static void ProcessThreads(SuspendedThreadsList const &, Frontier *, tid_t,
+ uptr) {}
-#else
+# else
-#if SANITIZER_ANDROID
+# if SANITIZER_ANDROID
// FIXME: Move this out into *libcdep.cpp
extern "C" SANITIZER_WEAK_ATTRIBUTE void __libc_iterate_dynamic_tls(
pid_t, void (*cb)(void *, void *, uptr, void *), void *);
-#endif
+# endif
static void ProcessThreadRegistry(Frontier *frontier) {
InternalMmapVector<uptr> ptrs;
- GetThreadRegistryLocked()->RunCallbackForEachThreadLocked(
- GetAdditionalThreadContextPtrs, &ptrs);
+ GetAdditionalThreadContextPtrsLocked(&ptrs);
for (uptr i = 0; i < ptrs.size(); ++i) {
void *ptr = reinterpret_cast<void *>(ptrs[i]);
// Scans thread data (stacks and TLS) for heap pointers.
static void ProcessThreads(SuspendedThreadsList const &suspended_threads,
- Frontier *frontier) {
+ Frontier *frontier, tid_t caller_tid,
+ uptr caller_sp) {
InternalMmapVector<uptr> registers;
+ InternalMmapVector<Range> extra_ranges;
for (uptr i = 0; i < suspended_threads.ThreadCount(); i++) {
tid_t os_id = static_cast<tid_t>(suspended_threads.GetThreadID(i));
- LOG_THREADS("Processing thread %d.\n", os_id);
+ LOG_THREADS("Processing thread %llu.\n", os_id);
uptr stack_begin, stack_end, tls_begin, tls_end, cache_begin, cache_end;
DTLS *dtls;
- bool thread_found = GetThreadRangesLocked(os_id, &stack_begin, &stack_end,
- &tls_begin, &tls_end,
- &cache_begin, &cache_end, &dtls);
+ bool thread_found =
+ GetThreadRangesLocked(os_id, &stack_begin, &stack_end, &tls_begin,
+ &tls_end, &cache_begin, &cache_end, &dtls);
if (!thread_found) {
// If a thread can't be found in the thread registry, it's probably in the
// process of destruction. Log this event and move on.
- LOG_THREADS("Thread %d not found in registry.\n", os_id);
+ LOG_THREADS("Thread %llu not found in registry.\n", os_id);
continue;
}
uptr sp;
PtraceRegistersStatus have_registers =
suspended_threads.GetRegistersAndSP(i, ®isters, &sp);
if (have_registers != REGISTERS_AVAILABLE) {
- Report("Unable to get registers from thread %d.\n", os_id);
+ Report("Unable to get registers from thread %llu.\n", os_id);
// If unable to get SP, consider the entire stack to be reachable unless
// GetRegistersAndSP failed with ESRCH.
- if (have_registers == REGISTERS_UNAVAILABLE_FATAL) continue;
+ if (have_registers == REGISTERS_UNAVAILABLE_FATAL)
+ continue;
sp = stack_begin;
}
+ if (suspended_threads.GetThreadID(i) == caller_tid) {
+ sp = caller_sp;
+ }
if (flags()->use_registers && have_registers) {
uptr registers_begin = reinterpret_cast<uptr>(registers.data());
}
if (flags()->use_stacks) {
- LOG_THREADS("Stack at %p-%p (SP = %p).\n", stack_begin, stack_end, sp);
+ LOG_THREADS("Stack at %p-%p (SP = %p).\n", (void *)stack_begin,
+ (void *)stack_end, (void *)sp);
if (sp < stack_begin || sp >= stack_end) {
// SP is outside the recorded stack range (e.g. the thread is running a
// signal handler on alternate stack, or swapcontext was used).
stack_begin += page_size;
}
LOG_THREADS("Skipped %d guard page(s) to obtain stack %p-%p.\n",
- skipped, stack_begin, stack_end);
+ skipped, (void *)stack_begin, (void *)stack_end);
} else {
// Shrink the stack range to ignore out-of-scope values.
stack_begin = sp;
}
ScanRangeForPointers(stack_begin, stack_end, frontier, "STACK",
kReachable);
- ForEachExtraStackRange(os_id, ForEachExtraStackRangeCb, frontier);
+ extra_ranges.clear();
+ GetThreadExtraStackRangesLocked(os_id, &extra_ranges);
+ ScanExtraStackRanges(extra_ranges, frontier);
}
if (flags()->use_tls) {
if (tls_begin) {
- LOG_THREADS("TLS at %p-%p.\n", tls_begin, tls_end);
+ LOG_THREADS("TLS at %p-%p.\n", (void *)tls_begin, (void *)tls_end);
// If the tls and cache ranges don't overlap, scan full tls range,
// otherwise, only scan the non-overlapping portions
if (cache_begin == cache_end || tls_end < cache_begin ||
kReachable);
}
}
-#if SANITIZER_ANDROID
+# if SANITIZER_ANDROID
auto *cb = +[](void *dtls_begin, void *dtls_end, uptr /*dso_idd*/,
void *arg) -> void {
ScanRangeForPointers(reinterpret_cast<uptr>(dtls_begin),
// thread is suspended in the middle of updating its DTLS. IOWs, we
// could scan already freed memory. (probably fine for now)
__libc_iterate_dynamic_tls(os_id, cb, frontier);
-#else
+# else
if (dtls && !DTLSInDestruction(dtls)) {
ForEachDVT(dtls, [&](const DTLS::DTV &dtv, int id) {
uptr dtls_beg = dtv.beg;
uptr dtls_end = dtls_beg + dtv.size;
if (dtls_beg < dtls_end) {
- LOG_THREADS("DTLS %zu at %p-%p.\n", id, dtls_beg, dtls_end);
+ LOG_THREADS("DTLS %d at %p-%p.\n", id, (void *)dtls_beg,
+ (void *)dtls_end);
ScanRangeForPointers(dtls_beg, dtls_end, frontier, "DTLS",
kReachable);
}
} else {
// We are handling a thread with DTLS under destruction. Log about
// this and continue.
- LOG_THREADS("Thread %d has DTLS under destruction.\n", os_id);
+ LOG_THREADS("Thread %llu has DTLS under destruction.\n", os_id);
}
-#endif
+# endif
}
}
ProcessThreadRegistry(frontier);
}
-#endif // SANITIZER_FUCHSIA
+# endif // SANITIZER_FUCHSIA
void ScanRootRegion(Frontier *frontier, const RootRegion &root_region,
uptr region_begin, uptr region_end, bool is_readable) {
uptr intersection_begin = Max(root_region.begin, region_begin);
uptr intersection_end = Min(region_end, root_region.begin + root_region.size);
- if (intersection_begin >= intersection_end) return;
+ if (intersection_begin >= intersection_end)
+ return;
LOG_POINTERS("Root region %p-%p intersects with mapped region %p-%p (%s)\n",
- root_region.begin, root_region.begin + root_region.size,
- region_begin, region_end,
+ (void *)root_region.begin,
+ (void *)(root_region.begin + root_region.size),
+ (void *)region_begin, (void *)region_end,
is_readable ? "readable" : "unreadable");
if (is_readable)
ScanRangeForPointers(intersection_begin, intersection_end, frontier, "ROOT",
// Scans root regions for heap pointers.
static void ProcessRootRegions(Frontier *frontier) {
- if (!flags()->use_root_regions) return;
- CHECK(root_regions);
- for (uptr i = 0; i < root_regions->size(); i++) {
- ProcessRootRegion(frontier, (*root_regions)[i]);
- }
+ if (!flags()->use_root_regions)
+ return;
+ for (uptr i = 0; i < root_regions.size(); i++)
+ ProcessRootRegion(frontier, root_regions[i]);
}
static void FloodFillTag(Frontier *frontier, ChunkTag tag) {
if (idx >= suppressed.size() || m.stack_trace_id() != suppressed[idx])
return;
- LOG_POINTERS("Suppressed: chunk %p-%p of size %zu.\n", chunk,
- chunk + m.requested_size(), m.requested_size());
+ LOG_POINTERS("Suppressed: chunk %p-%p of size %zu.\n", (void *)chunk,
+ (void *)(chunk + m.requested_size()), m.requested_size());
m.set_tag(kIgnored);
}
chunk = GetUserBegin(chunk);
LsanMetadata m(chunk);
if (m.allocated() && m.tag() == kIgnored) {
- LOG_POINTERS("Ignored: chunk %p-%p of size %zu.\n",
- chunk, chunk + m.requested_size(), m.requested_size());
+ LOG_POINTERS("Ignored: chunk %p-%p of size %zu.\n", (void *)chunk,
+ (void *)(chunk + m.requested_size()), m.requested_size());
reinterpret_cast<Frontier *>(arg)->push_back(chunk);
}
}
-static uptr GetCallerPC(u32 stack_id, StackDepotReverseMap *map) {
- CHECK(stack_id);
- StackTrace stack = map->Get(stack_id);
- // The top frame is our malloc/calloc/etc. The next frame is the caller.
- if (stack.size >= 2)
- return stack.trace[1];
- return 0;
-}
-
-struct InvalidPCParam {
- Frontier *frontier;
- StackDepotReverseMap *stack_depot_reverse_map;
- bool skip_linker_allocations;
-};
-
-// ForEachChunk callback. If the caller pc is invalid or is within the linker,
-// mark as reachable. Called by ProcessPlatformSpecificAllocations.
-static void MarkInvalidPCCb(uptr chunk, void *arg) {
- CHECK(arg);
- InvalidPCParam *param = reinterpret_cast<InvalidPCParam *>(arg);
- chunk = GetUserBegin(chunk);
- LsanMetadata m(chunk);
- if (m.allocated() && m.tag() != kReachable && m.tag() != kIgnored) {
- u32 stack_id = m.stack_trace_id();
- uptr caller_pc = 0;
- if (stack_id > 0)
- caller_pc = GetCallerPC(stack_id, param->stack_depot_reverse_map);
- // If caller_pc is unknown, this chunk may be allocated in a coroutine. Mark
- // it as reachable, as we can't properly report its allocation stack anyway.
- if (caller_pc == 0 || (param->skip_linker_allocations &&
- GetLinker()->containsAddress(caller_pc))) {
- m.set_tag(kReachable);
- param->frontier->push_back(chunk);
- }
- }
-}
-
-// On Linux, treats all chunks allocated from ld-linux.so as reachable, which
-// covers dynamically allocated TLS blocks, internal dynamic loader's loaded
-// modules accounting etc.
-// Dynamic TLS blocks contain the TLS variables of dynamically loaded modules.
-// They are allocated with a __libc_memalign() call in allocate_and_init()
-// (elf/dl-tls.c). Glibc won't tell us the address ranges occupied by those
-// blocks, but we can make sure they come from our own allocator by intercepting
-// __libc_memalign(). On top of that, there is no easy way to reach them. Their
-// addresses are stored in a dynamically allocated array (the DTV) which is
-// referenced from the static TLS. Unfortunately, we can't just rely on the DTV
-// being reachable from the static TLS, and the dynamic TLS being reachable from
-// the DTV. This is because the initial DTV is allocated before our interception
-// mechanism kicks in, and thus we don't recognize it as allocated memory. We
-// can't special-case it either, since we don't know its size.
-// Our solution is to include in the root set all allocations made from
-// ld-linux.so (which is where allocate_and_init() is implemented). This is
-// guaranteed to include all dynamic TLS blocks (and possibly other allocations
-// which we don't care about).
-// On all other platforms, this simply checks to ensure that the caller pc is
-// valid before reporting chunks as leaked.
-void ProcessPC(Frontier *frontier) {
- StackDepotReverseMap stack_depot_reverse_map;
- InvalidPCParam arg;
- arg.frontier = frontier;
- arg.stack_depot_reverse_map = &stack_depot_reverse_map;
- arg.skip_linker_allocations =
- flags()->use_tls && flags()->use_ld_allocations && GetLinker() != nullptr;
- ForEachChunk(MarkInvalidPCCb, &arg);
-}
-
// Sets the appropriate tag on each chunk.
static void ClassifyAllChunks(SuspendedThreadsList const &suspended_threads,
- Frontier *frontier) {
+ Frontier *frontier, tid_t caller_tid,
+ uptr caller_sp) {
const InternalMmapVector<u32> &suppressed_stacks =
GetSuppressionContext()->GetSortedSuppressedStacks();
if (!suppressed_stacks.empty()) {
}
ForEachChunk(CollectIgnoredCb, frontier);
ProcessGlobalRegions(frontier);
- ProcessThreads(suspended_threads, frontier);
+ ProcessThreads(suspended_threads, frontier, caller_tid, caller_sp);
ProcessRootRegions(frontier);
FloodFillTag(frontier, kReachable);
- CHECK_EQ(0, frontier->size());
- ProcessPC(frontier);
-
// The check here is relatively expensive, so we do this in a separate flood
// fill. That way we can skip the check for chunks that are reachable
// otherwise.
m.set_tag(kDirectlyLeaked);
}
-static void PrintStackTraceById(u32 stack_trace_id) {
- CHECK(stack_trace_id);
- StackDepotGet(stack_trace_id).Print();
-}
-
// ForEachChunk callback. Aggregates information about unreachable chunks into
// a LeakReport.
static void CollectLeaksCb(uptr chunk, void *arg) {
CHECK(arg);
- LeakReport *leak_report = reinterpret_cast<LeakReport *>(arg);
+ LeakedChunks *leaks = reinterpret_cast<LeakedChunks *>(arg);
chunk = GetUserBegin(chunk);
LsanMetadata m(chunk);
- if (!m.allocated()) return;
- if (m.tag() == kDirectlyLeaked || m.tag() == kIndirectlyLeaked) {
- u32 resolution = flags()->resolution;
- u32 stack_trace_id = 0;
- if (resolution > 0) {
- StackTrace stack = StackDepotGet(m.stack_trace_id());
- stack.size = Min(stack.size, resolution);
- stack_trace_id = StackDepotPut(stack);
- } else {
- stack_trace_id = m.stack_trace_id();
- }
- leak_report->AddLeakedChunk(chunk, stack_trace_id, m.requested_size(),
- m.tag());
- }
+ if (!m.allocated())
+ return;
+ if (m.tag() == kDirectlyLeaked || m.tag() == kIndirectlyLeaked)
+ leaks->push_back({chunk, m.stack_trace_id(), m.requested_size(), m.tag()});
}
void LeakSuppressionContext::PrintMatchedSuppressions() {
Printf("%s\n\n", line);
}
-static void ReportIfNotSuspended(ThreadContextBase *tctx, void *arg) {
- const InternalMmapVector<tid_t> &suspended_threads =
- *(const InternalMmapVector<tid_t> *)arg;
- if (tctx->status == ThreadStatusRunning) {
- uptr i = InternalLowerBound(suspended_threads, tctx->os_id);
- if (i >= suspended_threads.size() || suspended_threads[i] != tctx->os_id)
- Report("Running thread %d was not suspended. False leaks are possible.\n",
- tctx->os_id);
- }
-}
-
-#if SANITIZER_FUCHSIA
+# if SANITIZER_FUCHSIA
// Fuchsia provides a libc interface that guarantees all threads are
// covered, and SuspendedThreadList is never really used.
static void ReportUnsuspendedThreads(const SuspendedThreadsList &) {}
-#else // !SANITIZER_FUCHSIA
+# else // !SANITIZER_FUCHSIA
static void ReportUnsuspendedThreads(
const SuspendedThreadsList &suspended_threads) {
Sort(threads.data(), threads.size());
- GetThreadRegistryLocked()->RunCallbackForEachThreadLocked(
- &ReportIfNotSuspended, &threads);
+ InternalMmapVector<tid_t> unsuspended;
+ GetRunningThreadsLocked(&unsuspended);
+
+ for (auto os_id : unsuspended) {
+ uptr i = InternalLowerBound(threads, os_id);
+ if (i >= threads.size() || threads[i] != os_id)
+ Report(
+ "Running thread %zu was not suspended. False leaks are possible.\n",
+ os_id);
+ }
}
-#endif // !SANITIZER_FUCHSIA
+# endif // !SANITIZER_FUCHSIA
static void CheckForLeaksCallback(const SuspendedThreadsList &suspended_threads,
void *arg) {
CHECK(param);
CHECK(!param->success);
ReportUnsuspendedThreads(suspended_threads);
- ClassifyAllChunks(suspended_threads, ¶m->frontier);
- ForEachChunk(CollectLeaksCb, ¶m->leak_report);
+ ClassifyAllChunks(suspended_threads, ¶m->frontier, param->caller_tid,
+ param->caller_sp);
+ ForEachChunk(CollectLeaksCb, ¶m->leaks);
// Clean up for subsequent leak checks. This assumes we did not overwrite any
// kIgnored tags.
ForEachChunk(ResetTagsCb, nullptr);
}
static bool CheckForLeaks() {
- if (&__lsan_is_turned_off && __lsan_is_turned_off())
+ if (&__lsan_is_turned_off && __lsan_is_turned_off()) {
+ VReport(1, "LeakSanitizer is disabled");
return false;
+ }
+ VReport(1, "LeakSanitizer: checking for leaks");
// Inside LockStuffAndStopTheWorld we can't run symbolizer, so we can't match
// suppressions. However if a stack id was previously suppressed, it should be
// suppressed in future checks as well.
for (int i = 0;; ++i) {
EnsureMainThreadIDIsCorrect();
CheckForLeaksParam param;
+ // Capture calling thread's stack pointer early, to avoid false negatives.
+ // Old frame with dead pointers might be overlapped by new frame inside
+ // CheckForLeaks which does not use bytes with pointers before the
+ // threads are suspended and stack pointers captured.
+ param.caller_tid = GetTid();
+ param.caller_sp = reinterpret_cast<uptr>(__builtin_frame_address(0));
LockStuffAndStopTheWorld(CheckForLeaksCallback, ¶m);
if (!param.success) {
Report("LeakSanitizer has encountered a fatal error.\n");
"etc)\n");
Die();
}
+ LeakReport leak_report;
+ leak_report.AddLeakedChunks(param.leaks);
+
// No new suppressions stacks, so rerun will not help and we can report.
- if (!param.leak_report.ApplySuppressions())
- return PrintResults(param.leak_report);
+ if (!leak_report.ApplySuppressions())
+ return PrintResults(leak_report);
// No indirect leaks to report, so we are done here.
- if (!param.leak_report.IndirectUnsuppressedLeakCount())
- return PrintResults(param.leak_report);
+ if (!leak_report.IndirectUnsuppressedLeakCount())
+ return PrintResults(leak_report);
if (i >= 8) {
Report("WARNING: LeakSanitizer gave up on indirect leaks suppression.\n");
- return PrintResults(param.leak_report);
+ return PrintResults(leak_report);
}
// We found a new previously unseen suppressed call stack. Rerun to make
bool HasReportedLeaks() { return has_reported_leaks; }
void DoLeakCheck() {
- BlockingMutexLock l(&global_mutex);
+ Lock l(&global_mutex);
static bool already_done;
- if (already_done) return;
+ if (already_done)
+ return;
already_done = true;
has_reported_leaks = CheckForLeaks();
- if (has_reported_leaks) HandleLeaks();
+ if (has_reported_leaks)
+ HandleLeaks();
}
static int DoRecoverableLeakCheck() {
- BlockingMutexLock l(&global_mutex);
+ Lock l(&global_mutex);
bool have_leaks = CheckForLeaks();
return have_leaks ? 1 : 0;
}
void DoRecoverableLeakCheckVoid() { DoRecoverableLeakCheck(); }
-Suppression *LeakSuppressionContext::GetSuppressionForAddr(uptr addr) {
- Suppression *s = nullptr;
-
- // Suppress by module name.
- if (const char *module_name =
- Symbolizer::GetOrInit()->GetModuleNameForPc(addr))
- if (context.Match(module_name, kSuppressionLeak, &s))
- return s;
-
- // Suppress by file or function name.
- SymbolizedStack *frames = Symbolizer::GetOrInit()->SymbolizePC(addr);
- for (SymbolizedStack *cur = frames; cur; cur = cur->next) {
- if (context.Match(cur->info.function, kSuppressionLeak, &s) ||
- context.Match(cur->info.file, kSuppressionLeak, &s)) {
- break;
- }
- }
- frames->ClearAll();
- return s;
-}
-
-Suppression *LeakSuppressionContext::GetSuppressionForStack(
- u32 stack_trace_id) {
- LazyInit();
- StackTrace stack = StackDepotGet(stack_trace_id);
- for (uptr i = 0; i < stack.size; i++) {
- Suppression *s = GetSuppressionForAddr(
- StackTrace::GetPreviousInstructionPc(stack.trace[i]));
- if (s) {
- suppressed_stacks_sorted = false;
- suppressed_stacks.push_back(stack_trace_id);
- return s;
- }
- }
- return nullptr;
-}
-
///// LeakReport implementation. /////
// A hard limit on the number of distinct leaks, to avoid quadratic complexity
// in LeakReport::AddLeakedChunk(). We don't expect to ever see this many leaks
// in real-world applications.
-// FIXME: Get rid of this limit by changing the implementation of LeakReport to
-// use a hash table.
+// FIXME: Get rid of this limit by moving logic into DedupLeaks.
const uptr kMaxLeaksConsidered = 5000;
-void LeakReport::AddLeakedChunk(uptr chunk, u32 stack_trace_id,
- uptr leaked_size, ChunkTag tag) {
- CHECK(tag == kDirectlyLeaked || tag == kIndirectlyLeaked);
- bool is_directly_leaked = (tag == kDirectlyLeaked);
- uptr i;
- for (i = 0; i < leaks_.size(); i++) {
- if (leaks_[i].stack_trace_id == stack_trace_id &&
- leaks_[i].is_directly_leaked == is_directly_leaked) {
- leaks_[i].hit_count++;
- leaks_[i].total_size += leaked_size;
- break;
+void LeakReport::AddLeakedChunks(const LeakedChunks &chunks) {
+ for (const LeakedChunk &leak : chunks) {
+ uptr chunk = leak.chunk;
+ u32 stack_trace_id = leak.stack_trace_id;
+ uptr leaked_size = leak.leaked_size;
+ ChunkTag tag = leak.tag;
+ CHECK(tag == kDirectlyLeaked || tag == kIndirectlyLeaked);
+
+ if (u32 resolution = flags()->resolution) {
+ StackTrace stack = StackDepotGet(stack_trace_id);
+ stack.size = Min(stack.size, resolution);
+ stack_trace_id = StackDepotPut(stack);
+ }
+
+ bool is_directly_leaked = (tag == kDirectlyLeaked);
+ uptr i;
+ for (i = 0; i < leaks_.size(); i++) {
+ if (leaks_[i].stack_trace_id == stack_trace_id &&
+ leaks_[i].is_directly_leaked == is_directly_leaked) {
+ leaks_[i].hit_count++;
+ leaks_[i].total_size += leaked_size;
+ break;
+ }
+ }
+ if (i == leaks_.size()) {
+ if (leaks_.size() == kMaxLeaksConsidered)
+ return;
+ Leak leak = {next_id_++, /* hit_count */ 1,
+ leaked_size, stack_trace_id,
+ is_directly_leaked, /* is_suppressed */ false};
+ leaks_.push_back(leak);
+ }
+ if (flags()->report_objects) {
+ LeakedObject obj = {leaks_[i].id, chunk, leaked_size};
+ leaked_objects_.push_back(obj);
}
- }
- if (i == leaks_.size()) {
- if (leaks_.size() == kMaxLeaksConsidered) return;
- Leak leak = { next_id_++, /* hit_count */ 1, leaked_size, stack_trace_id,
- is_directly_leaked, /* is_suppressed */ false };
- leaks_.push_back(leak);
- }
- if (flags()->report_objects) {
- LeakedObject obj = {leaks_[i].id, chunk, leaked_size};
- leaked_objects_.push_back(obj);
}
}
CHECK(leaks_.size() <= kMaxLeaksConsidered);
Printf("\n");
if (leaks_.size() == kMaxLeaksConsidered)
- Printf("Too many leaks! Only the first %zu leaks encountered will be "
- "reported.\n",
- kMaxLeaksConsidered);
+ Printf(
+ "Too many leaks! Only the first %zu leaks encountered will be "
+ "reported.\n",
+ kMaxLeaksConsidered);
uptr unsuppressed_count = UnsuppressedLeakCount();
if (num_leaks_to_report > 0 && num_leaks_to_report < unsuppressed_count)
Sort(leaks_.data(), leaks_.size(), &LeakComparator);
uptr leaks_reported = 0;
for (uptr i = 0; i < leaks_.size(); i++) {
- if (leaks_[i].is_suppressed) continue;
+ if (leaks_[i].is_suppressed)
+ continue;
PrintReportForLeak(i);
leaks_reported++;
- if (leaks_reported == num_leaks_to_report) break;
+ if (leaks_reported == num_leaks_to_report)
+ break;
}
if (leaks_reported < unsuppressed_count) {
uptr remaining = unsuppressed_count - leaks_reported;
leaks_[index].total_size, leaks_[index].hit_count);
Printf("%s", d.Default());
- PrintStackTraceById(leaks_[index].stack_trace_id);
+ CHECK(leaks_[index].stack_trace_id);
+ StackDepotGet(leaks_[index].stack_trace_id).Print();
if (flags()->report_objects) {
Printf("Objects leaked above:\n");
u32 leak_id = leaks_[index].id;
for (uptr j = 0; j < leaked_objects_.size(); j++) {
if (leaked_objects_[j].leak_id == leak_id)
- Printf("%p (%zu bytes)\n", leaked_objects_[j].addr,
+ Printf("%p (%zu bytes)\n", (void *)leaked_objects_[j].addr,
leaked_objects_[j].size);
}
}
CHECK(leaks_.size() <= kMaxLeaksConsidered);
uptr bytes = 0, allocations = 0;
for (uptr i = 0; i < leaks_.size(); i++) {
- if (leaks_[i].is_suppressed) continue;
- bytes += leaks_[i].total_size;
- allocations += leaks_[i].hit_count;
+ if (leaks_[i].is_suppressed)
+ continue;
+ bytes += leaks_[i].total_size;
+ allocations += leaks_[i].hit_count;
}
InternalScopedString summary;
summary.append("%zu byte(s) leaked in %zu allocation(s).", bytes,
LeakSuppressionContext *suppressions = GetSuppressionContext();
uptr new_suppressions = false;
for (uptr i = 0; i < leaks_.size(); i++) {
- Suppression *s =
- suppressions->GetSuppressionForStack(leaks_[i].stack_trace_id);
- if (s) {
- s->weight += leaks_[i].total_size;
- atomic_store_relaxed(&s->hit_count, atomic_load_relaxed(&s->hit_count) +
- leaks_[i].hit_count);
+ if (suppressions->Suppress(leaks_[i].stack_trace_id, leaks_[i].hit_count,
+ leaks_[i].total_size)) {
leaks_[i].is_suppressed = true;
++new_suppressions;
}
uptr LeakReport::UnsuppressedLeakCount() {
uptr result = 0;
for (uptr i = 0; i < leaks_.size(); i++)
- if (!leaks_[i].is_suppressed) result++;
+ if (!leaks_[i].is_suppressed)
+ result++;
return result;
}
return result;
}
-} // namespace __lsan
-#else // CAN_SANITIZE_LEAKS
+} // namespace __lsan
+#else // CAN_SANITIZE_LEAKS
namespace __lsan {
-void InitCommonLsan() { }
-void DoLeakCheck() { }
-void DoRecoverableLeakCheckVoid() { }
-void DisableInThisThread() { }
-void EnableInThisThread() { }
-}
-#endif // CAN_SANITIZE_LEAKS
+void InitCommonLsan() {}
+void DoLeakCheck() {}
+void DoRecoverableLeakCheckVoid() {}
+void DisableInThisThread() {}
+void EnableInThisThread() {}
+} // namespace __lsan
+#endif // CAN_SANITIZE_LEAKS
using namespace __lsan;
return;
// Cannot use PointsIntoChunk or LsanMetadata here, since the allocator is not
// locked.
- BlockingMutexLock l(&global_mutex);
+ Lock l(&global_mutex);
IgnoreObjectResult res = IgnoreObjectLocked(p);
if (res == kIgnoreObjectInvalid)
- VReport(1, "__lsan_ignore_object(): no heap object found at %p", p);
+ VReport(1, "__lsan_ignore_object(): no heap object found at %p\n", p);
if (res == kIgnoreObjectAlreadyIgnored)
- VReport(1, "__lsan_ignore_object(): "
- "heap object at %p is already being ignored\n", p);
+ VReport(1,
+ "__lsan_ignore_object(): "
+ "heap object at %p is already being ignored\n",
+ p);
if (res == kIgnoreObjectSuccess)
VReport(1, "__lsan_ignore_object(): ignoring heap object at %p\n", p);
-#endif // CAN_SANITIZE_LEAKS
+#endif // CAN_SANITIZE_LEAKS
}
SANITIZER_INTERFACE_ATTRIBUTE
void __lsan_register_root_region(const void *begin, uptr size) {
#if CAN_SANITIZE_LEAKS
- BlockingMutexLock l(&global_mutex);
- CHECK(root_regions);
+ Lock l(&global_mutex);
RootRegion region = {reinterpret_cast<uptr>(begin), size};
- root_regions->push_back(region);
- VReport(1, "Registered root region at %p of size %llu\n", begin, size);
-#endif // CAN_SANITIZE_LEAKS
+ root_regions.push_back(region);
+ VReport(1, "Registered root region at %p of size %zu\n", begin, size);
+#endif // CAN_SANITIZE_LEAKS
}
SANITIZER_INTERFACE_ATTRIBUTE
void __lsan_unregister_root_region(const void *begin, uptr size) {
#if CAN_SANITIZE_LEAKS
- BlockingMutexLock l(&global_mutex);
- CHECK(root_regions);
+ Lock l(&global_mutex);
bool removed = false;
- for (uptr i = 0; i < root_regions->size(); i++) {
- RootRegion region = (*root_regions)[i];
+ for (uptr i = 0; i < root_regions.size(); i++) {
+ RootRegion region = root_regions[i];
if (region.begin == reinterpret_cast<uptr>(begin) && region.size == size) {
removed = true;
- uptr last_index = root_regions->size() - 1;
- (*root_regions)[i] = (*root_regions)[last_index];
- root_regions->pop_back();
- VReport(1, "Unregistered root region at %p of size %llu\n", begin, size);
+ uptr last_index = root_regions.size() - 1;
+ root_regions[i] = root_regions[last_index];
+ root_regions.pop_back();
+ VReport(1, "Unregistered root region at %p of size %zu\n", begin, size);
break;
}
}
if (!removed) {
Report(
- "__lsan_unregister_root_region(): region at %p of size %llu has not "
+ "__lsan_unregister_root_region(): region at %p of size %zu has not "
"been registered.\n",
begin, size);
Die();
}
-#endif // CAN_SANITIZE_LEAKS
+#endif // CAN_SANITIZE_LEAKS
}
SANITIZER_INTERFACE_ATTRIBUTE
#if CAN_SANITIZE_LEAKS
if (common_flags()->detect_leaks)
__lsan::DoLeakCheck();
-#endif // CAN_SANITIZE_LEAKS
+#endif // CAN_SANITIZE_LEAKS
}
SANITIZER_INTERFACE_ATTRIBUTE
#if CAN_SANITIZE_LEAKS
if (common_flags()->detect_leaks)
return __lsan::DoRecoverableLeakCheck();
-#endif // CAN_SANITIZE_LEAKS
+#endif // CAN_SANITIZE_LEAKS
return 0;
}
}
#if !SANITIZER_SUPPORTS_WEAK_HOOKS
-SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-int __lsan_is_turned_off() {
+SANITIZER_INTERFACE_WEAK_DEF(int, __lsan_is_turned_off, void) {
return 0;
}
-SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-const char *__lsan_default_suppressions() {
+SANITIZER_INTERFACE_WEAK_DEF(const char *, __lsan_default_suppressions, void) {
return "";
}
#endif
-} // extern "C"
+} // extern "C"
#include "sanitizer_common/sanitizer_common.h"
#include "sanitizer_common/sanitizer_internal_defs.h"
#include "sanitizer_common/sanitizer_platform.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
#include "sanitizer_common/sanitizer_stoptheworld.h"
#include "sanitizer_common/sanitizer_symbolizer.h"
+#include "sanitizer_common/sanitizer_thread_registry.h"
// LeakSanitizer relies on some Glibc's internals (e.g. TLS machinery) on Linux.
// Also, LSan doesn't like 32 bit architectures
// Exclude leak-detection on arm32 for Android because `__aeabi_read_tp`
// is missing. This caused a link error.
#if SANITIZER_ANDROID && (__ANDROID_API__ < 28 || defined(__arm__))
-#define CAN_SANITIZE_LEAKS 0
-#elif (SANITIZER_LINUX || SANITIZER_MAC) && (SANITIZER_WORDSIZE == 64) && \
+# define CAN_SANITIZE_LEAKS 0
+#elif (SANITIZER_LINUX || SANITIZER_APPLE) && (SANITIZER_WORDSIZE == 64) && \
(defined(__x86_64__) || defined(__mips64) || defined(__aarch64__) || \
defined(__powerpc64__) || defined(__s390x__))
-#define CAN_SANITIZE_LEAKS 1
-#elif defined(__i386__) && (SANITIZER_LINUX || SANITIZER_MAC)
-#define CAN_SANITIZE_LEAKS 1
+# define CAN_SANITIZE_LEAKS 1
+#elif defined(__i386__) && (SANITIZER_LINUX || SANITIZER_APPLE)
+# define CAN_SANITIZE_LEAKS 1
#elif defined(__arm__) && SANITIZER_LINUX
-#define CAN_SANITIZE_LEAKS 1
+# define CAN_SANITIZE_LEAKS 1
+#elif SANITIZER_LOONGARCH64 && SANITIZER_LINUX
+# define CAN_SANITIZE_LEAKS 1
#elif SANITIZER_RISCV64 && SANITIZER_LINUX
-#define CAN_SANITIZE_LEAKS 1
+# define CAN_SANITIZE_LEAKS 1
#elif SANITIZER_NETBSD || SANITIZER_FUCHSIA
-#define CAN_SANITIZE_LEAKS 1
+# define CAN_SANITIZE_LEAKS 1
#else
-#define CAN_SANITIZE_LEAKS 0
+# define CAN_SANITIZE_LEAKS 0
#endif
namespace __sanitizer {
struct DTLS;
}
+// This section defines function and class prototypes which must be implemented
+// by the parent tool linking in LSan. There are implementations provided by the
+// LSan library which will be linked in when LSan is used as a standalone tool.
namespace __lsan {
// Chunk tags.
kIgnored = 3
};
+enum IgnoreObjectResult {
+ kIgnoreObjectSuccess,
+ kIgnoreObjectAlreadyIgnored,
+ kIgnoreObjectInvalid
+};
+
+struct Range {
+ uptr begin;
+ uptr end;
+};
+
+//// --------------------------------------------------------------------------
+//// Poisoning prototypes.
+//// --------------------------------------------------------------------------
+
+// Returns true if [addr, addr + sizeof(void *)) is poisoned.
+bool WordIsPoisoned(uptr addr);
+
+//// --------------------------------------------------------------------------
+//// Thread prototypes.
+//// --------------------------------------------------------------------------
+
+// Wrappers for ThreadRegistry access.
+void LockThreadRegistry() SANITIZER_NO_THREAD_SAFETY_ANALYSIS;
+void UnlockThreadRegistry() SANITIZER_NO_THREAD_SAFETY_ANALYSIS;
+// If called from the main thread, updates the main thread's TID in the thread
+// registry. We need this to handle processes that fork() without a subsequent
+// exec(), which invalidates the recorded TID. To update it, we must call
+// gettid() from the main thread. Our solution is to call this function before
+// leak checking and also before every call to pthread_create() (to handle cases
+// where leak checking is initiated from a non-main thread).
+void EnsureMainThreadIDIsCorrect();
+
+bool GetThreadRangesLocked(tid_t os_id, uptr *stack_begin, uptr *stack_end,
+ uptr *tls_begin, uptr *tls_end, uptr *cache_begin,
+ uptr *cache_end, DTLS **dtls);
+void GetAllThreadAllocatorCachesLocked(InternalMmapVector<uptr> *caches);
+void GetThreadExtraStackRangesLocked(InternalMmapVector<Range> *ranges);
+void GetThreadExtraStackRangesLocked(tid_t os_id,
+ InternalMmapVector<Range> *ranges);
+void GetAdditionalThreadContextPtrsLocked(InternalMmapVector<uptr> *ptrs);
+void GetRunningThreadsLocked(InternalMmapVector<tid_t> *threads);
+
+//// --------------------------------------------------------------------------
+//// Allocator prototypes.
+//// --------------------------------------------------------------------------
+
+// Wrappers for allocator's ForceLock()/ForceUnlock().
+void LockAllocator();
+void UnlockAllocator();
+
+// Returns the address range occupied by the global allocator object.
+void GetAllocatorGlobalRange(uptr *begin, uptr *end);
+// If p points into a chunk that has been allocated to the user, returns its
+// user-visible address. Otherwise, returns 0.
+uptr PointsIntoChunk(void *p);
+// Returns address of user-visible chunk contained in this allocator chunk.
+uptr GetUserBegin(uptr chunk);
+
+// Wrapper for chunk metadata operations.
+class LsanMetadata {
+ public:
+ // Constructor accepts address of user-visible chunk.
+ explicit LsanMetadata(uptr chunk);
+ bool allocated() const;
+ ChunkTag tag() const;
+ void set_tag(ChunkTag value);
+ uptr requested_size() const;
+ u32 stack_trace_id() const;
+
+ private:
+ void *metadata_;
+};
+
+// Iterate over all existing chunks. Allocator must be locked.
+void ForEachChunk(ForEachChunkCallback callback, void *arg);
+
+// Helper for __lsan_ignore_object().
+IgnoreObjectResult IgnoreObjectLocked(const void *p);
+
+// The rest of the LSan interface which is implemented by library.
+
+struct ScopedStopTheWorldLock {
+ ScopedStopTheWorldLock() {
+ LockThreadRegistry();
+ LockAllocator();
+ }
+
+ ~ScopedStopTheWorldLock() {
+ UnlockAllocator();
+ UnlockThreadRegistry();
+ }
+
+ ScopedStopTheWorldLock &operator=(const ScopedStopTheWorldLock &) = delete;
+ ScopedStopTheWorldLock(const ScopedStopTheWorldLock &) = delete;
+};
+
struct Flags {
#define LSAN_FLAG(Type, Name, DefaultValue, Description) Type Name;
#include "lsan_flags.inc"
inline Flags *flags() { return &lsan_flags; }
void RegisterLsanFlags(FlagParser *parser, Flags *f);
+struct LeakedChunk {
+ uptr chunk;
+ u32 stack_trace_id;
+ uptr leaked_size;
+ ChunkTag tag;
+};
+
+using LeakedChunks = InternalMmapVector<LeakedChunk>;
+
struct Leak {
u32 id;
uptr hit_count;
class LeakReport {
public:
LeakReport() {}
- void AddLeakedChunk(uptr chunk, u32 stack_trace_id, uptr leaked_size,
- ChunkTag tag);
+ void AddLeakedChunks(const LeakedChunks &chunks);
void ReportTopLeaks(uptr max_leaks);
void PrintSummary();
uptr ApplySuppressions();
// threads and enumerating roots.
struct CheckForLeaksParam {
Frontier frontier;
- LeakReport leak_report;
+ LeakedChunks leaks;
+ tid_t caller_tid;
+ uptr caller_sp;
bool success = false;
};
-InternalMmapVector<RootRegion> const *GetRootRegions();
+InternalMmapVectorNoCtor<RootRegion> const *GetRootRegions();
void ScanRootRegion(Frontier *frontier, RootRegion const ®ion,
uptr region_begin, uptr region_end, bool is_readable);
-void ForEachExtraStackRangeCb(uptr begin, uptr end, void* arg);
-void GetAdditionalThreadContextPtrs(ThreadContextBase *tctx, void *ptrs);
// Run stoptheworld while holding any platform-specific locks, as well as the
// allocator and thread registry locks.
void LockStuffAndStopTheWorld(StopTheWorldCallback callback,
Frontier *frontier,
const char *region_type, ChunkTag tag);
void ScanGlobalRange(uptr begin, uptr end, Frontier *frontier);
-
-enum IgnoreObjectResult {
- kIgnoreObjectSuccess,
- kIgnoreObjectAlreadyIgnored,
- kIgnoreObjectInvalid
-};
+void ScanExtraStackRanges(const InternalMmapVector<Range> &ranges,
+ Frontier *frontier);
// Functions called from the parent tool.
const char *MaybeCallLsanDefaultOptions();
#endif
}
-// The following must be implemented in the parent tool.
-
-void ForEachChunk(ForEachChunkCallback callback, void *arg);
-// Returns the address range occupied by the global allocator object.
-void GetAllocatorGlobalRange(uptr *begin, uptr *end);
-// Wrappers for allocator's ForceLock()/ForceUnlock().
-void LockAllocator();
-void UnlockAllocator();
-// Returns true if [addr, addr + sizeof(void *)) is poisoned.
-bool WordIsPoisoned(uptr addr);
-// Wrappers for ThreadRegistry access.
-void LockThreadRegistry() NO_THREAD_SAFETY_ANALYSIS;
-void UnlockThreadRegistry() NO_THREAD_SAFETY_ANALYSIS;
-ThreadRegistry *GetThreadRegistryLocked();
-bool GetThreadRangesLocked(tid_t os_id, uptr *stack_begin, uptr *stack_end,
- uptr *tls_begin, uptr *tls_end, uptr *cache_begin,
- uptr *cache_end, DTLS **dtls);
-void GetAllThreadAllocatorCachesLocked(InternalMmapVector<uptr> *caches);
-void ForEachExtraStackRange(tid_t os_id, RangeIteratorCallback callback,
- void *arg);
-// If called from the main thread, updates the main thread's TID in the thread
-// registry. We need this to handle processes that fork() without a subsequent
-// exec(), which invalidates the recorded TID. To update it, we must call
-// gettid() from the main thread. Our solution is to call this function before
-// leak checking and also before every call to pthread_create() (to handle cases
-// where leak checking is initiated from a non-main thread).
-void EnsureMainThreadIDIsCorrect();
-// If p points into a chunk that has been allocated to the user, returns its
-// user-visible address. Otherwise, returns 0.
-uptr PointsIntoChunk(void *p);
-// Returns address of user-visible chunk contained in this allocator chunk.
-uptr GetUserBegin(uptr chunk);
-// Helper for __lsan_ignore_object().
-IgnoreObjectResult IgnoreObjectLocked(const void *p);
-
// Return the linker module, if valid for the platform.
LoadedModule *GetLinker();
// Run platform-specific leak handlers.
void HandleLeaks();
-// Wrapper for chunk metadata operations.
-class LsanMetadata {
- public:
- // Constructor accepts address of user-visible chunk.
- explicit LsanMetadata(uptr chunk);
- bool allocated() const;
- ChunkTag tag() const;
- void set_tag(ChunkTag value);
- uptr requested_size() const;
- u32 stack_trace_id() const;
- private:
- void *metadata_;
-};
-
} // namespace __lsan
extern "C" {
SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
const char *__lsan_default_suppressions();
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __lsan_register_root_region(const void *p, __lsan::uptr size);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __lsan_unregister_root_region(const void *p, __lsan::uptr size);
+
} // extern "C"
#endif // LSAN_COMMON_H
//===---------------------------------------------------------------------===//
#include "lsan_common.h"
+#include "lsan_thread.h"
#include "sanitizer_common/sanitizer_platform.h"
#if CAN_SANITIZE_LEAKS && SANITIZER_FUCHSIA
// behavior and causes rare race conditions.
void HandleLeaks() {}
+// This is defined differently in asan_fuchsia.cpp and lsan_fuchsia.cpp.
+bool UseExitcodeOnLeak();
+
int ExitHook(int status) {
+ if (common_flags()->detect_leaks && common_flags()->leak_check_at_exit) {
+ if (UseExitcodeOnLeak())
+ DoLeakCheck();
+ else
+ DoRecoverableLeakCheckVoid();
+ }
return status == 0 && HasReportedLeaks() ? common_flags()->exitcode : status;
}
void LockStuffAndStopTheWorld(StopTheWorldCallback callback,
CheckForLeaksParam *argument) {
- LockThreadRegistry();
- LockAllocator();
+ ScopedStopTheWorldLock lock;
struct Params {
InternalMmapVector<uptr> allocator_caches;
// We don't use the thread registry at all for enumerating the threads
// and their stacks, registers, and TLS regions. So use it separately
- // just for the allocator cache, and to call ForEachExtraStackRange,
+ // just for the allocator cache, and to call ScanExtraStackRanges,
// which ASan needs.
if (flags()->use_stacks) {
- GetThreadRegistryLocked()->RunCallbackForEachThreadLocked(
- [](ThreadContextBase *tctx, void *arg) {
- ForEachExtraStackRange(tctx->os_id, ForEachExtraStackRangeCb,
- arg);
- },
- ¶ms->argument->frontier);
+ InternalMmapVector<Range> ranges;
+ GetThreadExtraStackRangesLocked(&ranges);
+ ScanExtraStackRanges(ranges, ¶ms->argument->frontier);
}
-
params->callback(SuspendedThreadsListFuchsia(), params->argument);
},
¶ms);
-
- UnlockAllocator();
- UnlockThreadRegistry();
}
} // namespace __lsan
static int LockStuffAndStopTheWorldCallback(struct dl_phdr_info *info,
size_t size, void *data) {
- LockThreadRegistry();
- LockAllocator();
+ ScopedStopTheWorldLock lock;
DoStopTheWorldParam *param = reinterpret_cast<DoStopTheWorldParam *>(data);
StopTheWorld(param->callback, param->argument);
- UnlockAllocator();
- UnlockThreadRegistry();
return 1;
}
#include "sanitizer_common/sanitizer_libc.h"
#include "lsan_common.h"
-#if CAN_SANITIZE_LEAKS && SANITIZER_MAC
+#if CAN_SANITIZE_LEAKS && SANITIZER_APPLE
-#include "sanitizer_common/sanitizer_allocator_internal.h"
-#include "lsan_allocator.h"
+# include <mach/mach.h>
+# include <mach/vm_statistics.h>
+# include <pthread.h>
-#include <pthread.h>
+# include "lsan_allocator.h"
+# include "sanitizer_common/sanitizer_allocator_internal.h"
+namespace __lsan {
-#include <mach/mach.h>
+enum class SeenRegion {
+ None = 0,
+ AllocOnce = 1 << 0,
+ LibDispatch = 1 << 1,
+ Foundation = 1 << 2,
+ All = AllocOnce | LibDispatch | Foundation
+};
+
+inline SeenRegion operator|(SeenRegion left, SeenRegion right) {
+ return static_cast<SeenRegion>(static_cast<int>(left) |
+ static_cast<int>(right));
+}
-// Only introduced in Mac OS X 10.9.
-#ifdef VM_MEMORY_OS_ALLOC_ONCE
-static const int kSanitizerVmMemoryOsAllocOnce = VM_MEMORY_OS_ALLOC_ONCE;
-#else
-static const int kSanitizerVmMemoryOsAllocOnce = 73;
-#endif
+inline SeenRegion &operator|=(SeenRegion &left, const SeenRegion &right) {
+ left = left | right;
+ return left;
+}
-namespace __lsan {
+struct RegionScanState {
+ SeenRegion seen_regions = SeenRegion::None;
+ bool in_libdispatch = false;
+};
typedef struct {
int disable_counter;
}
void ProcessPlatformSpecificAllocations(Frontier *frontier) {
- unsigned depth = 1;
- vm_size_t size = 0;
vm_address_t address = 0;
kern_return_t err = KERN_SUCCESS;
- mach_msg_type_number_t count = VM_REGION_SUBMAP_INFO_COUNT_64;
- InternalMmapVector<RootRegion> const *root_regions = GetRootRegions();
+ InternalMmapVectorNoCtor<RootRegion> const *root_regions = GetRootRegions();
+ RegionScanState scan_state;
while (err == KERN_SUCCESS) {
+ vm_size_t size = 0;
+ unsigned depth = 1;
struct vm_region_submap_info_64 info;
+ mach_msg_type_number_t count = VM_REGION_SUBMAP_INFO_COUNT_64;
err = vm_region_recurse_64(mach_task_self(), &address, &size, &depth,
(vm_region_info_t)&info, &count);
uptr end_address = address + size;
-
- // libxpc stashes some pointers in the Kernel Alloc Once page,
- // make sure not to report those as leaks.
- if (info.user_tag == kSanitizerVmMemoryOsAllocOnce) {
+ if (info.user_tag == VM_MEMORY_OS_ALLOC_ONCE) {
+ // libxpc stashes some pointers in the Kernel Alloc Once page,
+ // make sure not to report those as leaks.
+ scan_state.seen_regions |= SeenRegion::AllocOnce;
ScanRangeForPointers(address, end_address, frontier, "GLOBAL",
kReachable);
+ } else if (info.user_tag == VM_MEMORY_FOUNDATION) {
+ // Objective-C block trampolines use the Foundation region.
+ scan_state.seen_regions |= SeenRegion::Foundation;
+ ScanRangeForPointers(address, end_address, frontier, "GLOBAL",
+ kReachable);
+ } else if (info.user_tag == VM_MEMORY_LIBDISPATCH) {
+ // Dispatch continuations use the libdispatch region. Empirically, there
+ // can be more than one region with this tag, so we'll optimistically
+ // assume that they're continguous. Otherwise, we would need to scan every
+ // region to ensure we find them all.
+ scan_state.in_libdispatch = true;
+ ScanRangeForPointers(address, end_address, frontier, "GLOBAL",
+ kReachable);
+ } else if (scan_state.in_libdispatch) {
+ scan_state.seen_regions |= SeenRegion::LibDispatch;
+ scan_state.in_libdispatch = false;
+ }
- // Recursing over the full memory map is very slow, break out
- // early if we don't need the full iteration.
- if (!flags()->use_root_regions || !root_regions->size())
- break;
+ // Recursing over the full memory map is very slow, break out
+ // early if we don't need the full iteration.
+ if (scan_state.seen_regions == SeenRegion::All &&
+ !(flags()->use_root_regions && root_regions->size() > 0)) {
+ break;
}
// This additional root region scan is required on Darwin in order to
void LockStuffAndStopTheWorld(StopTheWorldCallback callback,
CheckForLeaksParam *argument) {
- LockThreadRegistry();
- LockAllocator();
+ ScopedStopTheWorldLock lock;
StopTheWorld(callback, argument);
- UnlockAllocator();
- UnlockThreadRegistry();
}
-} // namespace __lsan
+} // namespace __lsan
-#endif // CAN_SANITIZE_LEAKS && SANITIZER_MAC
+#endif // CAN_SANITIZE_LEAKS && SANITIZER_APPLE
OnCreatedArgs args;
__sanitizer::GetThreadStackTopAndBottom(true, &args.stack_end,
&args.stack_begin);
- u32 tid = ThreadCreate(0, GetThreadSelf(), true, &args);
+ u32 tid = ThreadCreate(kMainTid, true, &args);
CHECK_EQ(tid, 0);
ThreadStart(tid);
}
void GetAllThreadAllocatorCachesLocked(InternalMmapVector<uptr> *caches) {
- GetThreadRegistryLocked()->RunCallbackForEachThreadLocked(
+ GetLsanThreadRegistryLocked()->RunCallbackForEachThreadLocked(
[](ThreadContextBase *tctx, void *arg) {
auto ctx = static_cast<ThreadContext *>(tctx);
static_cast<decltype(caches)>(arg)->push_back(ctx->cache_begin());
caches);
}
+// On Fuchsia, leak detection is done by a special hook after atexit hooks.
+// So this doesn't install any atexit hook like on other platforms.
+void InstallAtExitCheckLeaks() {}
+
+// ASan defines this to check its `halt_on_error` flag.
+bool UseExitcodeOnLeak() { return true; }
+
} // namespace __lsan
// These are declared (in extern "C") by <zircon/sanitizer.h>.
void *__sanitizer_before_thread_create_hook(thrd_t thread, bool detached,
const char *name, void *stack_base,
size_t stack_size) {
- uptr user_id = reinterpret_cast<uptr>(thread);
ENSURE_LSAN_INITED;
EnsureMainThreadIDIsCorrect();
OnCreatedArgs args;
args.stack_begin = reinterpret_cast<uptr>(stack_base);
args.stack_end = args.stack_begin + stack_size;
u32 parent_tid = GetCurrentThread();
- u32 tid = ThreadCreate(parent_tid, user_id, detached, &args);
+ u32 tid = ThreadCreate(parent_tid, detached, &args);
return reinterpret_cast<void *>(static_cast<uptr>(tid));
}
// On success, there is nothing to do here.
if (error != thrd_success) {
// Clean up the thread registry for the thread creation that didn't happen.
- GetThreadRegistryLocked()->FinishThread(tid);
+ GetLsanThreadRegistryLocked()->FinishThread(tid);
}
}
#include "interception/interception.h"
#include "sanitizer_common/sanitizer_allocator.h"
+#include "sanitizer_common/sanitizer_allocator_dlsym.h"
#include "sanitizer_common/sanitizer_allocator_report.h"
#include "sanitizer_common/sanitizer_atomic.h"
#include "sanitizer_common/sanitizer_common.h"
int pthread_setspecific(unsigned key, const void *v);
}
+struct DlsymAlloc : DlSymAllocator<DlsymAlloc> {
+ static bool UseImpl() { return lsan_init_is_running; }
+ static void OnAllocate(const void *ptr, uptr size) {
+#if CAN_SANITIZE_LEAKS
+ // Suppress leaks from dlerror(). Previously dlsym hack on global array was
+ // used by leak sanitizer as a root region.
+ __lsan_register_root_region(ptr, size);
+#endif
+ }
+ static void OnFree(const void *ptr, uptr size) {
+#if CAN_SANITIZE_LEAKS
+ __lsan_unregister_root_region(ptr, size);
+#endif
+ }
+};
+
///// Malloc/free interceptors. /////
namespace std {
enum class align_val_t: size_t;
}
-#if !SANITIZER_MAC
+#if !SANITIZER_APPLE
INTERCEPTOR(void*, malloc, uptr size) {
+ if (DlsymAlloc::Use())
+ return DlsymAlloc::Allocate(size);
ENSURE_LSAN_INITED;
GET_STACK_TRACE_MALLOC;
return lsan_malloc(size, stack);
}
INTERCEPTOR(void, free, void *p) {
+ if (DlsymAlloc::PointerIsMine(p))
+ return DlsymAlloc::Free(p);
ENSURE_LSAN_INITED;
lsan_free(p);
}
INTERCEPTOR(void*, calloc, uptr nmemb, uptr size) {
- // This hack is not required for Fuchsia because there are no dlsym calls
- // involved in setting up interceptors.
-#if !SANITIZER_FUCHSIA
- if (lsan_init_is_running) {
- // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
- const uptr kCallocPoolSize = 1024;
- static uptr calloc_memory_for_dlsym[kCallocPoolSize];
- static uptr allocated;
- uptr size_in_words = ((nmemb * size) + kWordSize - 1) / kWordSize;
- void *mem = (void*)&calloc_memory_for_dlsym[allocated];
- allocated += size_in_words;
- CHECK(allocated < kCallocPoolSize);
- return mem;
- }
-#endif // !SANITIZER_FUCHSIA
+ if (DlsymAlloc::Use())
+ return DlsymAlloc::Callocate(nmemb, size);
ENSURE_LSAN_INITED;
GET_STACK_TRACE_MALLOC;
return lsan_calloc(nmemb, size, stack);
}
-INTERCEPTOR(void*, realloc, void *q, uptr size) {
+INTERCEPTOR(void *, realloc, void *ptr, uptr size) {
+ if (DlsymAlloc::Use() || DlsymAlloc::PointerIsMine(ptr))
+ return DlsymAlloc::Realloc(ptr, size);
ENSURE_LSAN_INITED;
GET_STACK_TRACE_MALLOC;
- return lsan_realloc(q, size, stack);
+ return lsan_realloc(ptr, size, stack);
}
INTERCEPTOR(void*, reallocarray, void *q, uptr nmemb, uptr size) {
GET_STACK_TRACE_MALLOC;
return lsan_valloc(size, stack);
}
-#endif // !SANITIZER_MAC
+#endif // !SANITIZER_APPLE
#if SANITIZER_INTERCEPT_MEMALIGN
INTERCEPTOR(void*, memalign, uptr alignment, uptr size) {
// libstdc++, each of has its implementation of new and delete.
// To make sure that C++ allocation/deallocation operators are overridden on
// OS X we need to intercept them using their mangled names.
-#if !SANITIZER_MAC
+#if !SANITIZER_APPLE
INTERCEPTOR_ATTRIBUTE
void *operator new(size_t size) { OPERATOR_NEW_BODY(false /*nothrow*/); }
void operator delete[](void *ptr, size_t size, std::align_val_t) NOEXCEPT
{ OPERATOR_DELETE_BODY; }
-#else // SANITIZER_MAC
+#else // SANITIZER_APPLE
INTERCEPTOR(void *, _Znwm, size_t size)
{ OPERATOR_NEW_BODY(false /*nothrow*/); }
INTERCEPTOR(void, _ZdaPvRKSt9nothrow_t, void *ptr, std::nothrow_t const&)
{ OPERATOR_DELETE_BODY; }
-#endif // !SANITIZER_MAC
+#endif // !SANITIZER_APPLE
///// Thread initialization and finalization. /////
res = REAL(pthread_create)(th, attr, __lsan_thread_start_func, &p);
}
if (res == 0) {
- int tid = ThreadCreate(GetCurrentThread(), *(uptr *)th,
- IsStateDetached(detached));
+ int tid = ThreadCreate(GetCurrentThread(), IsStateDetached(detached));
CHECK_NE(tid, kMainTid);
atomic_store(&p.tid, tid, memory_order_release);
while (atomic_load(&p.tid, memory_order_acquire) != 0)
return res;
}
-INTERCEPTOR(int, pthread_join, void *th, void **ret) {
- ENSURE_LSAN_INITED;
- int tid = ThreadTid((uptr)th);
- int res = REAL(pthread_join)(th, ret);
- if (res == 0)
- ThreadJoin(tid);
- return res;
+INTERCEPTOR(int, pthread_join, void *t, void **arg) {
+ return REAL(pthread_join)(t, arg);
}
-INTERCEPTOR(int, pthread_detach, void *th) {
- ENSURE_LSAN_INITED;
- int tid = ThreadTid((uptr)th);
- int res = REAL(pthread_detach)(th);
- if (res == 0)
- ThreadDetach(tid);
- return res;
-}
+DEFINE_REAL_PTHREAD_FUNCTIONS
INTERCEPTOR(void, _exit, int status) {
if (status == 0 && HasReportedLeaks()) status = common_flags()->exitcode;
LSAN_MAYBE_INTERCEPT_MALLINFO;
LSAN_MAYBE_INTERCEPT_MALLOPT;
INTERCEPT_FUNCTION(pthread_create);
- INTERCEPT_FUNCTION(pthread_detach);
INTERCEPT_FUNCTION(pthread_join);
INTERCEPT_FUNCTION(_exit);
//===----------------------------------------------------------------------===//
#include "sanitizer_common/sanitizer_platform.h"
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
#include "interception/interception.h"
#include "lsan.h"
ALWAYS_INLINE
void lsan_register_worker_thread(int parent_tid) {
if (GetCurrentThread() == kInvalidTid) {
- u32 tid = ThreadCreate(parent_tid, 0, true);
+ u32 tid = ThreadCreate(parent_tid, true);
ThreadStart(tid, GetTid());
SetCurrentThread(tid);
}
}
#endif
-#endif // SANITIZER_MAC
+#endif // SANITIZER_APPLE
//===----------------------------------------------------------------------===//
#include "sanitizer_common/sanitizer_platform.h"
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
#include "lsan.h"
#include "lsan_allocator.h"
#include "sanitizer_common/sanitizer_malloc_mac.inc"
-#endif // SANITIZER_MAC
+#endif // SANITIZER_APPLE
#if SANITIZER_POSIX
#include "lsan.h"
#include "lsan_allocator.h"
+#include "lsan_thread.h"
#include "sanitizer_common/sanitizer_stacktrace.h"
#include "sanitizer_common/sanitizer_tls_get_addr.h"
uptr *tls_begin, uptr *tls_end, uptr *cache_begin,
uptr *cache_end, DTLS **dtls) {
ThreadContext *context = static_cast<ThreadContext *>(
- GetThreadRegistryLocked()->FindThreadContextByOsIDLocked(os_id));
+ GetLsanThreadRegistryLocked()->FindThreadContextByOsIDLocked(os_id));
if (!context)
return false;
*stack_begin = context->stack_begin();
}
void InitializeMainThread() {
- u32 tid = ThreadCreate(kMainTid, 0, true);
+ u32 tid = ThreadCreate(kMainTid, true);
CHECK_EQ(tid, kMainTid);
ThreadStart(tid, GetTid());
}
nullptr);
}
+void InstallAtExitCheckLeaks() {
+ if (common_flags()->detect_leaks && common_flags()->leak_check_at_exit)
+ Atexit(DoLeakCheck);
+}
+
} // namespace __lsan
#endif // SANITIZER_POSIX
DTLS_Destroy();
}
-u32 ThreadCreate(u32 parent_tid, uptr user_id, bool detached, void *arg) {
- return thread_registry->CreateThread(user_id, detached, parent_tid, arg);
+u32 ThreadCreate(u32 parent_tid, bool detached, void *arg) {
+ return thread_registry->CreateThread(0, detached, parent_tid, arg);
}
void ThreadContextLsanBase::ThreadStart(u32 tid, tid_t os_id,
return (ThreadContext *)thread_registry->GetThreadLocked(GetCurrentThread());
}
-static bool FindThreadByUid(ThreadContextBase *tctx, void *arg) {
- uptr uid = (uptr)arg;
- if (tctx->user_id == uid && tctx->status != ThreadStatusInvalid) {
- return true;
- }
- return false;
-}
-
-u32 ThreadTid(uptr uid) {
- return thread_registry->FindThread(FindThreadByUid, (void *)uid);
-}
-
-void ThreadDetach(u32 tid) {
- CHECK_NE(tid, kInvalidTid);
- thread_registry->DetachThread(tid, /* arg */ nullptr);
-}
-
-void ThreadJoin(u32 tid) {
- CHECK_NE(tid, kInvalidTid);
- thread_registry->JoinThread(tid, /* arg */ nullptr);
-}
-
void EnsureMainThreadIDIsCorrect() {
if (GetCurrentThread() == kMainTid)
CurrentThreadContext()->os_id = GetTid();
///// Interface to the common LSan module. /////
-void ForEachExtraStackRange(tid_t os_id, RangeIteratorCallback callback,
- void *arg) {}
+void GetThreadExtraStackRangesLocked(tid_t os_id,
+ InternalMmapVector<Range> *ranges) {}
+void GetThreadExtraStackRangesLocked(InternalMmapVector<Range> *ranges) {}
void LockThreadRegistry() { thread_registry->Lock(); }
void UnlockThreadRegistry() { thread_registry->Unlock(); }
-ThreadRegistry *GetThreadRegistryLocked() {
+ThreadRegistry *GetLsanThreadRegistryLocked() {
thread_registry->CheckLocked();
return thread_registry;
}
+void GetRunningThreadsLocked(InternalMmapVector<tid_t> *threads) {
+ GetLsanThreadRegistryLocked()->RunCallbackForEachThreadLocked(
+ [](ThreadContextBase *tctx, void *threads) {
+ if (tctx->status == ThreadStatusRunning) {
+ reinterpret_cast<InternalMmapVector<tid_t> *>(threads)->push_back(
+ tctx->os_id);
+ }
+ },
+ threads);
+}
+
} // namespace __lsan
void InitializeThreadRegistry();
void InitializeMainThread();
-u32 ThreadCreate(u32 tid, uptr uid, bool detached, void *arg = nullptr);
+ThreadRegistry *GetLsanThreadRegistryLocked();
+
+u32 ThreadCreate(u32 tid, bool detached, void *arg = nullptr);
void ThreadFinish();
-void ThreadDetach(u32 tid);
-void ThreadJoin(u32 tid);
-u32 ThreadTid(uptr uid);
u32 GetCurrentThread();
void SetCurrentThread(u32 tid);
memprof_interceptors_memintrinsics.cpp
memprof_linux.cpp
memprof_malloc_linux.cpp
+ memprof_mibmap.cpp
memprof_posix.cpp
+ memprof_rawprofile.cpp
memprof_rtl.cpp
memprof_shadow_setup.cpp
memprof_stack.cpp
memprof_interface_internal.h
memprof_internal.h
memprof_mapping.h
+ memprof_meminfoblock.h
+ memprof_mibmap.h
+ memprof_rawprofile.h
memprof_stack.h
memprof_stats.h
memprof_thread.h
)
include_directories(..)
+include_directories(../../include)
set(MEMPROF_CFLAGS ${SANITIZER_COMMON_CFLAGS})
set(MEMPROF_COMMON_DEFINITIONS "")
+# Too many existing bugs, needs cleanup.
+append_list_if(COMPILER_RT_HAS_WNO_FORMAT -Wno-format MEMPROF_CFLAGS)
+
append_rtti_flag(OFF MEMPROF_CFLAGS)
set(MEMPROF_DYNAMIC_LINK_FLAGS ${SANITIZER_COMMON_LINK_FLAGS})
append_list_if(COMPILER_RT_HAS_FTLS_MODEL_INITIAL_EXEC
-ftls-model=initial-exec MEMPROF_DYNAMIC_CFLAGS)
-set(MEMPROF_DYNAMIC_LIBS ${SANITIZER_CXX_ABI_LIBRARIES} ${SANITIZER_COMMON_LINK_LIBS})
+set(MEMPROF_DYNAMIC_LIBS
+ ${COMPILER_RT_UNWINDER_LINK_LIBS}
+ ${SANITIZER_CXX_ABI_LIBRARIES}
+ ${SANITIZER_COMMON_LINK_LIBS})
append_list_if(COMPILER_RT_HAS_LIBDL dl MEMPROF_DYNAMIC_LIBS)
append_list_if(COMPILER_RT_HAS_LIBRT rt MEMPROF_DYNAMIC_LIBS)
add_dependencies(memprof clang_rt.memprof-${arch}-symbols)
endif()
endforeach()
+
+
+if(COMPILER_RT_INCLUDE_TESTS)
+ add_subdirectory(tests)
+endif()
#include "memprof_allocator.h"
#include "memprof_mapping.h"
+#include "memprof_mibmap.h"
+#include "memprof_rawprofile.h"
#include "memprof_stack.h"
#include "memprof_thread.h"
+#include "profile/MemProfData.inc"
#include "sanitizer_common/sanitizer_allocator_checks.h"
#include "sanitizer_common/sanitizer_allocator_interface.h"
#include "sanitizer_common/sanitizer_allocator_report.h"
#include "sanitizer_common/sanitizer_file.h"
#include "sanitizer_common/sanitizer_flags.h"
#include "sanitizer_common/sanitizer_internal_defs.h"
-#include "sanitizer_common/sanitizer_list.h"
+#include "sanitizer_common/sanitizer_procmaps.h"
#include "sanitizer_common/sanitizer_stackdepot.h"
#include <sched.h>
-#include <stdlib.h>
#include <time.h>
namespace __memprof {
+namespace {
+using ::llvm::memprof::MemInfoBlock;
+
+void Print(const MemInfoBlock &M, const u64 id, bool print_terse) {
+ u64 p;
+
+ if (print_terse) {
+ p = M.TotalSize * 100 / M.AllocCount;
+ Printf("MIB:%llu/%u/%llu.%02llu/%u/%u/", id, M.AllocCount, p / 100, p % 100,
+ M.MinSize, M.MaxSize);
+ p = M.TotalAccessCount * 100 / M.AllocCount;
+ Printf("%llu.%02llu/%llu/%llu/", p / 100, p % 100, M.MinAccessCount,
+ M.MaxAccessCount);
+ p = M.TotalLifetime * 100 / M.AllocCount;
+ Printf("%llu.%02llu/%u/%u/", p / 100, p % 100, M.MinLifetime,
+ M.MaxLifetime);
+ Printf("%u/%u/%u/%u\n", M.NumMigratedCpu, M.NumLifetimeOverlaps,
+ M.NumSameAllocCpu, M.NumSameDeallocCpu);
+ } else {
+ p = M.TotalSize * 100 / M.AllocCount;
+ Printf("Memory allocation stack id = %llu\n", id);
+ Printf("\talloc_count %u, size (ave/min/max) %llu.%02llu / %u / %u\n",
+ M.AllocCount, p / 100, p % 100, M.MinSize, M.MaxSize);
+ p = M.TotalAccessCount * 100 / M.AllocCount;
+ Printf("\taccess_count (ave/min/max): %llu.%02llu / %llu / %llu\n", p / 100,
+ p % 100, M.MinAccessCount, M.MaxAccessCount);
+ p = M.TotalLifetime * 100 / M.AllocCount;
+ Printf("\tlifetime (ave/min/max): %llu.%02llu / %u / %u\n", p / 100,
+ p % 100, M.MinLifetime, M.MaxLifetime);
+ Printf("\tnum migrated: %u, num lifetime overlaps: %u, num same alloc "
+ "cpu: %u, num same dealloc_cpu: %u\n",
+ M.NumMigratedCpu, M.NumLifetimeOverlaps, M.NumSameAllocCpu,
+ M.NumSameDeallocCpu);
+ }
+}
+} // namespace
static int GetCpuId(void) {
// _memprof_preinit is called via the preinit_array, which subsequently calls
return &ms->allocator_cache;
}
-struct MemInfoBlock {
- u32 alloc_count;
- u64 total_access_count, min_access_count, max_access_count;
- u64 total_size;
- u32 min_size, max_size;
- u32 alloc_timestamp, dealloc_timestamp;
- u64 total_lifetime;
- u32 min_lifetime, max_lifetime;
- u32 alloc_cpu_id, dealloc_cpu_id;
- u32 num_migrated_cpu;
-
- // Only compared to prior deallocated object currently.
- u32 num_lifetime_overlaps;
- u32 num_same_alloc_cpu;
- u32 num_same_dealloc_cpu;
-
- u64 data_type_id; // TODO: hash of type name
-
- MemInfoBlock() : alloc_count(0) {}
-
- MemInfoBlock(u32 size, u64 access_count, u32 alloc_timestamp,
- u32 dealloc_timestamp, u32 alloc_cpu, u32 dealloc_cpu)
- : alloc_count(1), total_access_count(access_count),
- min_access_count(access_count), max_access_count(access_count),
- total_size(size), min_size(size), max_size(size),
- alloc_timestamp(alloc_timestamp), dealloc_timestamp(dealloc_timestamp),
- total_lifetime(dealloc_timestamp - alloc_timestamp),
- min_lifetime(total_lifetime), max_lifetime(total_lifetime),
- alloc_cpu_id(alloc_cpu), dealloc_cpu_id(dealloc_cpu),
- num_lifetime_overlaps(0), num_same_alloc_cpu(0),
- num_same_dealloc_cpu(0) {
- num_migrated_cpu = alloc_cpu_id != dealloc_cpu_id;
- }
-
- void Print(u64 id) {
- u64 p;
- if (flags()->print_terse) {
- p = total_size * 100 / alloc_count;
- Printf("MIB:%llu/%u/%d.%02d/%u/%u/", id, alloc_count, p / 100, p % 100,
- min_size, max_size);
- p = total_access_count * 100 / alloc_count;
- Printf("%d.%02d/%u/%u/", p / 100, p % 100, min_access_count,
- max_access_count);
- p = total_lifetime * 100 / alloc_count;
- Printf("%d.%02d/%u/%u/", p / 100, p % 100, min_lifetime, max_lifetime);
- Printf("%u/%u/%u/%u\n", num_migrated_cpu, num_lifetime_overlaps,
- num_same_alloc_cpu, num_same_dealloc_cpu);
- } else {
- p = total_size * 100 / alloc_count;
- Printf("Memory allocation stack id = %llu\n", id);
- Printf("\talloc_count %u, size (ave/min/max) %d.%02d / %u / %u\n",
- alloc_count, p / 100, p % 100, min_size, max_size);
- p = total_access_count * 100 / alloc_count;
- Printf("\taccess_count (ave/min/max): %d.%02d / %u / %u\n", p / 100,
- p % 100, min_access_count, max_access_count);
- p = total_lifetime * 100 / alloc_count;
- Printf("\tlifetime (ave/min/max): %d.%02d / %u / %u\n", p / 100, p % 100,
- min_lifetime, max_lifetime);
- Printf("\tnum migrated: %u, num lifetime overlaps: %u, num same alloc "
- "cpu: %u, num same dealloc_cpu: %u\n",
- num_migrated_cpu, num_lifetime_overlaps, num_same_alloc_cpu,
- num_same_dealloc_cpu);
- }
- }
-
- static void printHeader() {
- CHECK(flags()->print_terse);
- Printf("MIB:StackID/AllocCount/AveSize/MinSize/MaxSize/AveAccessCount/"
- "MinAccessCount/MaxAccessCount/AveLifetime/MinLifetime/MaxLifetime/"
- "NumMigratedCpu/NumLifetimeOverlaps/NumSameAllocCpu/"
- "NumSameDeallocCpu\n");
- }
-
- void Merge(MemInfoBlock &newMIB) {
- alloc_count += newMIB.alloc_count;
-
- total_access_count += newMIB.total_access_count;
- min_access_count = Min(min_access_count, newMIB.min_access_count);
- max_access_count = Max(max_access_count, newMIB.max_access_count);
-
- total_size += newMIB.total_size;
- min_size = Min(min_size, newMIB.min_size);
- max_size = Max(max_size, newMIB.max_size);
-
- total_lifetime += newMIB.total_lifetime;
- min_lifetime = Min(min_lifetime, newMIB.min_lifetime);
- max_lifetime = Max(max_lifetime, newMIB.max_lifetime);
-
- // We know newMIB was deallocated later, so just need to check if it was
- // allocated before last one deallocated.
- num_lifetime_overlaps += newMIB.alloc_timestamp < dealloc_timestamp;
- alloc_timestamp = newMIB.alloc_timestamp;
- dealloc_timestamp = newMIB.dealloc_timestamp;
-
- num_same_alloc_cpu += alloc_cpu_id == newMIB.alloc_cpu_id;
- num_same_dealloc_cpu += dealloc_cpu_id == newMIB.dealloc_cpu_id;
- alloc_cpu_id = newMIB.alloc_cpu_id;
- dealloc_cpu_id = newMIB.dealloc_cpu_id;
- }
-};
-
-static u32 AccessCount = 0;
-static u32 MissCount = 0;
-
-struct SetEntry {
- SetEntry() : id(0), MIB() {}
- bool Empty() { return id == 0; }
- void Print() {
- CHECK(!Empty());
- MIB.Print(id);
- }
- // The stack id
- u64 id;
- MemInfoBlock MIB;
-};
-
-struct CacheSet {
- enum { kSetSize = 4 };
-
- void PrintAll() {
- for (int i = 0; i < kSetSize; i++) {
- if (Entries[i].Empty())
- continue;
- Entries[i].Print();
- }
- }
- void insertOrMerge(u64 new_id, MemInfoBlock &newMIB) {
- AccessCount++;
- SetAccessCount++;
-
- for (int i = 0; i < kSetSize; i++) {
- auto id = Entries[i].id;
- // Check if this is a hit or an empty entry. Since we always move any
- // filled locations to the front of the array (see below), we don't need
- // to look after finding the first empty entry.
- if (id == new_id || !id) {
- if (id == 0) {
- Entries[i].id = new_id;
- Entries[i].MIB = newMIB;
- } else {
- Entries[i].MIB.Merge(newMIB);
- }
- // Assuming some id locality, we try to swap the matching entry
- // into the first set position.
- if (i != 0) {
- auto tmp = Entries[0];
- Entries[0] = Entries[i];
- Entries[i] = tmp;
- }
- return;
- }
- }
-
- // Miss
- MissCount++;
- SetMissCount++;
-
- // We try to find the entries with the lowest alloc count to be evicted:
- int min_idx = 0;
- u64 min_count = Entries[0].MIB.alloc_count;
- for (int i = 1; i < kSetSize; i++) {
- CHECK(!Entries[i].Empty());
- if (Entries[i].MIB.alloc_count < min_count) {
- min_idx = i;
- min_count = Entries[i].MIB.alloc_count;
- }
- }
-
- // Print the evicted entry profile information
- if (!flags()->print_terse)
- Printf("Evicted:\n");
- Entries[min_idx].Print();
-
- // Similar to the hit case, put new MIB in first set position.
- if (min_idx != 0)
- Entries[min_idx] = Entries[0];
- Entries[0].id = new_id;
- Entries[0].MIB = newMIB;
- }
-
- void PrintMissRate(int i) {
- u64 p = SetAccessCount ? SetMissCount * 10000ULL / SetAccessCount : 0;
- Printf("Set %d miss rate: %d / %d = %5d.%02d%%\n", i, SetMissCount,
- SetAccessCount, p / 100, p % 100);
- }
-
- SetEntry Entries[kSetSize];
- u32 SetAccessCount = 0;
- u32 SetMissCount = 0;
-};
-
-struct MemInfoBlockCache {
- MemInfoBlockCache() {
- if (common_flags()->print_module_map)
- DumpProcessMap();
- if (flags()->print_terse)
- MemInfoBlock::printHeader();
- Sets =
- (CacheSet *)malloc(sizeof(CacheSet) * flags()->mem_info_cache_entries);
- Constructed = true;
- }
-
- ~MemInfoBlockCache() { free(Sets); }
-
- void insertOrMerge(u64 new_id, MemInfoBlock &newMIB) {
- u64 hv = new_id;
-
- // Use mod method where number of entries should be a prime close to power
- // of 2.
- hv %= flags()->mem_info_cache_entries;
-
- return Sets[hv].insertOrMerge(new_id, newMIB);
- }
-
- void PrintAll() {
- for (int i = 0; i < flags()->mem_info_cache_entries; i++) {
- Sets[i].PrintAll();
- }
- }
-
- void PrintMissRate() {
- if (!flags()->print_mem_info_cache_miss_rate)
- return;
- u64 p = AccessCount ? MissCount * 10000ULL / AccessCount : 0;
- Printf("Overall miss rate: %d / %d = %5d.%02d%%\n", MissCount, AccessCount,
- p / 100, p % 100);
- if (flags()->print_mem_info_cache_miss_rate_details)
- for (int i = 0; i < flags()->mem_info_cache_entries; i++)
- Sets[i].PrintMissRate(i);
- }
-
- CacheSet *Sets;
- // Flag when the Sets have been allocated, in case a deallocation is called
- // very early before the static init of the Allocator and therefore this table
- // have completed.
- bool Constructed = false;
-};
-
// Accumulates the access count from the shadow for the given pointer and size.
u64 GetShadowCount(uptr p, u32 size) {
u64 *shadow = (u64 *)MEM_TO_SHADOW(p);
AllocatorCache fallback_allocator_cache;
uptr max_user_defined_malloc_size;
- atomic_uint8_t rss_limit_exceeded;
- MemInfoBlockCache MemInfoBlockTable;
- bool destructing;
+ // Holds the mapping of stack ids to MemInfoBlocks.
+ MIBMapTy MIBMap;
+
+ atomic_uint8_t destructing;
+ atomic_uint8_t constructed;
+ bool print_text;
// ------------------- Initialization ------------------------
- explicit Allocator(LinkerInitialized) : destructing(false) {}
+ explicit Allocator(LinkerInitialized) : print_text(flags()->print_text) {
+ atomic_store_relaxed(&destructing, 0);
+ atomic_store_relaxed(&constructed, 1);
+ }
+
+ ~Allocator() {
+ atomic_store_relaxed(&destructing, 1);
+ FinishAndWrite();
+ }
+
+ static void PrintCallback(const uptr Key, LockedMemInfoBlock *const &Value,
+ void *Arg) {
+ SpinMutexLock l(&Value->mutex);
+ Print(Value->mib, Key, bool(Arg));
+ }
- ~Allocator() { FinishAndPrint(); }
+ void FinishAndWrite() {
+ if (print_text && common_flags()->print_module_map)
+ DumpProcessMap();
- void FinishAndPrint() {
- if (!flags()->print_terse)
- Printf("Live on exit:\n");
allocator.ForceLock();
+
+ InsertLiveBlocks();
+ if (print_text) {
+ if (!flags()->print_terse)
+ Printf("Recorded MIBs (incl. live on exit):\n");
+ MIBMap.ForEach(PrintCallback,
+ reinterpret_cast<void *>(flags()->print_terse));
+ StackDepotPrintAll();
+ } else {
+ // Serialize the contents to a raw profile. Format documented in
+ // memprof_rawprofile.h.
+ char *Buffer = nullptr;
+
+ MemoryMappingLayout Layout(/*cache_enabled=*/true);
+ u64 BytesSerialized = SerializeToRawProfile(MIBMap, Layout, Buffer);
+ CHECK(Buffer && BytesSerialized && "could not serialize to buffer");
+ report_file.Write(Buffer, BytesSerialized);
+ }
+
+ allocator.ForceUnlock();
+ }
+
+ // Inserts any blocks which have been allocated but not yet deallocated.
+ void InsertLiveBlocks() {
allocator.ForEachChunk(
[](uptr chunk, void *alloc) {
u64 user_requested_size;
+ Allocator *A = (Allocator *)alloc;
MemprofChunk *m =
- ((Allocator *)alloc)
- ->GetMemprofChunk((void *)chunk, user_requested_size);
+ A->GetMemprofChunk((void *)chunk, user_requested_size);
if (!m)
return;
uptr user_beg = ((uptr)m) + kChunkHeaderSize;
long curtime = GetTimestamp();
MemInfoBlock newMIB(user_requested_size, c, m->timestamp_ms, curtime,
m->cpu_id, GetCpuId());
- ((Allocator *)alloc)
- ->MemInfoBlockTable.insertOrMerge(m->alloc_context_id, newMIB);
+ InsertOrMerge(m->alloc_context_id, newMIB, A->MIBMap);
},
this);
- allocator.ForceUnlock();
-
- destructing = true;
- MemInfoBlockTable.PrintMissRate();
- MemInfoBlockTable.PrintAll();
- StackDepotPrintAll();
}
void InitLinkerInitialized() {
: kMaxAllowedMallocSize;
}
- bool RssLimitExceeded() {
- return atomic_load(&rss_limit_exceeded, memory_order_relaxed);
- }
-
- void SetRssLimitExceeded(bool limit_exceeded) {
- atomic_store(&rss_limit_exceeded, limit_exceeded, memory_order_relaxed);
- }
-
// -------------------- Allocation/Deallocation routines ---------------
void *Allocate(uptr size, uptr alignment, BufferedStackTrace *stack,
AllocType alloc_type) {
if (UNLIKELY(!memprof_inited))
MemprofInitFromRtl();
- if (RssLimitExceeded()) {
+ if (UNLIKELY(IsRssLimitExceeded())) {
if (AllocatorMayReturnNull())
return nullptr;
ReportRssLimitExceeded(stack);
if (size > kMaxAllowedMallocSize || needed_size > kMaxAllowedMallocSize ||
size > max_user_defined_malloc_size) {
if (AllocatorMayReturnNull()) {
- Report("WARNING: MemProfiler failed to allocate 0x%zx bytes\n",
- (void *)size);
+ Report("WARNING: MemProfiler failed to allocate 0x%zx bytes\n", size);
return nullptr;
}
uptr malloc_limit =
CHECK_LE(alloc_beg + sizeof(LargeChunkHeader), chunk_beg);
reinterpret_cast<LargeChunkHeader *>(alloc_beg)->Set(m);
}
- MEMPROF_MALLOC_HOOK(res, size);
+ RunMallocHooks(res, size);
return res;
}
if (p == 0)
return;
- MEMPROF_FREE_HOOK(ptr);
+ RunFreeHooks(ptr);
uptr chunk_beg = p - kChunkHeaderSize;
MemprofChunk *m = reinterpret_cast<MemprofChunk *>(chunk_beg);
u64 user_requested_size =
atomic_exchange(&m->user_requested_size, 0, memory_order_acquire);
- if (memprof_inited && memprof_init_done && !destructing &&
- MemInfoBlockTable.Constructed) {
+ if (memprof_inited && memprof_init_done &&
+ atomic_load_relaxed(&constructed) &&
+ !atomic_load_relaxed(&destructing)) {
u64 c = GetShadowCount(p, user_requested_size);
long curtime = GetTimestamp();
MemInfoBlock newMIB(user_requested_size, c, m->timestamp_ms, curtime,
m->cpu_id, GetCpuId());
- {
- SpinMutexLock l(&fallback_mutex);
- MemInfoBlockTable.insertOrMerge(m->alloc_context_id, newMIB);
- }
+ InsertOrMerge(m->alloc_context_id, newMIB, MIBMap);
}
MemprofStats &thread_stats = GetCurrentThreadStats();
void PrintStats() { allocator.PrintStats(); }
- void ForceLock() NO_THREAD_SAFETY_ANALYSIS {
+ void ForceLock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
allocator.ForceLock();
fallback_mutex.Lock();
}
- void ForceUnlock() NO_THREAD_SAFETY_ANALYSIS {
+ void ForceUnlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
fallback_mutex.Unlock();
allocator.ForceUnlock();
}
return usable_size;
}
-void MemprofSoftRssLimitExceededCallback(bool limit_exceeded) {
- instance.SetRssLimitExceeded(limit_exceeded);
-}
-
} // namespace __memprof
// ---------------------- Interface ---------------- {{{1
using namespace __memprof;
-#if !SANITIZER_SUPPORTS_WEAK_HOOKS
-// Provide default (no-op) implementation of malloc hooks.
-SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_malloc_hook, void *ptr,
- uptr size) {
- (void)ptr;
- (void)size;
-}
-
-SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_free_hook, void *ptr) {
- (void)ptr;
-}
-#endif
-
uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; }
int __sanitizer_get_ownership(const void *p) {
}
int __memprof_profile_dump() {
- instance.FinishAndPrint();
+ instance.FinishAndWrite();
// In the future we may want to return non-zero if there are any errors
// detected during the dumping process.
return 0;
uptr memprof_malloc_usable_size(const void *ptr, uptr pc, uptr bp);
void PrintInternalAllocatorStats();
-void MemprofSoftRssLimitExceededCallback(bool exceeded);
} // namespace __memprof
#endif // MEMPROF_ALLOCATOR_H
"realloc(p, 0) is equivalent to free(p) by default (Same as the "
"POSIX standard). If set to false, realloc(p, 0) will return a "
"pointer to an allocated space which can not be used.")
+MEMPROF_FLAG(bool, print_text, false,
+ "If set, prints the heap profile in text format. Else use the raw binary serialization format.")
MEMPROF_FLAG(bool, print_terse, false,
- "If set, prints memory profile in a terse format.")
-
-MEMPROF_FLAG(
- int, mem_info_cache_entries, 16381,
- "Size in entries of the mem info block cache, should be closest prime"
- " number to a power of two for best hashing.")
-MEMPROF_FLAG(bool, print_mem_info_cache_miss_rate, false,
- "If set, prints the miss rate of the mem info block cache.")
-MEMPROF_FLAG(
- bool, print_mem_info_cache_miss_rate_details, false,
- "If set, prints detailed miss rates of the mem info block cache sets.")
+ "If set, prints memory profile in a terse format. Only applicable if print_text = true.")
do { \
} while (false)
#define COMMON_INTERCEPTOR_BLOCK_REAL(name) REAL(name)
-#define COMMON_INTERCEPTOR_ON_DLOPEN(filename, flag) \
- do { \
- CheckNoDeepBind(filename, flag); \
- } while (false)
#define COMMON_INTERCEPTOR_ON_EXIT(ctx) OnExit()
#define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, handle)
#define COMMON_INTERCEPTOR_LIBRARY_UNLOADED()
void *ctx;
MEMPROF_INTERCEPTOR_ENTER(ctx, strcat);
ENSURE_MEMPROF_INITED();
- uptr from_length = REAL(strlen)(from);
+ uptr from_length = internal_strlen(from);
MEMPROF_READ_RANGE(from, from_length + 1);
- uptr to_length = REAL(strlen)(to);
+ uptr to_length = internal_strlen(to);
MEMPROF_READ_STRING(to, to_length);
MEMPROF_WRITE_RANGE(to + to_length, from_length + 1);
return REAL(strcat)(to, from);
uptr from_length = MaybeRealStrnlen(from, size);
uptr copy_length = Min(size, from_length + 1);
MEMPROF_READ_RANGE(from, copy_length);
- uptr to_length = REAL(strlen)(to);
+ uptr to_length = internal_strlen(to);
MEMPROF_READ_STRING(to, to_length);
MEMPROF_WRITE_RANGE(to + to_length, from_length + 1);
return REAL(strncat)(to, from, size);
return REAL(strcpy)(to, from);
}
ENSURE_MEMPROF_INITED();
- uptr from_size = REAL(strlen)(from) + 1;
+ uptr from_size = internal_strlen(from) + 1;
MEMPROF_READ_RANGE(from, from_size);
MEMPROF_WRITE_RANGE(to, from_size);
return REAL(strcpy)(to, from);
if (UNLIKELY(!memprof_inited))
return internal_strdup(s);
ENSURE_MEMPROF_INITED();
- uptr length = REAL(strlen)(s);
+ uptr length = internal_strlen(s);
MEMPROF_READ_RANGE(s, length + 1);
GET_STACK_TRACE_MALLOC;
void *new_mem = memprof_malloc(length + 1, &stack);
if (UNLIKELY(!memprof_inited))
return internal_strdup(s);
ENSURE_MEMPROF_INITED();
- uptr length = REAL(strlen)(s);
+ uptr length = internal_strlen(s);
MEMPROF_READ_RANGE(s, length + 1);
GET_STACK_TRACE_MALLOC;
void *new_mem = memprof_malloc(length + 1, &stack);
#define MEMPROF_INTERCEPT_FUNC_VER(name, ver) \
do { \
if (!INTERCEPT_FUNCTION_VER(name, ver)) \
- VReport(1, "MemProfiler: failed to intercept '%s@@%s'\n", #name, #ver); \
+ VReport(1, "MemProfiler: failed to intercept '%s@@%s'\n", #name, ver); \
} while (0)
#define MEMPROF_INTERCEPT_FUNC_VER_UNVERSIONED_FALLBACK(name, ver) \
do { \
if (!INTERCEPT_FUNCTION_VER(name, ver) && !INTERCEPT_FUNCTION(name)) \
VReport(1, "MemProfiler: failed to intercept '%s@@%s' or '%s'\n", #name, \
- #ver, #name); \
+ ver, #name); \
} while (0)
#endif // MEMPROF_INTERCEPTORS_H
// memprof_thread.cpp
MemprofThread *CreateMainThread();
-void ReadContextStack(void *context, uptr *stack, uptr *ssize);
-
// Wrapper for TLS/TSD.
void TSDInit(void (*destructor)(void *tsd));
void *TSDGet();
void *MemprofDlSymNext(const char *sym);
-// Add convenient macro for interface functions that may be represented as
-// weak hooks.
-#define MEMPROF_MALLOC_HOOK(ptr, size) \
- do { \
- if (&__sanitizer_malloc_hook) \
- __sanitizer_malloc_hook(ptr, size); \
- RunMallocHooks(ptr, size); \
- } while (false)
-#define MEMPROF_FREE_HOOK(ptr) \
- do { \
- if (&__sanitizer_free_hook) \
- __sanitizer_free_hook(ptr); \
- RunFreeHooks(ptr); \
- } while (false)
-
extern int memprof_inited;
extern int memprof_timestamp_inited;
extern int memprof_init_done;
/*min_shadow_base_alignment*/ 0, kHighMemEnd);
}
-void ReadContextStack(void *context, uptr *stack, uptr *ssize) {
- ucontext_t *ucp = (ucontext_t *)context;
- *stack = (uptr)ucp->uc_stack.ss_sp;
- *ssize = ucp->uc_stack.ss_size;
-}
-
void *MemprofDlSymNext(const char *sym) { return dlsym(RTLD_NEXT, sym); }
} // namespace __memprof
#include "memprof_internal.h"
#include "memprof_stack.h"
#include "sanitizer_common/sanitizer_allocator_checks.h"
+#include "sanitizer_common/sanitizer_allocator_dlsym.h"
#include "sanitizer_common/sanitizer_errno.h"
#include "sanitizer_common/sanitizer_tls_get_addr.h"
// ---------------------- Replacement functions ---------------- {{{1
using namespace __memprof;
-static uptr allocated_for_dlsym;
-static uptr last_dlsym_alloc_size_in_words;
-static const uptr kDlsymAllocPoolSize = 1024;
-static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize];
-
-static inline bool IsInDlsymAllocPool(const void *ptr) {
- uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
- return off < allocated_for_dlsym * sizeof(alloc_memory_for_dlsym[0]);
-}
-
-static void *AllocateFromLocalPool(uptr size_in_bytes) {
- uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize;
- void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym];
- last_dlsym_alloc_size_in_words = size_in_words;
- allocated_for_dlsym += size_in_words;
- CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize);
- return mem;
-}
-
-static void DeallocateFromLocalPool(const void *ptr) {
- // Hack: since glibc 2.27 dlsym no longer uses stack-allocated memory to store
- // error messages and instead uses malloc followed by free. To avoid pool
- // exhaustion due to long object filenames, handle that special case here.
- uptr prev_offset = allocated_for_dlsym - last_dlsym_alloc_size_in_words;
- void *prev_mem = (void *)&alloc_memory_for_dlsym[prev_offset];
- if (prev_mem == ptr) {
- REAL(memset)(prev_mem, 0, last_dlsym_alloc_size_in_words * kWordSize);
- allocated_for_dlsym = prev_offset;
- last_dlsym_alloc_size_in_words = 0;
- }
-}
-
-static int PosixMemalignFromLocalPool(void **memptr, uptr alignment,
- uptr size_in_bytes) {
- if (UNLIKELY(!CheckPosixMemalignAlignment(alignment)))
- return errno_EINVAL;
-
- CHECK(alignment >= kWordSize);
-
- uptr addr = (uptr)&alloc_memory_for_dlsym[allocated_for_dlsym];
- uptr aligned_addr = RoundUpTo(addr, alignment);
- uptr aligned_size = RoundUpTo(size_in_bytes, kWordSize);
-
- uptr *end_mem = (uptr *)(aligned_addr + aligned_size);
- uptr allocated = end_mem - alloc_memory_for_dlsym;
- if (allocated >= kDlsymAllocPoolSize)
- return errno_ENOMEM;
-
- allocated_for_dlsym = allocated;
- *memptr = (void *)aligned_addr;
- return 0;
-}
-
-static inline bool MaybeInDlsym() { return memprof_init_is_running; }
-
-static inline bool UseLocalPool() { return MaybeInDlsym(); }
-
-static void *ReallocFromLocalPool(void *ptr, uptr size) {
- const uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
- const uptr copy_size = Min(size, kDlsymAllocPoolSize - offset);
- void *new_ptr;
- if (UNLIKELY(UseLocalPool())) {
- new_ptr = AllocateFromLocalPool(size);
- } else {
- ENSURE_MEMPROF_INITED();
- GET_STACK_TRACE_MALLOC;
- new_ptr = memprof_malloc(size, &stack);
- }
- internal_memcpy(new_ptr, ptr, copy_size);
- return new_ptr;
-}
+struct DlsymAlloc : public DlSymAllocator<DlsymAlloc> {
+ static bool UseImpl() { return memprof_init_is_running; }
+};
INTERCEPTOR(void, free, void *ptr) {
+ if (DlsymAlloc::PointerIsMine(ptr))
+ return DlsymAlloc::Free(ptr);
GET_STACK_TRACE_FREE;
- if (UNLIKELY(IsInDlsymAllocPool(ptr))) {
- DeallocateFromLocalPool(ptr);
- return;
- }
memprof_free(ptr, &stack, FROM_MALLOC);
}
#if SANITIZER_INTERCEPT_CFREE
INTERCEPTOR(void, cfree, void *ptr) {
+ if (DlsymAlloc::PointerIsMine(ptr))
+ return DlsymAlloc::Free(ptr);
GET_STACK_TRACE_FREE;
- if (UNLIKELY(IsInDlsymAllocPool(ptr)))
- return;
memprof_free(ptr, &stack, FROM_MALLOC);
}
#endif // SANITIZER_INTERCEPT_CFREE
INTERCEPTOR(void *, malloc, uptr size) {
- if (UNLIKELY(UseLocalPool()))
- // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym.
- return AllocateFromLocalPool(size);
+ if (DlsymAlloc::Use())
+ return DlsymAlloc::Allocate(size);
ENSURE_MEMPROF_INITED();
GET_STACK_TRACE_MALLOC;
return memprof_malloc(size, &stack);
}
INTERCEPTOR(void *, calloc, uptr nmemb, uptr size) {
- if (UNLIKELY(UseLocalPool()))
- // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
- return AllocateFromLocalPool(nmemb * size);
+ if (DlsymAlloc::Use())
+ return DlsymAlloc::Callocate(nmemb, size);
ENSURE_MEMPROF_INITED();
GET_STACK_TRACE_MALLOC;
return memprof_calloc(nmemb, size, &stack);
}
INTERCEPTOR(void *, realloc, void *ptr, uptr size) {
- if (UNLIKELY(IsInDlsymAllocPool(ptr)))
- return ReallocFromLocalPool(ptr, size);
- if (UNLIKELY(UseLocalPool()))
- return AllocateFromLocalPool(size);
+ if (DlsymAlloc::Use() || DlsymAlloc::PointerIsMine(ptr))
+ return DlsymAlloc::Realloc(ptr, size);
ENSURE_MEMPROF_INITED();
GET_STACK_TRACE_MALLOC;
return memprof_realloc(ptr, size, &stack);
#endif // SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO
INTERCEPTOR(int, posix_memalign, void **memptr, uptr alignment, uptr size) {
- if (UNLIKELY(UseLocalPool()))
- return PosixMemalignFromLocalPool(memptr, alignment, size);
GET_STACK_TRACE_MALLOC;
return memprof_posix_memalign(memptr, alignment, size, &stack);
}
--- /dev/null
+//===-- memprof_mibmap.cpp -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of MemProfiler, a memory profiler.
+//
+//===----------------------------------------------------------------------===//
+
+#include "memprof_mibmap.h"
+#include "profile/MemProfData.inc"
+#include "sanitizer_common/sanitizer_allocator_internal.h"
+#include "sanitizer_common/sanitizer_mutex.h"
+
+namespace __memprof {
+using ::llvm::memprof::MemInfoBlock;
+
+void InsertOrMerge(const uptr Id, const MemInfoBlock &Block, MIBMapTy &Map) {
+ MIBMapTy::Handle h(&Map, static_cast<uptr>(Id), /*remove=*/false,
+ /*create=*/true);
+ if (h.created()) {
+ LockedMemInfoBlock *lmib =
+ (LockedMemInfoBlock *)InternalAlloc(sizeof(LockedMemInfoBlock));
+ lmib->mutex.Init();
+ lmib->mib = Block;
+ *h = lmib;
+ } else {
+ LockedMemInfoBlock *lmib = *h;
+ SpinMutexLock lock(&lmib->mutex);
+ lmib->mib.Merge(Block);
+ }
+}
+
+} // namespace __memprof
--- /dev/null
+#ifndef MEMPROF_MIBMAP_H_
+#define MEMPROF_MIBMAP_H_
+
+#include <stdint.h>
+
+#include "profile/MemProfData.inc"
+#include "sanitizer_common/sanitizer_addrhashmap.h"
+#include "sanitizer_common/sanitizer_mutex.h"
+
+namespace __memprof {
+
+struct LockedMemInfoBlock {
+ __sanitizer::StaticSpinMutex mutex;
+ ::llvm::memprof::MemInfoBlock mib;
+};
+
+// The MIB map stores a mapping from stack ids to MemInfoBlocks.
+typedef __sanitizer::AddrHashMap<LockedMemInfoBlock *, 200003> MIBMapTy;
+
+// Insert a new MemInfoBlock or merge with an existing block identified by the
+// stack id.
+void InsertOrMerge(const uptr Id, const ::llvm::memprof::MemInfoBlock &Block,
+ MIBMapTy &Map);
+
+} // namespace __memprof
+
+#endif // MEMPROF_MIBMAP_H_
--- /dev/null
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "memprof_rawprofile.h"
+#include "profile/MemProfData.inc"
+#include "sanitizer_common/sanitizer_allocator_internal.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_linux.h"
+#include "sanitizer_common/sanitizer_procmaps.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
+#include "sanitizer_common/sanitizer_stackdepotbase.h"
+#include "sanitizer_common/sanitizer_stacktrace.h"
+#include "sanitizer_common/sanitizer_vector.h"
+
+namespace __memprof {
+using ::__sanitizer::Vector;
+using ::llvm::memprof::MemInfoBlock;
+using SegmentEntry = ::llvm::memprof::SegmentEntry;
+using Header = ::llvm::memprof::Header;
+
+namespace {
+template <class T> char *WriteBytes(T Pod, char *&Buffer) {
+ *(T *)Buffer = Pod;
+ return Buffer + sizeof(T);
+}
+
+void RecordStackId(const uptr Key, UNUSED LockedMemInfoBlock *const &MIB,
+ void *Arg) {
+ // No need to touch the MIB value here since we are only recording the key.
+ auto *StackIds = reinterpret_cast<Vector<u64> *>(Arg);
+ StackIds->PushBack(Key);
+}
+} // namespace
+
+u64 SegmentSizeBytes(MemoryMappingLayoutBase &Layout) {
+ u64 NumSegmentsToRecord = 0;
+ MemoryMappedSegment segment;
+ for (Layout.Reset(); Layout.Next(&segment);)
+ if (segment.IsReadable() && segment.IsExecutable())
+ NumSegmentsToRecord++;
+
+ return sizeof(u64) // A header which stores the number of records.
+ + sizeof(SegmentEntry) * NumSegmentsToRecord;
+}
+
+// The segment section uses the following format:
+// ---------- Segment Info
+// Num Entries
+// ---------- Segment Entry
+// Start
+// End
+// Offset
+// BuildID 32B
+// ----------
+// ...
+void SerializeSegmentsToBuffer(MemoryMappingLayoutBase &Layout,
+ const u64 ExpectedNumBytes, char *&Buffer) {
+ char *Ptr = Buffer;
+ // Reserve space for the final count.
+ Ptr += sizeof(u64);
+
+ u64 NumSegmentsRecorded = 0;
+ MemoryMappedSegment segment;
+
+ for (Layout.Reset(); Layout.Next(&segment);) {
+ if (segment.IsReadable() && segment.IsExecutable()) {
+ // TODO: Record segment.uuid when it is implemented for Linux-Elf.
+ SegmentEntry Entry(segment.start, segment.end, segment.offset);
+ memcpy(Ptr, &Entry, sizeof(SegmentEntry));
+ Ptr += sizeof(SegmentEntry);
+ NumSegmentsRecorded++;
+ }
+ }
+
+ // Store the number of segments we recorded in the space we reserved.
+ *((u64 *)Buffer) = NumSegmentsRecorded;
+ CHECK(ExpectedNumBytes >= static_cast<u64>(Ptr - Buffer) &&
+ "Expected num bytes != actual bytes written");
+}
+
+u64 StackSizeBytes(const Vector<u64> &StackIds) {
+ u64 NumBytesToWrite = sizeof(u64);
+
+ const u64 NumIds = StackIds.Size();
+ for (unsigned k = 0; k < NumIds; ++k) {
+ const u64 Id = StackIds[k];
+ // One entry for the id and then one more for the number of stack pcs.
+ NumBytesToWrite += 2 * sizeof(u64);
+ const StackTrace St = StackDepotGet(Id);
+
+ CHECK(St.trace != nullptr && St.size > 0 && "Empty stack trace");
+ for (uptr i = 0; i < St.size && St.trace[i] != 0; i++) {
+ NumBytesToWrite += sizeof(u64);
+ }
+ }
+ return NumBytesToWrite;
+}
+
+// The stack info section uses the following format:
+//
+// ---------- Stack Info
+// Num Entries
+// ---------- Stack Entry
+// Num Stacks
+// PC1
+// PC2
+// ...
+// ----------
+void SerializeStackToBuffer(const Vector<u64> &StackIds,
+ const u64 ExpectedNumBytes, char *&Buffer) {
+ const u64 NumIds = StackIds.Size();
+ char *Ptr = Buffer;
+ Ptr = WriteBytes(static_cast<u64>(NumIds), Ptr);
+
+ for (unsigned k = 0; k < NumIds; ++k) {
+ const u64 Id = StackIds[k];
+ Ptr = WriteBytes(Id, Ptr);
+ Ptr += sizeof(u64); // Bump it by u64, we will fill this in later.
+ u64 Count = 0;
+ const StackTrace St = StackDepotGet(Id);
+ for (uptr i = 0; i < St.size && St.trace[i] != 0; i++) {
+ // PCs in stack traces are actually the return addresses, that is,
+ // addresses of the next instructions after the call.
+ uptr pc = StackTrace::GetPreviousInstructionPc(St.trace[i]);
+ Ptr = WriteBytes(static_cast<u64>(pc), Ptr);
+ ++Count;
+ }
+ // Store the count in the space we reserved earlier.
+ *(u64 *)(Ptr - (Count + 1) * sizeof(u64)) = Count;
+ }
+
+ CHECK(ExpectedNumBytes >= static_cast<u64>(Ptr - Buffer) &&
+ "Expected num bytes != actual bytes written");
+}
+
+// The MIB section has the following format:
+// ---------- MIB Info
+// Num Entries
+// ---------- MIB Entry 0
+// Alloc Count
+// ...
+// ---------- MIB Entry 1
+// Alloc Count
+// ...
+// ----------
+void SerializeMIBInfoToBuffer(MIBMapTy &MIBMap, const Vector<u64> &StackIds,
+ const u64 ExpectedNumBytes, char *&Buffer) {
+ char *Ptr = Buffer;
+ const u64 NumEntries = StackIds.Size();
+ Ptr = WriteBytes(NumEntries, Ptr);
+
+ for (u64 i = 0; i < NumEntries; i++) {
+ const u64 Key = StackIds[i];
+ MIBMapTy::Handle h(&MIBMap, Key, /*remove=*/true, /*create=*/false);
+ CHECK(h.exists());
+ Ptr = WriteBytes(Key, Ptr);
+ Ptr = WriteBytes((*h)->mib, Ptr);
+ }
+
+ CHECK(ExpectedNumBytes >= static_cast<u64>(Ptr - Buffer) &&
+ "Expected num bytes != actual bytes written");
+}
+
+// Format
+// ---------- Header
+// Magic
+// Version
+// Total Size
+// Segment Offset
+// MIB Info Offset
+// Stack Offset
+// ---------- Segment Info
+// Num Entries
+// ---------- Segment Entry
+// Start
+// End
+// Offset
+// BuildID 32B
+// ----------
+// ...
+// ----------
+// Optional Padding Bytes
+// ---------- MIB Info
+// Num Entries
+// ---------- MIB Entry
+// Alloc Count
+// ...
+// ----------
+// Optional Padding Bytes
+// ---------- Stack Info
+// Num Entries
+// ---------- Stack Entry
+// Num Stacks
+// PC1
+// PC2
+// ...
+// ----------
+// Optional Padding Bytes
+// ...
+u64 SerializeToRawProfile(MIBMapTy &MIBMap, MemoryMappingLayoutBase &Layout,
+ char *&Buffer) {
+ // Each section size is rounded up to 8b since the first entry in each section
+ // is a u64 which holds the number of entries in the section by convention.
+ const u64 NumSegmentBytes = RoundUpTo(SegmentSizeBytes(Layout), 8);
+
+ Vector<u64> StackIds;
+ MIBMap.ForEach(RecordStackId, reinterpret_cast<void *>(&StackIds));
+ // The first 8b are for the total number of MIB records. Each MIB record is
+ // preceded by a 8b stack id which is associated with stack frames in the next
+ // section.
+ const u64 NumMIBInfoBytes = RoundUpTo(
+ sizeof(u64) + StackIds.Size() * (sizeof(u64) + sizeof(MemInfoBlock)), 8);
+
+ const u64 NumStackBytes = RoundUpTo(StackSizeBytes(StackIds), 8);
+
+ // Ensure that the profile is 8b aligned. We allow for some optional padding
+ // at the end so that any subsequent profile serialized to the same file does
+ // not incur unaligned accesses.
+ const u64 TotalSizeBytes = RoundUpTo(
+ sizeof(Header) + NumSegmentBytes + NumStackBytes + NumMIBInfoBytes, 8);
+
+ // Allocate the memory for the entire buffer incl. info blocks.
+ Buffer = (char *)InternalAlloc(TotalSizeBytes);
+ char *Ptr = Buffer;
+
+ Header header{MEMPROF_RAW_MAGIC_64,
+ MEMPROF_RAW_VERSION,
+ static_cast<u64>(TotalSizeBytes),
+ sizeof(Header),
+ sizeof(Header) + NumSegmentBytes,
+ sizeof(Header) + NumSegmentBytes + NumMIBInfoBytes};
+ Ptr = WriteBytes(header, Ptr);
+
+ SerializeSegmentsToBuffer(Layout, NumSegmentBytes, Ptr);
+ Ptr += NumSegmentBytes;
+
+ SerializeMIBInfoToBuffer(MIBMap, StackIds, NumMIBInfoBytes, Ptr);
+ Ptr += NumMIBInfoBytes;
+
+ SerializeStackToBuffer(StackIds, NumStackBytes, Ptr);
+
+ return TotalSizeBytes;
+}
+
+} // namespace __memprof
--- /dev/null
+#ifndef MEMPROF_RAWPROFILE_H_
+#define MEMPROF_RAWPROFILE_H_
+
+#include "memprof_mibmap.h"
+#include "sanitizer_common/sanitizer_procmaps.h"
+
+namespace __memprof {
+// Serialize the in-memory representation of the memprof profile to the raw
+// binary format. The format itself is documented memprof_rawprofile.cpp.
+u64 SerializeToRawProfile(MIBMapTy &BlockCache, MemoryMappingLayoutBase &Layout,
+ char *&Buffer);
+} // namespace __memprof
+
+#endif // MEMPROF_RAWPROFILE_H_
#include "memprof_thread.h"
#include "sanitizer_common/sanitizer_atomic.h"
#include "sanitizer_common/sanitizer_flags.h"
+#include "sanitizer_common/sanitizer_interface_internal.h"
#include "sanitizer_common/sanitizer_libc.h"
#include "sanitizer_common/sanitizer_symbolizer.h"
if (atomic_fetch_add(&num_calls, 1, memory_order_relaxed) != 0) {
// Don't die twice - run a busy loop.
while (1) {
+ internal_sched_yield();
}
}
if (common_flags()->print_module_map >= 1)
}
}
+static void MemprofOnDeadlySignal(int signo, void *siginfo, void *context) {
+ // We call StartReportDeadlySignal not HandleDeadlySignal so we get the
+ // deadly signal message to stderr but no writing to the profile output file
+ StartReportDeadlySignal();
+ __memprof_profile_dump();
+ Die();
+}
+
static void CheckUnwind() {
GET_STACK_TRACE(kStackTraceMax, common_flags()->fast_unwind_on_check);
stack.Print();
CHECK(SHADOW_SCALE >= 3 && SHADOW_SCALE <= 7);
}
-static bool UNUSED __local_memprof_dyninit = [] {
- MaybeStartBackgroudThread();
- SetSoftRssLimitExceededCallback(MemprofSoftRssLimitExceededCallback);
-
- return false;
-}();
-
static void MemprofInitInternal() {
if (LIKELY(memprof_inited))
return;
__sanitizer::InitializePlatformEarly();
- // Re-exec ourselves if we need to set additional env or command line args.
- MaybeReexec();
-
// Setup internal allocator callback.
SetLowLevelAllocateMinAlignment(SHADOW_GRANULARITY);
InitializeShadowMemory();
TSDInit(PlatformTSDDtor);
+ InstallDeadlySignalHandlers(MemprofOnDeadlySignal);
InitializeAllocator();
__memprof::RecordAccess((uptr)addr);
}
-// We only record the access on the first location in the range,
-// since we will later accumulate the access counts across the
-// full allocation, and we don't want to inflate the hotness from
-// a memory intrinsic on a large range of memory.
-// TODO: Should we do something else so we can better track utilization?
-void __memprof_record_access_range(void const volatile *addr,
- UNUSED uptr size) {
- __memprof::RecordAccess((uptr)addr);
+void __memprof_record_access_range(void const volatile *addr, uptr size) {
+ for (uptr a = (uptr)addr; a < (uptr)addr + size; a += kWordSize)
+ __memprof::RecordAccess(a);
}
extern "C" SANITIZER_INTERFACE_ATTRIBUTE u16
dst_ptr[i] += src_ptr[i];
}
-static BlockingMutex print_lock(LINKER_INITIALIZED);
+static Mutex print_lock;
static MemprofStats unknown_thread_stats(LINKER_INITIALIZED);
static MemprofStats dead_threads_stats(LINKER_INITIALIZED);
-static BlockingMutex dead_threads_stats_lock(LINKER_INITIALIZED);
+static Mutex dead_threads_stats_lock;
// Required for malloc_zone_statistics() on OS X. This can't be stored in
// per-thread MemprofStats.
static uptr max_malloced_memory;
}
stats->MergeFrom(&unknown_thread_stats);
{
- BlockingMutexLock lock(&dead_threads_stats_lock);
+ Lock lock(&dead_threads_stats_lock);
stats->MergeFrom(&dead_threads_stats);
}
// This is not very accurate: we may miss allocation peaks that happen
}
void FlushToDeadThreadStats(MemprofStats *stats) {
- BlockingMutexLock lock(&dead_threads_stats_lock);
+ Lock lock(&dead_threads_stats_lock);
dead_threads_stats.MergeFrom(stats);
stats->Clear();
}
MemprofStats stats;
GetAccumulatedStats(&stats);
// Use lock to keep reports from mixing up.
- BlockingMutexLock lock(&print_lock);
+ Lock lock(&print_lock);
stats.Print();
- StackDepotStats *stack_depot_stats = StackDepotGetStats();
+ StackDepotStats stack_depot_stats = StackDepotGetStats();
Printf("Stats: StackDepot: %zd ids; %zdM allocated\n",
- stack_depot_stats->n_uniq_ids, stack_depot_stats->allocated >> 20);
+ stack_depot_stats.n_uniq_ids, stack_depot_stats.allocated >> 20);
PrintInternalAllocatorStats();
}
static ALIGNED(16) char thread_registry_placeholder[sizeof(ThreadRegistry)];
static ThreadRegistry *memprof_thread_registry;
-static BlockingMutex mu_for_thread_context(LINKER_INITIALIZED);
+static Mutex mu_for_thread_context;
static LowLevelAllocator allocator_for_thread_context;
static ThreadContextBase *GetMemprofThreadContext(u32 tid) {
- BlockingMutexLock lock(&mu_for_thread_context);
+ Lock lock(&mu_for_thread_context);
return new (allocator_for_thread_context) MemprofThreadContext(tid);
}
thread->start_routine_ = start_routine;
thread->arg_ = arg;
MemprofThreadContext::CreateThreadContextArgs args = {thread, stack};
- memprofThreadRegistry().CreateThread(*reinterpret_cast<uptr *>(thread),
- detached, parent_tid, &args);
+ memprofThreadRegistry().CreateThread(0, detached, parent_tid, &args);
return thread;
}
int local = 0;
VReport(1, "T%d: stack [%p,%p) size 0x%zx; local=%p\n", tid(),
(void *)stack_bottom_, (void *)stack_top_, stack_top_ - stack_bottom_,
- &local);
+ (void *)&local);
}
thread_return_t
void SetCurrentThread(MemprofThread *t) {
CHECK(t->context());
- VReport(2, "SetCurrentThread: %p for thread %p\n", t->context(),
+ VReport(2, "SetCurrentThread: %p for thread %p\n", (void *)t->context(),
(void *)GetThreadSelf());
// Make sure we do not reset the current MemprofThread.
CHECK_EQ(0, TSDGet());
--- /dev/null
+include(CheckCXXCompilerFlag)
+include(CompilerRTCompile)
+include(CompilerRTLink)
+
+set(MEMPROF_UNITTEST_CFLAGS
+ ${COMPILER_RT_UNITTEST_CFLAGS}
+ ${COMPILER_RT_GTEST_CFLAGS}
+ ${COMPILER_RT_GMOCK_CFLAGS}
+ ${SANITIZER_TEST_CXX_CFLAGS}
+ -I${COMPILER_RT_SOURCE_DIR}/lib/
+ -O2
+ -g
+ -fno-rtti
+ -Wno-pedantic
+ -fno-omit-frame-pointer)
+
+# Suppress warnings for gmock variadic macros for clang and gcc respectively.
+append_list_if(SUPPORTS_GNU_ZERO_VARIADIC_MACRO_ARGUMENTS_FLAG -Wno-gnu-zero-variadic-macro-arguments MEMPROF_UNITTEST_CFLAGS)
+append_list_if(COMPILER_RT_HAS_WVARIADIC_MACROS_FLAG -Wno-variadic-macros MEMPROF_UNITTEST_CFLAGS)
+
+file(GLOB MEMPROF_HEADERS ../*.h)
+
+set(MEMPROF_SOURCES
+ ../memprof_mibmap.cpp
+ ../memprof_rawprofile.cpp)
+
+set(MEMPROF_UNITTESTS
+ rawprofile.cpp
+ driver.cpp)
+
+include_directories(../../../include)
+
+set(MEMPROF_UNIT_TEST_HEADERS
+ ${MEMPROF_HEADERS})
+
+set(MEMPROF_UNITTEST_LINK_FLAGS
+ ${COMPILER_RT_UNITTEST_LINK_FLAGS})
+
+if(NOT WIN32)
+ list(APPEND MEMPROF_UNITTEST_LINK_FLAGS -pthread)
+endif()
+
+set(MEMPROF_UNITTEST_LINK_LIBRARIES
+ ${COMPILER_RT_UNWINDER_LINK_LIBS}
+ ${SANITIZER_TEST_CXX_LIBRARIES})
+list(APPEND MEMPROF_UNITTEST_LINK_LIBRARIES "dl")
+
+if(COMPILER_RT_DEFAULT_TARGET_ARCH IN_LIST MEMPROF_SUPPORTED_ARCH)
+ # MemProf unit tests are only run on the host machine.
+ set(arch ${COMPILER_RT_DEFAULT_TARGET_ARCH})
+
+ add_executable(MemProfUnitTests
+ ${MEMPROF_UNITTESTS}
+ ${COMPILER_RT_GTEST_SOURCE}
+ ${COMPILER_RT_GMOCK_SOURCE}
+ ${MEMPROF_SOURCES}
+ $<TARGET_OBJECTS:RTSanitizerCommon.${arch}>
+ $<TARGET_OBJECTS:RTSanitizerCommonCoverage.${arch}>
+ $<TARGET_OBJECTS:RTSanitizerCommonLibc.${arch}>
+ $<TARGET_OBJECTS:RTSanitizerCommonSymbolizer.${arch}>)
+ set_target_compile_flags(MemProfUnitTests ${MEMPROF_UNITTEST_CFLAGS})
+ set_target_link_flags(MemProfUnitTests ${MEMPROF_UNITTEST_LINK_FLAGS})
+ target_link_libraries(MemProfUnitTests ${MEMPROF_UNITTEST_LINK_LIBRARIES})
+
+ if (TARGET cxx-headers OR HAVE_LIBCXX)
+ add_dependencies(MemProfUnitTests cxx-headers)
+ endif()
+
+ set_target_properties(MemProfUnitTests PROPERTIES
+ RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
+endif()
--- /dev/null
+//===-- driver.cpp ----------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "gtest/gtest.h"
+
+int main(int argc, char **argv) {
+ testing::InitGoogleTest(&argc, argv);
+ return RUN_ALL_TESTS();
+}
--- /dev/null
+#include "memprof/memprof_rawprofile.h"
+
+#include <cstdint>
+#include <memory>
+
+#include "profile/MemProfData.inc"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_procmaps.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
+#include "sanitizer_common/sanitizer_stacktrace.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+namespace {
+
+using ::__memprof::MIBMapTy;
+using ::__memprof::SerializeToRawProfile;
+using ::__sanitizer::MemoryMappedSegment;
+using ::__sanitizer::MemoryMappingLayoutBase;
+using ::__sanitizer::StackDepotPut;
+using ::__sanitizer::StackTrace;
+using ::llvm::memprof::MemInfoBlock;
+using ::testing::_;
+using ::testing::Action;
+using ::testing::DoAll;
+using ::testing::Return;
+using ::testing::SetArgPointee;
+
+class MockMemoryMappingLayout final : public MemoryMappingLayoutBase {
+public:
+ MOCK_METHOD(bool, Next, (MemoryMappedSegment *), (override));
+ MOCK_METHOD(void, Reset, (), (override));
+};
+
+uint64_t PopulateFakeMap(const MemInfoBlock &FakeMIB, uint64_t StackPCBegin,
+ MIBMapTy &FakeMap) {
+ constexpr int kSize = 5;
+ uint64_t array[kSize];
+ for (int i = 0; i < kSize; i++) {
+ array[i] = StackPCBegin + i;
+ }
+ StackTrace St(array, kSize);
+ uint32_t Id = StackDepotPut(St);
+
+ InsertOrMerge(Id, FakeMIB, FakeMap);
+ return Id;
+}
+
+template <class T = uint64_t> T Read(char *&Buffer) {
+ static_assert(std::is_pod<T>::value, "Must be a POD type.");
+ assert(reinterpret_cast<size_t>(Buffer) % sizeof(T) == 0 &&
+ "Unaligned read!");
+ T t = *reinterpret_cast<T *>(Buffer);
+ Buffer += sizeof(T);
+ return t;
+}
+
+TEST(MemProf, Basic) {
+ MockMemoryMappingLayout Layout;
+ MemoryMappedSegment FakeSegment;
+ memset(&FakeSegment, 0, sizeof(FakeSegment));
+ FakeSegment.start = 0x10;
+ FakeSegment.end = 0x20;
+ FakeSegment.offset = 0x10;
+ uint8_t uuid[__sanitizer::kModuleUUIDSize] = {0xC, 0x0, 0xF, 0xF, 0xE, 0xE};
+ memcpy(FakeSegment.uuid, uuid, __sanitizer::kModuleUUIDSize);
+ FakeSegment.protection =
+ __sanitizer::kProtectionExecute | __sanitizer::kProtectionRead;
+
+ const Action<bool(MemoryMappedSegment *)> SetSegment =
+ DoAll(SetArgPointee<0>(FakeSegment), Return(true));
+ EXPECT_CALL(Layout, Next(_))
+ .WillOnce(SetSegment)
+ .WillOnce(Return(false))
+ .WillOnce(SetSegment)
+ .WillRepeatedly(Return(false));
+
+ EXPECT_CALL(Layout, Reset).Times(2);
+
+ MIBMapTy FakeMap;
+ MemInfoBlock FakeMIB;
+ // Since we want to override the constructor set vals to make it easier to
+ // test.
+ memset(&FakeMIB, 0, sizeof(MemInfoBlock));
+ FakeMIB.AllocCount = 0x1;
+ FakeMIB.TotalAccessCount = 0x2;
+
+ uint64_t FakeIds[2];
+ FakeIds[0] = PopulateFakeMap(FakeMIB, /*StackPCBegin=*/2, FakeMap);
+ FakeIds[1] = PopulateFakeMap(FakeMIB, /*StackPCBegin=*/3, FakeMap);
+
+ char *Ptr = nullptr;
+ uint64_t NumBytes = SerializeToRawProfile(FakeMap, Layout, Ptr);
+ const char *Buffer = Ptr;
+
+ ASSERT_GT(NumBytes, 0ULL);
+ ASSERT_TRUE(Ptr);
+
+ // Check the header.
+ EXPECT_THAT(Read(Ptr), MEMPROF_RAW_MAGIC_64);
+ EXPECT_THAT(Read(Ptr), MEMPROF_RAW_VERSION);
+ const uint64_t TotalSize = Read(Ptr);
+ const uint64_t SegmentOffset = Read(Ptr);
+ const uint64_t MIBOffset = Read(Ptr);
+ const uint64_t StackOffset = Read(Ptr);
+
+ // ============= Check sizes and padding.
+ EXPECT_EQ(TotalSize, NumBytes);
+ EXPECT_EQ(TotalSize % 8, 0ULL);
+
+ // Should be equal to the size of the raw profile header.
+ EXPECT_EQ(SegmentOffset, 48ULL);
+
+ // We expect only 1 segment entry, 8b for the count and 56b for SegmentEntry
+ // in memprof_rawprofile.cpp.
+ EXPECT_EQ(MIBOffset - SegmentOffset, 64ULL);
+
+ EXPECT_EQ(MIBOffset, 112ULL);
+ // We expect 2 mib entry, 8b for the count and sizeof(uint64_t) +
+ // sizeof(MemInfoBlock) contains stack id + MeminfoBlock.
+ EXPECT_EQ(StackOffset - MIBOffset, 8 + 2 * (8 + sizeof(MemInfoBlock)));
+
+ EXPECT_EQ(StackOffset, 336ULL);
+ // We expect 2 stack entries, with 5 frames - 8b for total count,
+ // 2 * (8b for id, 8b for frame count and 5*8b for fake frames).
+ // Since this is the last section, there may be additional padding at the end
+ // to make the total profile size 8b aligned.
+ EXPECT_GE(TotalSize - StackOffset, 8ULL + 2 * (8 + 8 + 5 * 8));
+
+ // ============= Check contents.
+ // The Uuid field is not yet populated on Linux-Elf by the sanitizer procmaps
+ // library, so we expect it to be filled with 0 for now.
+ unsigned char ExpectedSegmentBytes[64] = {
+ 0x01, 0, 0, 0, 0, 0, 0, 0, // Number of entries
+ 0x10, 0, 0, 0, 0, 0, 0, 0, // Start
+ 0x20, 0, 0, 0, 0, 0, 0, 0, // End
+ 0x10, 0, 0, 0, 0, 0, 0, 0, // Offset
+ 0x0, // Uuid
+ };
+ EXPECT_EQ(memcmp(Buffer + SegmentOffset, ExpectedSegmentBytes, 64), 0);
+
+ // Check that the number of entries is 2.
+ EXPECT_EQ(*reinterpret_cast<const uint64_t *>(Buffer + MIBOffset), 2ULL);
+ // Check that stack id is set.
+ EXPECT_EQ(*reinterpret_cast<const uint64_t *>(Buffer + MIBOffset + 8),
+ FakeIds[0]);
+
+ // Only check a few fields of the first MemInfoBlock.
+ unsigned char ExpectedMIBBytes[sizeof(MemInfoBlock)] = {
+ 0x01, 0, 0, 0, // Alloc count
+ 0x02, 0, 0, 0, // Total access count
+ };
+ // Compare contents of 1st MIB after skipping count and stack id.
+ EXPECT_EQ(
+ memcmp(Buffer + MIBOffset + 16, ExpectedMIBBytes, sizeof(MemInfoBlock)),
+ 0);
+ // Compare contents of 2nd MIB after skipping count and stack id for the first
+ // and only the id for the second.
+ EXPECT_EQ(memcmp(Buffer + MIBOffset + 16 + sizeof(MemInfoBlock) + 8,
+ ExpectedMIBBytes, sizeof(MemInfoBlock)),
+ 0);
+
+ // Check that the number of entries is 2.
+ EXPECT_EQ(*reinterpret_cast<const uint64_t *>(Buffer + StackOffset), 2ULL);
+ // Check that the 1st stack id is set.
+ EXPECT_EQ(*reinterpret_cast<const uint64_t *>(Buffer + StackOffset + 8),
+ FakeIds[0]);
+ // Contents are num pcs, value of each pc - 1.
+ unsigned char ExpectedStackBytes[2][6 * 8] = {
+ {
+ 0x5, 0, 0, 0, 0, 0, 0, 0, // Number of PCs
+ 0x1, 0, 0, 0, 0, 0, 0, 0, // PC ...
+ 0x2, 0, 0, 0, 0, 0, 0, 0, 0x3, 0, 0, 0, 0, 0, 0, 0,
+ 0x4, 0, 0, 0, 0, 0, 0, 0, 0x5, 0, 0, 0, 0, 0, 0, 0,
+ },
+ {
+ 0x5, 0, 0, 0, 0, 0, 0, 0, // Number of PCs
+ 0x2, 0, 0, 0, 0, 0, 0, 0, // PC ...
+ 0x3, 0, 0, 0, 0, 0, 0, 0, 0x4, 0, 0, 0, 0, 0, 0, 0,
+ 0x5, 0, 0, 0, 0, 0, 0, 0, 0x6, 0, 0, 0, 0, 0, 0, 0,
+ },
+ };
+ EXPECT_EQ(memcmp(Buffer + StackOffset + 16, ExpectedStackBytes[0],
+ sizeof(ExpectedStackBytes[0])),
+ 0);
+
+ // Check that the 2nd stack id is set.
+ EXPECT_EQ(
+ *reinterpret_cast<const uint64_t *>(Buffer + StackOffset + 8 + 6 * 8 + 8),
+ FakeIds[1]);
+
+ EXPECT_EQ(memcmp(Buffer + StackOffset + 16 + 6 * 8 + 8, ExpectedStackBytes[1],
+ sizeof(ExpectedStackBytes[1])),
+ 0);
+}
+
+} // namespace
//===----------------------------------------------------------------------===//
#include "msan.h"
+
#include "msan_chained_origin_depot.h"
#include "msan_origin.h"
+#include "msan_poisoning.h"
#include "msan_report.h"
#include "msan_thread.h"
-#include "msan_poisoning.h"
#include "sanitizer_common/sanitizer_atomic.h"
#include "sanitizer_common/sanitizer_common.h"
-#include "sanitizer_common/sanitizer_flags.h"
#include "sanitizer_common/sanitizer_flag_parser.h"
+#include "sanitizer_common/sanitizer_flags.h"
+#include "sanitizer_common/sanitizer_interface_internal.h"
#include "sanitizer_common/sanitizer_libc.h"
#include "sanitizer_common/sanitizer_procmaps.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
#include "sanitizer_common/sanitizer_stacktrace.h"
#include "sanitizer_common/sanitizer_symbolizer.h"
-#include "sanitizer_common/sanitizer_stackdepot.h"
#include "ubsan/ubsan_flags.h"
#include "ubsan/ubsan_init.h"
SANITIZER_INTERFACE_ATTRIBUTE
THREADLOCAL u32 __msan_origin_tls;
-static THREADLOCAL int is_in_symbolizer;
-
extern "C" SANITIZER_WEAK_ATTRIBUTE const int __msan_track_origins;
int __msan_get_track_origins() {
namespace __msan {
-void EnterSymbolizer() { ++is_in_symbolizer; }
-void ExitSymbolizer() { --is_in_symbolizer; }
-bool IsInSymbolizer() { return is_in_symbolizer; }
+static THREADLOCAL int is_in_symbolizer_or_unwinder;
+static void EnterSymbolizerOrUnwider() { ++is_in_symbolizer_or_unwinder; }
+static void ExitSymbolizerOrUnwider() { --is_in_symbolizer_or_unwinder; }
+bool IsInSymbolizerOrUnwider() { return is_in_symbolizer_or_unwinder; }
+
+struct UnwinderScope {
+ UnwinderScope() { EnterSymbolizerOrUnwider(); }
+ ~UnwinderScope() { ExitSymbolizerOrUnwider(); }
+};
static Flags msan_flags;
-Flags *flags() {
- return &msan_flags;
-}
+Flags *flags() { return &msan_flags; }
int msan_inited = 0;
bool msan_init_is_running;
if (f->store_context_size < 1) f->store_context_size = 1;
}
-void PrintWarning(uptr pc, uptr bp) {
- PrintWarningWithOrigin(pc, bp, __msan_origin_tls);
-}
-
void PrintWarningWithOrigin(uptr pc, uptr bp, u32 origin) {
if (msan_expect_umr) {
// Printf("Expected UMR\n");
return chained.raw_id();
}
-} // namespace __msan
+// Current implementation separates the 'id_ptr' from the 'descr' and makes
+// 'descr' constant.
+// Previous implementation 'descr' is created at compile time and contains
+// '----' in the beginning. When we see descr for the first time we replace
+// '----' with a uniq id and set the origin to (id | (31-th bit)).
+static inline void SetAllocaOrigin(void *a, uptr size, u32 *id_ptr, char *descr,
+ uptr pc) {
+ static const u32 dash = '-';
+ static const u32 first_timer =
+ dash + (dash << 8) + (dash << 16) + (dash << 24);
+ u32 id = *id_ptr;
+ if (id == 0 || id == first_timer) {
+ u32 idx = atomic_fetch_add(&NumStackOriginDescrs, 1, memory_order_relaxed);
+ CHECK_LT(idx, kNumStackOriginDescrs);
+ StackOriginDescr[idx] = descr;
+ StackOriginPC[idx] = pc;
+ id = Origin::CreateStackOrigin(idx).raw_id();
+ *id_ptr = id;
+ }
+ __msan_set_origin(a, size, id);
+}
+
+} // namespace __msan
void __sanitizer::BufferedStackTrace::UnwindImpl(
uptr pc, uptr bp, void *context, bool request_fast, u32 max_depth) {
MsanThread *t = GetCurrentThread();
if (!t || !StackTrace::WillUseFastUnwind(request_fast)) {
// Block reports from our interceptors during _Unwind_Backtrace.
- SymbolizerScope sym_scope;
+ UnwinderScope sym_scope;
return Unwind(max_depth, pc, bp, context, t ? t->stack_top() : 0,
t ? t->stack_bottom() : 0, false);
}
void __msan_warning() {
GET_CALLER_PC_BP_SP;
(void)sp;
- PrintWarning(pc, bp);
+ PrintWarningWithOrigin(pc, bp, 0);
if (__msan::flags()->halt_on_error) {
if (__msan::flags()->print_stats)
ReportStats();
void __msan_warning_noreturn() {
GET_CALLER_PC_BP_SP;
(void)sp;
- PrintWarning(pc, bp);
+ PrintWarningWithOrigin(pc, bp, 0);
if (__msan::flags()->print_stats)
ReportStats();
Printf("Exiting\n");
Die();
}
- Symbolizer::GetOrInit()->AddHooks(EnterSymbolizer, ExitSymbolizer);
+ Symbolizer::GetOrInit()->AddHooks(EnterSymbolizerOrUnwider,
+ ExitSymbolizerOrUnwider);
InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir);
MsanThread *main_thread = MsanThread::Create(nullptr, nullptr);
SetCurrentThread(main_thread);
- main_thread->ThreadStart();
+ main_thread->Init();
#if MSAN_CONTAINS_UBSAN
__ubsan::InitAsPlugin();
}
unsigned char *s = (unsigned char*)MEM_TO_SHADOW(x);
+ Printf("%p[%p] ", (void *)s, x);
for (uptr i = 0; i < size; i++)
Printf("%x%x ", s[i] >> 4, s[i] & 0xf);
Printf("\n");
if (__msan_get_track_origins()) SetOrigin(a, size, origin);
}
-// 'descr' is created at compile time and contains '----' in the beginning.
-// When we see descr for the first time we replace '----' with a uniq id
-// and set the origin to (id | (31-th bit)).
void __msan_set_alloca_origin(void *a, uptr size, char *descr) {
- __msan_set_alloca_origin4(a, size, descr, 0);
+ SetAllocaOrigin(a, size, reinterpret_cast<u32 *>(descr), descr + 4,
+ GET_CALLER_PC());
}
void __msan_set_alloca_origin4(void *a, uptr size, char *descr, uptr pc) {
- static const u32 dash = '-';
- static const u32 first_timer =
- dash + (dash << 8) + (dash << 16) + (dash << 24);
- u32 *id_ptr = (u32*)descr;
- bool print = false; // internal_strstr(descr + 4, "AllocaTOTest") != 0;
- u32 id = *id_ptr;
- if (id == first_timer) {
- u32 idx = atomic_fetch_add(&NumStackOriginDescrs, 1, memory_order_relaxed);
- CHECK_LT(idx, kNumStackOriginDescrs);
- StackOriginDescr[idx] = descr + 4;
-#if SANITIZER_PPC64V1
- // On PowerPC64 ELFv1, the address of a function actually points to a
- // three-doubleword data structure with the first field containing
- // the address of the function's code.
- if (pc)
- pc = *reinterpret_cast<uptr*>(pc);
-#endif
- StackOriginPC[idx] = pc;
- id = Origin::CreateStackOrigin(idx).raw_id();
- *id_ptr = id;
- if (print)
- Printf("First time: idx=%d id=%d %s %p \n", idx, id, descr + 4, pc);
- }
- if (print)
- Printf("__msan_set_alloca_origin: descr=%s id=%x\n", descr + 4, id);
- __msan_set_origin(a, size, id);
+ // Intentionally ignore pc and use return address. This function is here for
+ // compatibility, in case program is linked with library instrumented by
+ // older clang.
+ SetAllocaOrigin(a, size, reinterpret_cast<u32 *>(descr), descr + 4,
+ GET_CALLER_PC());
+}
+
+void __msan_set_alloca_origin_with_descr(void *a, uptr size, u32 *id_ptr,
+ char *descr) {
+ SetAllocaOrigin(a, size, id_ptr, descr, GET_CALLER_PC());
+}
+
+void __msan_set_alloca_origin_no_descr(void *a, uptr size, u32 *id_ptr) {
+ SetAllocaOrigin(a, size, id_ptr, nullptr, GET_CALLER_PC());
}
u32 __msan_chain_origin(u32 id) {
#elif SANITIZER_LINUX && defined(__aarch64__)
-// The mapping describes both 39-bits, 42-bits, and 48-bits VMA. AArch64
-// maps:
-// - 0x0000000000000-0x0000010000000: 39/42/48-bits program own segments
-// - 0x0005500000000-0x0005600000000: 39-bits PIE program segments
-// - 0x0007f80000000-0x0007fffffffff: 39-bits libraries segments
-// - 0x002aa00000000-0x002ab00000000: 42-bits PIE program segments
-// - 0x003ff00000000-0x003ffffffffff: 42-bits libraries segments
-// - 0x0aaaaa0000000-0x0aaab00000000: 48-bits PIE program segments
-// - 0xffff000000000-0x1000000000000: 48-bits libraries segments
-// It is fragmented in multiples segments to increase the memory available
-// on 42-bits (12.21% of total VMA available for 42-bits and 13.28 for
-// 39 bits). The 48-bits segments only cover the usual PIE/default segments
-// plus some more segments (262144GB total, 0.39% total VMA).
+// The mapping assumes 48-bit VMA. AArch64 maps:
+// - 0x0000000000000-0x0100000000000: 39/42/48-bits program own segments
+// - 0x0a00000000000-0x0b00000000000: 48-bits PIE program segments
+// Ideally, this would extend to 0x0c00000000000 (2^45 bytes - the
+// maximum ASLR region for 48-bit VMA) but it is too hard to fit in
+// the larger app/shadow/origin regions.
+// - 0x0e00000000000-0x1000000000000: 48-bits libraries segments
const MappingDesc kMemoryLayout[] = {
- {0x00000000000ULL, 0x01000000000ULL, MappingDesc::INVALID, "invalid"},
- {0x01000000000ULL, 0x02000000000ULL, MappingDesc::SHADOW, "shadow-2"},
- {0x02000000000ULL, 0x03000000000ULL, MappingDesc::ORIGIN, "origin-2"},
- {0x03000000000ULL, 0x04000000000ULL, MappingDesc::SHADOW, "shadow-1"},
- {0x04000000000ULL, 0x05000000000ULL, MappingDesc::ORIGIN, "origin-1"},
- {0x05000000000ULL, 0x06000000000ULL, MappingDesc::APP, "app-1"},
- {0x06000000000ULL, 0x07000000000ULL, MappingDesc::INVALID, "invalid"},
- {0x07000000000ULL, 0x08000000000ULL, MappingDesc::APP, "app-2"},
- {0x08000000000ULL, 0x09000000000ULL, MappingDesc::INVALID, "invalid"},
- // The mappings below are used only for 42-bits VMA.
- {0x09000000000ULL, 0x0A000000000ULL, MappingDesc::SHADOW, "shadow-3"},
- {0x0A000000000ULL, 0x0B000000000ULL, MappingDesc::ORIGIN, "origin-3"},
- {0x0B000000000ULL, 0x0F000000000ULL, MappingDesc::INVALID, "invalid"},
- {0x0F000000000ULL, 0x10000000000ULL, MappingDesc::APP, "app-3"},
- {0x10000000000ULL, 0x11000000000ULL, MappingDesc::INVALID, "invalid"},
- {0x11000000000ULL, 0x12000000000ULL, MappingDesc::APP, "app-4"},
- {0x12000000000ULL, 0x17000000000ULL, MappingDesc::INVALID, "invalid"},
- {0x17000000000ULL, 0x18000000000ULL, MappingDesc::SHADOW, "shadow-4"},
- {0x18000000000ULL, 0x19000000000ULL, MappingDesc::ORIGIN, "origin-4"},
- {0x19000000000ULL, 0x20000000000ULL, MappingDesc::INVALID, "invalid"},
- {0x20000000000ULL, 0x21000000000ULL, MappingDesc::APP, "app-5"},
- {0x21000000000ULL, 0x26000000000ULL, MappingDesc::INVALID, "invalid"},
- {0x26000000000ULL, 0x27000000000ULL, MappingDesc::SHADOW, "shadow-5"},
- {0x27000000000ULL, 0x28000000000ULL, MappingDesc::ORIGIN, "origin-5"},
- {0x28000000000ULL, 0x29000000000ULL, MappingDesc::SHADOW, "shadow-7"},
- {0x29000000000ULL, 0x2A000000000ULL, MappingDesc::ORIGIN, "origin-7"},
- {0x2A000000000ULL, 0x2B000000000ULL, MappingDesc::APP, "app-6"},
- {0x2B000000000ULL, 0x2C000000000ULL, MappingDesc::INVALID, "invalid"},
- {0x2C000000000ULL, 0x2D000000000ULL, MappingDesc::SHADOW, "shadow-6"},
- {0x2D000000000ULL, 0x2E000000000ULL, MappingDesc::ORIGIN, "origin-6"},
- {0x2E000000000ULL, 0x2F000000000ULL, MappingDesc::APP, "app-7"},
- {0x2F000000000ULL, 0x39000000000ULL, MappingDesc::INVALID, "invalid"},
- {0x39000000000ULL, 0x3A000000000ULL, MappingDesc::SHADOW, "shadow-9"},
- {0x3A000000000ULL, 0x3B000000000ULL, MappingDesc::ORIGIN, "origin-9"},
- {0x3B000000000ULL, 0x3C000000000ULL, MappingDesc::APP, "app-8"},
- {0x3C000000000ULL, 0x3D000000000ULL, MappingDesc::INVALID, "invalid"},
- {0x3D000000000ULL, 0x3E000000000ULL, MappingDesc::SHADOW, "shadow-8"},
- {0x3E000000000ULL, 0x3F000000000ULL, MappingDesc::ORIGIN, "origin-8"},
- {0x3F000000000ULL, 0x40000000000ULL, MappingDesc::APP, "app-9"},
- // The mappings below are used only for 48-bits VMA.
- // TODO(unknown): 48-bit mapping ony covers the usual PIE, non-PIE
- // segments and some more segments totalizing 262144GB of VMA (which cover
- // only 0.32% of all 48-bit VMA). Memory avaliability can be increase by
- // adding multiple application segments like 39 and 42 mapping.
- {0x0040000000000ULL, 0x0041000000000ULL, MappingDesc::INVALID, "invalid"},
- {0x0041000000000ULL, 0x0042000000000ULL, MappingDesc::APP, "app-10"},
- {0x0042000000000ULL, 0x0047000000000ULL, MappingDesc::INVALID, "invalid"},
- {0x0047000000000ULL, 0x0048000000000ULL, MappingDesc::SHADOW, "shadow-10"},
- {0x0048000000000ULL, 0x0049000000000ULL, MappingDesc::ORIGIN, "origin-10"},
- {0x0049000000000ULL, 0x0050000000000ULL, MappingDesc::INVALID, "invalid"},
- {0x0050000000000ULL, 0x0051000000000ULL, MappingDesc::APP, "app-11"},
- {0x0051000000000ULL, 0x0056000000000ULL, MappingDesc::INVALID, "invalid"},
- {0x0056000000000ULL, 0x0057000000000ULL, MappingDesc::SHADOW, "shadow-11"},
- {0x0057000000000ULL, 0x0058000000000ULL, MappingDesc::ORIGIN, "origin-11"},
- {0x0058000000000ULL, 0x0059000000000ULL, MappingDesc::APP, "app-12"},
- {0x0059000000000ULL, 0x005E000000000ULL, MappingDesc::INVALID, "invalid"},
- {0x005E000000000ULL, 0x005F000000000ULL, MappingDesc::SHADOW, "shadow-12"},
- {0x005F000000000ULL, 0x0060000000000ULL, MappingDesc::ORIGIN, "origin-12"},
- {0x0060000000000ULL, 0x0061000000000ULL, MappingDesc::INVALID, "invalid"},
- {0x0061000000000ULL, 0x0062000000000ULL, MappingDesc::APP, "app-13"},
- {0x0062000000000ULL, 0x0067000000000ULL, MappingDesc::INVALID, "invalid"},
- {0x0067000000000ULL, 0x0068000000000ULL, MappingDesc::SHADOW, "shadow-13"},
- {0x0068000000000ULL, 0x0069000000000ULL, MappingDesc::ORIGIN, "origin-13"},
- {0x0069000000000ULL, 0x0AAAAA0000000ULL, MappingDesc::INVALID, "invalid"},
- {0x0AAAAA0000000ULL, 0x0AAAB00000000ULL, MappingDesc::APP, "app-14"},
- {0x0AAAB00000000ULL, 0x0AACAA0000000ULL, MappingDesc::INVALID, "invalid"},
- {0x0AACAA0000000ULL, 0x0AACB00000000ULL, MappingDesc::SHADOW, "shadow-14"},
- {0x0AACB00000000ULL, 0x0AADAA0000000ULL, MappingDesc::INVALID, "invalid"},
- {0x0AADAA0000000ULL, 0x0AADB00000000ULL, MappingDesc::ORIGIN, "origin-14"},
- {0x0AADB00000000ULL, 0x0FF9F00000000ULL, MappingDesc::INVALID, "invalid"},
- {0x0FF9F00000000ULL, 0x0FFA000000000ULL, MappingDesc::SHADOW, "shadow-15"},
- {0x0FFA000000000ULL, 0x0FFAF00000000ULL, MappingDesc::INVALID, "invalid"},
- {0x0FFAF00000000ULL, 0x0FFB000000000ULL, MappingDesc::ORIGIN, "origin-15"},
- {0x0FFB000000000ULL, 0x0FFFF00000000ULL, MappingDesc::INVALID, "invalid"},
- {0x0FFFF00000000ULL, 0x1000000000000ULL, MappingDesc::APP, "app-15"},
+ {0X0000000000000, 0X0100000000000, MappingDesc::APP, "app-10-13"},
+ {0X0100000000000, 0X0200000000000, MappingDesc::SHADOW, "shadow-14"},
+ {0X0200000000000, 0X0300000000000, MappingDesc::INVALID, "invalid"},
+ {0X0300000000000, 0X0400000000000, MappingDesc::ORIGIN, "origin-14"},
+ {0X0400000000000, 0X0600000000000, MappingDesc::SHADOW, "shadow-15"},
+ {0X0600000000000, 0X0800000000000, MappingDesc::ORIGIN, "origin-15"},
+ {0X0800000000000, 0X0A00000000000, MappingDesc::INVALID, "invalid"},
+ {0X0A00000000000, 0X0B00000000000, MappingDesc::APP, "app-14"},
+ {0X0B00000000000, 0X0C00000000000, MappingDesc::SHADOW, "shadow-10-13"},
+ {0X0C00000000000, 0X0D00000000000, MappingDesc::INVALID, "invalid"},
+ {0X0D00000000000, 0X0E00000000000, MappingDesc::ORIGIN, "origin-10-13"},
+ {0X0E00000000000, 0X1000000000000, MappingDesc::APP, "app-15"},
};
-# define MEM_TO_SHADOW(mem) ((uptr)mem ^ 0x6000000000ULL)
-# define SHADOW_TO_ORIGIN(shadow) (((uptr)(shadow)) + 0x1000000000ULL)
+# define MEM_TO_SHADOW(mem) ((uptr)mem ^ 0xB00000000000ULL)
+# define SHADOW_TO_ORIGIN(shadow) (((uptr)(shadow)) + 0x200000000000ULL)
#elif SANITIZER_LINUX && SANITIZER_PPC64
const MappingDesc kMemoryLayout[] = {
((((uptr)(mem)) & ~0xC00000000000ULL) + 0x080000000000ULL)
#define SHADOW_TO_ORIGIN(shadow) (((uptr)(shadow)) + 0x140000000000ULL)
+#elif SANITIZER_FREEBSD && defined(__aarch64__)
+
+// Low memory: main binary, MAP_32BIT mappings and modules
+// High memory: heap, modules and main thread stack
+const MappingDesc kMemoryLayout[] = {
+ {0x000000000000ULL, 0x020000000000ULL, MappingDesc::APP, "low memory"},
+ {0x020000000000ULL, 0x200000000000ULL, MappingDesc::INVALID, "invalid"},
+ {0x200000000000ULL, 0x620000000000ULL, MappingDesc::SHADOW, "shadow"},
+ {0x620000000000ULL, 0x700000000000ULL, MappingDesc::INVALID, "invalid"},
+ {0x700000000000ULL, 0xb20000000000ULL, MappingDesc::ORIGIN, "origin"},
+ {0xb20000000000ULL, 0xc00000000000ULL, MappingDesc::INVALID, "invalid"},
+ {0xc00000000000ULL, 0x1000000000000ULL, MappingDesc::APP, "high memory"}};
+
+// Maps low and high app ranges to contiguous space with zero base:
+// Low: 0000 0000 0000 - 01ff ffff ffff -> 4000 0000 0000 - 41ff ffff ffff
+// High: c000 0000 0000 - ffff ffff ffff -> 0000 0000 0000 - 3fff ffff ffff
+#define LINEARIZE_MEM(mem) \
+ (((uptr)(mem) & ~0x1800000000000ULL) ^ 0x400000000000ULL)
+#define MEM_TO_SHADOW(mem) (LINEARIZE_MEM((mem)) + 0x200000000000ULL)
+#define SHADOW_TO_ORIGIN(shadow) (((uptr)(shadow)) + 0x500000000000)
+
#elif SANITIZER_FREEBSD && SANITIZER_WORDSIZE == 64
// Low memory: main binary, MAP_32BIT mappings and modules
#elif SANITIZER_NETBSD || (SANITIZER_LINUX && SANITIZER_WORDSIZE == 64)
-#ifdef MSAN_LINUX_X86_64_OLD_MAPPING
-// Requries PIE binary and ASLR enabled.
-// Main thread stack and DSOs at 0x7f0000000000 (sometimes 0x7e0000000000).
-// Heap at 0x600000000000.
-const MappingDesc kMemoryLayout[] = {
- {0x000000000000ULL, 0x200000000000ULL, MappingDesc::INVALID, "invalid"},
- {0x200000000000ULL, 0x400000000000ULL, MappingDesc::SHADOW, "shadow"},
- {0x400000000000ULL, 0x600000000000ULL, MappingDesc::ORIGIN, "origin"},
- {0x600000000000ULL, 0x800000000000ULL, MappingDesc::APP, "app"}};
-
-#define MEM_TO_SHADOW(mem) (((uptr)(mem)) & ~0x400000000000ULL)
-#define SHADOW_TO_ORIGIN(mem) (((uptr)(mem)) + 0x200000000000ULL)
-#else // MSAN_LINUX_X86_64_OLD_MAPPING
// All of the following configurations are supported.
// ASLR disabled: main executable and DSOs at 0x555550000000
// PIE and ASLR: main executable and DSOs at 0x7f0000000000
{0x700000000000ULL, 0x800000000000ULL, MappingDesc::APP, "app-3"}};
#define MEM_TO_SHADOW(mem) (((uptr)(mem)) ^ 0x500000000000ULL)
#define SHADOW_TO_ORIGIN(mem) (((uptr)(mem)) + 0x100000000000ULL)
-#endif // MSAN_LINUX_X86_64_OLD_MAPPING
#else
#error "Unsupported platform"
const char *GetStackOriginDescr(u32 id, uptr *pc);
-void EnterSymbolizer();
-void ExitSymbolizer();
-bool IsInSymbolizer();
-
-struct SymbolizerScope {
- SymbolizerScope() { EnterSymbolizer(); }
- ~SymbolizerScope() { ExitSymbolizer(); }
-};
+bool IsInSymbolizerOrUnwider();
void PrintWarning(uptr pc, uptr bp);
void PrintWarningWithOrigin(uptr pc, uptr bp, u32 origin);
u32 ChainOrigin(u32 id, StackTrace *stack);
const int STACK_TRACE_TAG_POISON = StackTrace::TAG_CUSTOM + 1;
+const int STACK_TRACE_TAG_FIELDS = STACK_TRACE_TAG_POISON + 1;
+const int STACK_TRACE_TAG_VPTR = STACK_TRACE_TAG_FIELDS + 1;
#define GET_MALLOC_STACK_TRACE \
BufferedStackTrace stack; \
} // namespace __msan
-#define MSAN_MALLOC_HOOK(ptr, size) \
- do { \
- if (&__sanitizer_malloc_hook) { \
- UnpoisonParam(2); \
- __sanitizer_malloc_hook(ptr, size); \
- } \
- RunMallocHooks(ptr, size); \
- } while (false)
-#define MSAN_FREE_HOOK(ptr) \
- do { \
- if (&__sanitizer_free_hook) { \
- UnpoisonParam(1); \
- __sanitizer_free_hook(ptr); \
- } \
- RunFreeHooks(ptr); \
- } while (false)
-
#endif // MSAN_H
};
typedef SizeClassAllocator32<AP32> PrimaryAllocator;
#elif defined(__x86_64__)
-#if SANITIZER_NETBSD || \
- (SANITIZER_LINUX && !defined(MSAN_LINUX_X86_64_OLD_MAPPING))
+#if SANITIZER_NETBSD || SANITIZER_LINUX
static const uptr kAllocatorSpace = 0x700000000000ULL;
#else
static const uptr kAllocatorSpace = 0x600000000000ULL;
typedef SizeClassAllocator64<AP64> PrimaryAllocator;
#elif defined(__aarch64__)
-static const uptr kMaxAllowedMallocSize = 2UL << 30; // 2G
+static const uptr kMaxAllowedMallocSize = 8UL << 30;
-struct AP32 {
- static const uptr kSpaceBeg = 0;
- static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE;
+struct AP64 {
+ static const uptr kSpaceBeg = 0xE00000000000ULL;
+ static const uptr kSpaceSize = 0x40000000000; // 4T.
static const uptr kMetadataSize = sizeof(Metadata);
- typedef __sanitizer::CompactSizeClassMap SizeClassMap;
- static const uptr kRegionSizeLog = 20;
- using AddressSpaceView = LocalAddressSpaceView;
+ typedef DefaultSizeClassMap SizeClassMap;
typedef MsanMapUnmapCallback MapUnmapCallback;
static const uptr kFlags = 0;
+ using AddressSpaceView = LocalAddressSpaceView;
};
-typedef SizeClassAllocator32<AP32> PrimaryAllocator;
+typedef SizeClassAllocator64<AP64> PrimaryAllocator;
#endif
typedef CombinedAllocator<PrimaryAllocator> Allocator;
typedef Allocator::AllocatorCache AllocatorCache;
}
ReportAllocationSizeTooBig(size, max_malloc_size, stack);
}
+ if (UNLIKELY(IsRssLimitExceeded())) {
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ ReportRssLimitExceeded(stack);
+ }
MsanThread *t = GetCurrentThread();
void *allocated;
if (t) {
__msan_set_origin(allocated, size, o.raw_id());
}
}
- MSAN_MALLOC_HOOK(allocated, size);
+ UnpoisonParam(2);
+ RunMallocHooks(allocated, size);
return allocated;
}
void MsanDeallocate(StackTrace *stack, void *p) {
CHECK(p);
- MSAN_FREE_HOOK(p);
+ UnpoisonParam(1);
+ RunFreeHooks(p);
+
Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(p));
uptr size = meta->requested_size;
meta->requested_size = 0;
static ChainedOriginDepot chainedOriginDepot;
-StackDepotStats *ChainedOriginDepotGetStats() {
+StackDepotStats ChainedOriginDepotGetStats() {
return chainedOriginDepot.GetStats();
}
namespace __msan {
// Gets the statistic of the origin chain storage.
-StackDepotStats *ChainedOriginDepotGetStats();
+StackDepotStats ChainedOriginDepotGetStats();
// Stores a chain with StackDepot ID here_id and previous chain ID prev_id.
// If successful, returns true and the new chain id new_id.
MSAN_FLAG(bool, poison_stack_with_zeroes, false, "")
MSAN_FLAG(bool, poison_in_malloc, true, "")
MSAN_FLAG(bool, poison_in_free, true, "")
-MSAN_FLAG(bool, poison_in_dtor, false, "")
+MSAN_FLAG(bool, poison_in_dtor, true, "")
MSAN_FLAG(bool, report_umrs, true, "")
MSAN_FLAG(bool, wrap_signals, true, "")
MSAN_FLAG(bool, print_stats, false, "")
#include "msan.h"
#include "msan_chained_origin_depot.h"
#include "msan_origin.h"
+#include "msan_poisoning.h"
#include "msan_report.h"
#include "msan_thread.h"
-#include "msan_poisoning.h"
-#include "sanitizer_common/sanitizer_errno_codes.h"
-#include "sanitizer_common/sanitizer_platform_limits_posix.h"
-#include "sanitizer_common/sanitizer_platform_limits_netbsd.h"
#include "sanitizer_common/sanitizer_allocator.h"
+#include "sanitizer_common/sanitizer_allocator_dlsym.h"
#include "sanitizer_common/sanitizer_allocator_interface.h"
-#include "sanitizer_common/sanitizer_allocator_internal.h"
#include "sanitizer_common/sanitizer_atomic.h"
#include "sanitizer_common/sanitizer_common.h"
#include "sanitizer_common/sanitizer_errno.h"
-#include "sanitizer_common/sanitizer_stackdepot.h"
+#include "sanitizer_common/sanitizer_errno_codes.h"
+#include "sanitizer_common/sanitizer_glibc_version.h"
#include "sanitizer_common/sanitizer_libc.h"
#include "sanitizer_common/sanitizer_linux.h"
+#include "sanitizer_common/sanitizer_platform_limits_netbsd.h"
+#include "sanitizer_common/sanitizer_platform_limits_posix.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
#include "sanitizer_common/sanitizer_tls_get_addr.h"
#include "sanitizer_common/sanitizer_vector.h"
return in_interceptor_scope;
}
-static uptr allocated_for_dlsym;
-static const uptr kDlsymAllocPoolSize = 1024;
-static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize];
-
-static bool IsInDlsymAllocPool(const void *ptr) {
- uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
- return off < sizeof(alloc_memory_for_dlsym);
-}
-
-static void *AllocateFromLocalPool(uptr size_in_bytes) {
- uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize;
- void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym];
- allocated_for_dlsym += size_in_words;
- CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize);
- return mem;
-}
+struct DlsymAlloc : public DlSymAllocator<DlsymAlloc> {
+ static bool UseImpl() { return !msan_inited; }
+};
#define ENSURE_MSAN_INITED() do { \
CHECK(!msan_init_is_running); \
#define CHECK_UNPOISONED_0(x, n) \
do { \
sptr __offset = __msan_test_shadow(x, n); \
- if (__msan::IsInSymbolizer()) break; \
+ if (__msan::IsInSymbolizerOrUnwider()) \
+ break; \
if (__offset >= 0 && __msan::flags()->report_umrs) { \
GET_CALLER_PC_BP_SP; \
(void)sp; \
#endif
INTERCEPTOR(void, free, void *ptr) {
+ if (UNLIKELY(!ptr))
+ return;
+ if (DlsymAlloc::PointerIsMine(ptr))
+ return DlsymAlloc::Free(ptr);
GET_MALLOC_STACK_TRACE;
- if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return;
MsanDeallocate(&stack, ptr);
}
#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
INTERCEPTOR(void, cfree, void *ptr) {
+ if (UNLIKELY(!ptr))
+ return;
+ if (DlsymAlloc::PointerIsMine(ptr))
+ return DlsymAlloc::Free(ptr);
GET_MALLOC_STACK_TRACE;
- if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return;
MsanDeallocate(&stack, ptr);
}
-#define MSAN_MAYBE_INTERCEPT_CFREE INTERCEPT_FUNCTION(cfree)
+# define MSAN_MAYBE_INTERCEPT_CFREE INTERCEPT_FUNCTION(cfree)
#else
#define MSAN_MAYBE_INTERCEPT_CFREE
#endif
INTERCEPTOR(char *, strcpy, char *dest, const char *src) {
ENSURE_MSAN_INITED();
GET_STORE_STACK_TRACE;
- SIZE_T n = REAL(strlen)(src);
+ SIZE_T n = internal_strlen(src);
CHECK_UNPOISONED_STRING(src + n, 0);
char *res = REAL(strcpy)(dest, src);
CopyShadowAndOrigin(dest, src, n + 1, &stack);
INTERCEPTOR(char *, strncpy, char *dest, const char *src, SIZE_T n) {
ENSURE_MSAN_INITED();
GET_STORE_STACK_TRACE;
- SIZE_T copy_size = REAL(strnlen)(src, n);
+ SIZE_T copy_size = internal_strnlen(src, n);
if (copy_size < n)
copy_size++; // trailing \0
char *res = REAL(strncpy)(dest, src, n);
INTERCEPTOR(char *, stpcpy, char *dest, const char *src) {
ENSURE_MSAN_INITED();
GET_STORE_STACK_TRACE;
- SIZE_T n = REAL(strlen)(src);
+ SIZE_T n = internal_strlen(src);
CHECK_UNPOISONED_STRING(src + n, 0);
char *res = REAL(stpcpy)(dest, src);
CopyShadowAndOrigin(dest, src, n + 1, &stack);
return res;
}
-#define MSAN_MAYBE_INTERCEPT_STPCPY INTERCEPT_FUNCTION(stpcpy)
+
+INTERCEPTOR(char *, stpncpy, char *dest, const char *src, SIZE_T n) {
+ ENSURE_MSAN_INITED();
+ GET_STORE_STACK_TRACE;
+ SIZE_T copy_size = Min(n, internal_strnlen(src, n) + 1);
+ char *res = REAL(stpncpy)(dest, src, n);
+ CopyShadowAndOrigin(dest, src, copy_size, &stack);
+ __msan_unpoison(dest + copy_size, n - copy_size);
+ return res;
+}
+# define MSAN_MAYBE_INTERCEPT_STPCPY INTERCEPT_FUNCTION(stpcpy)
+# define MSAN_MAYBE_INTERCEPT_STPNCPY INTERCEPT_FUNCTION(stpncpy)
#else
#define MSAN_MAYBE_INTERCEPT_STPCPY
+# define MSAN_MAYBE_INTERCEPT_STPNCPY
#endif
INTERCEPTOR(char *, strdup, char *src) {
GET_STORE_STACK_TRACE;
// On FreeBSD strdup() leverages strlen().
InterceptorScope interceptor_scope;
- SIZE_T n = REAL(strlen)(src);
+ SIZE_T n = internal_strlen(src);
CHECK_UNPOISONED_STRING(src + n, 0);
char *res = REAL(strdup)(src);
CopyShadowAndOrigin(res, src, n + 1, &stack);
INTERCEPTOR(char *, __strdup, char *src) {
ENSURE_MSAN_INITED();
GET_STORE_STACK_TRACE;
- SIZE_T n = REAL(strlen)(src);
+ SIZE_T n = internal_strlen(src);
CHECK_UNPOISONED_STRING(src + n, 0);
char *res = REAL(__strdup)(src);
CopyShadowAndOrigin(res, src, n + 1, &stack);
INTERCEPTOR(char *, gcvt, double number, SIZE_T ndigit, char *buf) {
ENSURE_MSAN_INITED();
char *res = REAL(gcvt)(number, ndigit, buf);
- SIZE_T n = REAL(strlen)(buf);
+ SIZE_T n = internal_strlen(buf);
__msan_unpoison(buf, n + 1);
return res;
}
INTERCEPTOR(char *, strcat, char *dest, const char *src) {
ENSURE_MSAN_INITED();
GET_STORE_STACK_TRACE;
- SIZE_T src_size = REAL(strlen)(src);
- SIZE_T dest_size = REAL(strlen)(dest);
+ SIZE_T src_size = internal_strlen(src);
+ SIZE_T dest_size = internal_strlen(dest);
CHECK_UNPOISONED_STRING(src + src_size, 0);
CHECK_UNPOISONED_STRING(dest + dest_size, 0);
char *res = REAL(strcat)(dest, src);
INTERCEPTOR(char *, strncat, char *dest, const char *src, SIZE_T n) {
ENSURE_MSAN_INITED();
GET_STORE_STACK_TRACE;
- SIZE_T dest_size = REAL(strlen)(dest);
- SIZE_T copy_size = REAL(strnlen)(src, n);
+ SIZE_T dest_size = internal_strlen(dest);
+ SIZE_T copy_size = internal_strnlen(src, n);
CHECK_UNPOISONED_STRING(dest + dest_size, 0);
char *res = REAL(strncat)(dest, src, n);
CopyShadowAndOrigin(dest + dest_size, src, copy_size, &stack);
char *res = REAL(fcvt)(x, a, b, c);
__msan_unpoison(b, sizeof(*b));
__msan_unpoison(c, sizeof(*c));
- if (res) __msan_unpoison(res, REAL(strlen)(res) + 1);
+ if (res)
+ __msan_unpoison(res, internal_strlen(res) + 1);
return res;
}
#define MSAN_MAYBE_INTERCEPT_FCVT INTERCEPT_FUNCTION(fcvt)
return REAL(getenv)(name);
ENSURE_MSAN_INITED();
char *res = REAL(getenv)(name);
- if (res) __msan_unpoison(res, REAL(strlen)(res) + 1);
+ if (res)
+ __msan_unpoison(res, internal_strlen(res) + 1);
return res;
}
char **envp = environ;
for (; *envp; ++envp) {
__msan_unpoison(envp, sizeof(*envp));
- __msan_unpoison(*envp, REAL(strlen)(*envp) + 1);
+ __msan_unpoison(*envp, internal_strlen(*envp) + 1);
}
// Trailing NULL pointer.
__msan_unpoison(envp, sizeof(*envp));
return res;
}
-#if SANITIZER_FREEBSD || SANITIZER_NETBSD
+#define SANITIZER_STAT_LINUX (SANITIZER_LINUX && __GLIBC_PREREQ(2, 33))
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_STAT_LINUX
INTERCEPTOR(int, fstat, int fd, void *buf) {
ENSURE_MSAN_INITED();
int res = REAL(fstat)(fd, buf);
__msan_unpoison(buf, __sanitizer::struct_stat_sz);
return res;
}
-#define MSAN_MAYBE_INTERCEPT_FSTAT INTERCEPT_FUNCTION(fstat)
+# define MSAN_MAYBE_INTERCEPT_FSTAT MSAN_INTERCEPT_FUNC(fstat)
#else
#define MSAN_MAYBE_INTERCEPT_FSTAT
#endif
-#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
+#if SANITIZER_STAT_LINUX
+INTERCEPTOR(int, fstat64, int fd, void *buf) {
+ ENSURE_MSAN_INITED();
+ int res = REAL(fstat64)(fd, buf);
+ if (!res)
+ __msan_unpoison(buf, __sanitizer::struct_stat64_sz);
+ return res;
+}
+# define MSAN_MAYBE_INTERCEPT_FSTAT64 MSAN_INTERCEPT_FUNC(fstat64)
+#else
+# define MSAN_MAYBE_INTERCEPT_FSTAT64
+#endif
+
+#if SANITIZER_GLIBC
INTERCEPTOR(int, __fxstat, int magic, int fd, void *buf) {
ENSURE_MSAN_INITED();
int res = REAL(__fxstat)(magic, fd, buf);
__msan_unpoison(buf, __sanitizer::struct_stat_sz);
return res;
}
-#define MSAN_MAYBE_INTERCEPT___FXSTAT INTERCEPT_FUNCTION(__fxstat)
+# define MSAN_MAYBE_INTERCEPT___FXSTAT MSAN_INTERCEPT_FUNC(__fxstat)
#else
#define MSAN_MAYBE_INTERCEPT___FXSTAT
#endif
-#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
+#if SANITIZER_GLIBC
INTERCEPTOR(int, __fxstat64, int magic, int fd, void *buf) {
ENSURE_MSAN_INITED();
int res = REAL(__fxstat64)(magic, fd, buf);
__msan_unpoison(buf, __sanitizer::struct_stat64_sz);
return res;
}
-#define MSAN_MAYBE_INTERCEPT___FXSTAT64 INTERCEPT_FUNCTION(__fxstat64)
+# define MSAN_MAYBE_INTERCEPT___FXSTAT64 MSAN_INTERCEPT_FUNC(__fxstat64)
#else
-#define MSAN_MAYBE_INTERCEPT___FXSTAT64
+# define MSAN_MAYBE_INTERCEPT___FXSTAT64
#endif
-#if SANITIZER_FREEBSD || SANITIZER_NETBSD
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_STAT_LINUX
INTERCEPTOR(int, fstatat, int fd, char *pathname, void *buf, int flags) {
ENSURE_MSAN_INITED();
int res = REAL(fstatat)(fd, pathname, buf, flags);
if (!res) __msan_unpoison(buf, __sanitizer::struct_stat_sz);
return res;
}
-# define MSAN_INTERCEPT_FSTATAT INTERCEPT_FUNCTION(fstatat)
+# define MSAN_MAYBE_INTERCEPT_FSTATAT MSAN_INTERCEPT_FUNC(fstatat)
#else
+# define MSAN_MAYBE_INTERCEPT_FSTATAT
+#endif
+
+#if SANITIZER_STAT_LINUX
+INTERCEPTOR(int, fstatat64, int fd, char *pathname, void *buf, int flags) {
+ ENSURE_MSAN_INITED();
+ int res = REAL(fstatat64)(fd, pathname, buf, flags);
+ if (!res)
+ __msan_unpoison(buf, __sanitizer::struct_stat64_sz);
+ return res;
+}
+# define MSAN_MAYBE_INTERCEPT_FSTATAT64 MSAN_INTERCEPT_FUNC(fstatat64)
+#else
+# define MSAN_MAYBE_INTERCEPT_FSTATAT64
+#endif
+
+#if SANITIZER_GLIBC
INTERCEPTOR(int, __fxstatat, int magic, int fd, char *pathname, void *buf,
int flags) {
ENSURE_MSAN_INITED();
if (!res) __msan_unpoison(buf, __sanitizer::struct_stat_sz);
return res;
}
-# define MSAN_INTERCEPT_FSTATAT INTERCEPT_FUNCTION(__fxstatat)
+# define MSAN_MAYBE_INTERCEPT___FXSTATAT MSAN_INTERCEPT_FUNC(__fxstatat)
+#else
+# define MSAN_MAYBE_INTERCEPT___FXSTATAT
#endif
-#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
+#if SANITIZER_GLIBC
INTERCEPTOR(int, __fxstatat64, int magic, int fd, char *pathname, void *buf,
int flags) {
ENSURE_MSAN_INITED();
if (!res) __msan_unpoison(buf, __sanitizer::struct_stat64_sz);
return res;
}
-#define MSAN_MAYBE_INTERCEPT___FXSTATAT64 INTERCEPT_FUNCTION(__fxstatat64)
+# define MSAN_MAYBE_INTERCEPT___FXSTATAT64 MSAN_INTERCEPT_FUNC(__fxstatat64)
#else
-#define MSAN_MAYBE_INTERCEPT___FXSTATAT64
+# define MSAN_MAYBE_INTERCEPT___FXSTATAT64
#endif
INTERCEPTOR(int, pipe, int pipefd[2]) {
ENSURE_MSAN_INITED();
char *res = REAL(fgets_unlocked)(s, size, stream);
if (res)
- __msan_unpoison(s, REAL(strlen)(s) + 1);
+ __msan_unpoison(s, internal_strlen(s) + 1);
return res;
}
#define MSAN_MAYBE_INTERCEPT_FGETS_UNLOCKED INTERCEPT_FUNCTION(fgets_unlocked)
ENSURE_MSAN_INITED();
int res = REAL(gethostname)(name, len);
if (!res || (res == -1 && errno == errno_ENAMETOOLONG)) {
- SIZE_T real_len = REAL(strnlen)(name, len);
+ SIZE_T real_len = internal_strnlen(name, len);
if (real_len < len)
++real_len;
__msan_unpoison(name, real_len);
INTERCEPTOR(void *, calloc, SIZE_T nmemb, SIZE_T size) {
GET_MALLOC_STACK_TRACE;
- if (UNLIKELY(!msan_inited))
- // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
- return AllocateFromLocalPool(nmemb * size);
+ if (DlsymAlloc::Use())
+ return DlsymAlloc::Callocate(nmemb, size);
return msan_calloc(nmemb, size, &stack);
}
INTERCEPTOR(void *, realloc, void *ptr, SIZE_T size) {
+ if (DlsymAlloc::Use() || DlsymAlloc::PointerIsMine(ptr))
+ return DlsymAlloc::Realloc(ptr, size);
GET_MALLOC_STACK_TRACE;
- if (UNLIKELY(IsInDlsymAllocPool(ptr))) {
- uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
- uptr copy_size = Min(size, kDlsymAllocPoolSize - offset);
- void *new_ptr;
- if (UNLIKELY(!msan_inited)) {
- new_ptr = AllocateFromLocalPool(copy_size);
- } else {
- copy_size = size;
- new_ptr = msan_malloc(copy_size, &stack);
- }
- internal_memcpy(new_ptr, ptr, copy_size);
- return new_ptr;
- }
return msan_realloc(ptr, size, &stack);
}
}
INTERCEPTOR(void *, malloc, SIZE_T size) {
+ if (DlsymAlloc::Use())
+ return DlsymAlloc::Allocate(size);
GET_MALLOC_STACK_TRACE;
- if (UNLIKELY(!msan_inited))
- // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym.
- return AllocateFromLocalPool(size);
return msan_malloc(size, &stack);
}
void __msan_allocated_memory(const void *data, uptr size) {
- GET_MALLOC_STACK_TRACE;
if (flags()->poison_in_malloc) {
+ GET_MALLOC_STACK_TRACE;
stack.tag = STACK_TRACE_TAG_POISON;
PoisonMemory(data, size, &stack);
}
}
void __sanitizer_dtor_callback(const void *data, uptr size) {
- GET_MALLOC_STACK_TRACE;
if (flags()->poison_in_dtor) {
+ GET_MALLOC_STACK_TRACE;
stack.tag = STACK_TRACE_TAG_POISON;
PoisonMemory(data, size, &stack);
}
}
+void __sanitizer_dtor_callback_fields(const void *data, uptr size) {
+ if (flags()->poison_in_dtor) {
+ GET_MALLOC_STACK_TRACE;
+ stack.tag = STACK_TRACE_TAG_FIELDS;
+ PoisonMemory(data, size, &stack);
+ }
+}
+
+void __sanitizer_dtor_callback_vptr(const void *data) {
+ if (flags()->poison_in_dtor) {
+ GET_MALLOC_STACK_TRACE;
+ stack.tag = STACK_TRACE_TAG_VPTR;
+ PoisonMemory(data, sizeof(void *), &stack);
+ }
+}
+
template <class Mmap>
static void *mmap_interceptor(Mmap real_mmap, void *addr, SIZE_T length,
int prot, int flags, int fd, OFF64_T offset) {
ScopedThreadLocalStateBackup stlsb;
UnpoisonParam(3);
__msan_unpoison(si, sizeof(__sanitizer_sigaction));
- __msan_unpoison(uc, __sanitizer::ucontext_t_sz);
+ __msan_unpoison(uc, ucontext_t_sz(uc));
typedef void (*sigaction_cb)(int, void *, void *);
sigaction_cb cb =
(sigaction_cb)atomic_load(&sigactions[signo], memory_order_relaxed);
cb(signo, si, uc);
+ CHECK_UNPOISONED(uc, ucontext_t_sz(uc));
}
static void read_sigaction(const __sanitizer_sigaction *act) {
static void *MsanThreadStartFunc(void *arg) {
MsanThread *t = (MsanThread *)arg;
SetCurrentThread(t);
+ t->Init();
+ SetSigProcMask(&t->starting_sigset_, nullptr);
return t->ThreadStart();
}
AdjustStackSize(attr);
MsanThread *t = MsanThread::Create(callback, param);
-
+ ScopedBlockSignals block(&t->starting_sigset_);
int res = REAL(pthread_create)(th, attr, MsanThreadStartFunc, t);
if (attr == &myattr)
return res;
}
+DEFINE_REAL_PTHREAD_FUNCTIONS
+
extern char *tzname[2];
INTERCEPTOR(void, tzset, int fake) {
InterceptorScope interceptor_scope;
REAL(tzset)(fake);
if (tzname[0])
- __msan_unpoison(tzname[0], REAL(strlen)(tzname[0]) + 1);
+ __msan_unpoison(tzname[0], internal_strlen(tzname[0]) + 1);
if (tzname[1])
- __msan_unpoison(tzname[1], REAL(strlen)(tzname[1]) + 1);
+ __msan_unpoison(tzname[1], internal_strlen(tzname[1]) + 1);
return;
}
};
struct InterceptorContext {
- BlockingMutex atexit_mu;
+ Mutex atexit_mu;
Vector<struct MSanAtExitRecord *> AtExitStack;
InterceptorContext()
void MSanAtExitWrapper() {
MSanAtExitRecord *r;
{
- BlockingMutexLock l(&interceptor_ctx()->atexit_mu);
+ Lock l(&interceptor_ctx()->atexit_mu);
uptr element = interceptor_ctx()->AtExitStack.Size() - 1;
r = interceptor_ctx()->AtExitStack[element];
// Unpoison argument shadow for C++ module destructors.
INTERCEPTOR(int, atexit, void (*func)()) {
- // Avoid calling real atexit as it is unrechable on at least on Linux.
+ // Avoid calling real atexit as it is unreachable on at least on Linux.
if (msan_init_is_running)
return REAL(__cxa_atexit)((void (*)(void *a))func, 0, 0);
return setup_at_exit_wrapper((void(*)())func, 0, 0);
// NetBSD does not preserve the 2nd argument if dso is equal to 0
// Store ctx in a local stack-like structure
- BlockingMutexLock l(&interceptor_ctx()->atexit_mu);
+ Lock l(&interceptor_ctx()->atexit_mu);
res = REAL(__cxa_atexit)((void (*)(void *a))MSanAtExitWrapper, 0, 0);
if (!res) {
do { \
if (!INTERCEPT_FUNCTION_VER(name, ver)) \
VReport(1, "MemorySanitizer: failed to intercept '%s@@%s'\n", #name, \
- #ver); \
+ ver); \
} while (0)
#define MSAN_INTERCEPT_FUNC_VER_UNVERSIONED_FALLBACK(name, ver) \
do { \
if (!INTERCEPT_FUNCTION_VER(name, ver) && !INTERCEPT_FUNCTION(name)) \
VReport(1, "MemorySanitizer: failed to intercept '%s@@%s' or '%s'\n", \
- #name, #ver, #name); \
+ #name, ver, #name); \
} while (0)
#define COMMON_INTERCEPT_FUNCTION(name) MSAN_INTERCEPT_FUNC(name)
CHECK_UNPOISONED_CTX(ctx, ptr, size)
#define COMMON_INTERCEPTOR_INITIALIZE_RANGE(ptr, size) \
__msan_unpoison(ptr, size)
-#define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \
- if (msan_init_is_running) return REAL(func)(__VA_ARGS__); \
- ENSURE_MSAN_INITED(); \
- MSanInterceptorContext msan_ctx = {IsInInterceptorScope()}; \
- ctx = (void *)&msan_ctx; \
- (void)ctx; \
- InterceptorScope interceptor_scope; \
- __msan_unpoison(__errno_location(), sizeof(int)); /* NOLINT */
+#define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \
+ if (msan_init_is_running) \
+ return REAL(func)(__VA_ARGS__); \
+ ENSURE_MSAN_INITED(); \
+ MSanInterceptorContext msan_ctx = {IsInInterceptorScope()}; \
+ ctx = (void *)&msan_ctx; \
+ (void)ctx; \
+ InterceptorScope interceptor_scope; \
+ __msan_unpoison(__errno_location(), sizeof(int));
#define COMMON_INTERCEPTOR_DIR_ACQUIRE(ctx, path) \
do { \
} while (false)
#include "sanitizer_common/sanitizer_common_syscalls.inc"
#include "sanitizer_common/sanitizer_syscalls_netbsd.inc"
+INTERCEPTOR(const char *, strsignal, int sig) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, strsignal, sig);
+ const char *res = REAL(strsignal)(sig);
+ if (res)
+ __msan_unpoison(res, internal_strlen(res) + 1);
+ return res;
+}
+
struct dlinfo {
char *dli_fname;
void *dli_fbase;
if (res != 0) {
__msan_unpoison(info, sizeof(*info));
if (info->dli_fname)
- __msan_unpoison(info->dli_fname, REAL(strlen)(info->dli_fname) + 1);
+ __msan_unpoison(info->dli_fname, internal_strlen(info->dli_fname) + 1);
if (info->dli_sname)
- __msan_unpoison(info->dli_sname, REAL(strlen)(info->dli_sname) + 1);
+ __msan_unpoison(info->dli_sname, internal_strlen(info->dli_sname) + 1);
}
return res;
}
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, dlerror, fake);
char *res = REAL(dlerror)(fake);
- if (res) __msan_unpoison(res, REAL(strlen)(res) + 1);
+ if (res)
+ __msan_unpoison(res, internal_strlen(res) + 1);
return res;
}
if (info->dlpi_phdr && info->dlpi_phnum)
__msan_unpoison(info->dlpi_phdr, struct_ElfW_Phdr_sz * info->dlpi_phnum);
if (info->dlpi_name)
- __msan_unpoison(info->dlpi_name, REAL(strlen)(info->dlpi_name) + 1);
+ __msan_unpoison(info->dlpi_name, internal_strlen(info->dlpi_name) + 1);
}
dl_iterate_phdr_data *cbdata = (dl_iterate_phdr_data *)data;
UnpoisonParam(3);
ENSURE_MSAN_INITED();
GET_STORE_STACK_TRACE;
wchar_t *res = REAL(wcscpy)(dest, src);
- CopyShadowAndOrigin(dest, src, sizeof(wchar_t) * (REAL(wcslen)(src) + 1),
+ CopyShadowAndOrigin(dest, src, sizeof(wchar_t) * (internal_wcslen(src) + 1),
&stack);
return res;
}
INTERCEPTOR(wchar_t *, wcsncpy, wchar_t *dest, const wchar_t *src, SIZE_T n) {
ENSURE_MSAN_INITED();
GET_STORE_STACK_TRACE;
- SIZE_T copy_size = REAL(wcsnlen)(src, n);
+ SIZE_T copy_size = internal_wcsnlen(src, n);
if (copy_size < n) copy_size++; // trailing \0
wchar_t *res = REAL(wcsncpy)(dest, src, n);
CopyShadowAndOrigin(dest, src, copy_size * sizeof(wchar_t), &stack);
void *__msan_memcpy(void *dest, const void *src, SIZE_T n) {
if (!msan_inited) return internal_memcpy(dest, src, n);
- if (msan_init_is_running || __msan::IsInSymbolizer())
+ if (msan_init_is_running || __msan::IsInSymbolizerOrUnwider())
return REAL(memcpy)(dest, src, n);
ENSURE_MSAN_INITED();
GET_STORE_STACK_TRACE;
void __msan_unpoison_string(const char* s) {
if (!MEM_IS_APP(s)) return;
- __msan_unpoison(s, REAL(strlen)(s) + 1);
+ __msan_unpoison(s, internal_strlen(s) + 1);
}
namespace __msan {
INTERCEPT_FUNCTION(wmemmove);
INTERCEPT_FUNCTION(strcpy);
MSAN_MAYBE_INTERCEPT_STPCPY;
+ MSAN_MAYBE_INTERCEPT_STPNCPY;
INTERCEPT_FUNCTION(strdup);
MSAN_MAYBE_INTERCEPT___STRDUP;
INTERCEPT_FUNCTION(strncpy);
INTERCEPT_FUNCTION(gettimeofday);
MSAN_MAYBE_INTERCEPT_FCVT;
MSAN_MAYBE_INTERCEPT_FSTAT;
+ MSAN_MAYBE_INTERCEPT_FSTAT64;
MSAN_MAYBE_INTERCEPT___FXSTAT;
- MSAN_INTERCEPT_FSTATAT;
+ MSAN_MAYBE_INTERCEPT_FSTATAT;
+ MSAN_MAYBE_INTERCEPT_FSTATAT64;
+ MSAN_MAYBE_INTERCEPT___FXSTATAT;
MSAN_MAYBE_INTERCEPT___FXSTAT64;
MSAN_MAYBE_INTERCEPT___FXSTATAT64;
INTERCEPT_FUNCTION(pipe);
INTERCEPT_FUNCTION(gethostname);
MSAN_MAYBE_INTERCEPT_EPOLL_WAIT;
MSAN_MAYBE_INTERCEPT_EPOLL_PWAIT;
+ INTERCEPT_FUNCTION(strsignal);
INTERCEPT_FUNCTION(dladdr);
INTERCEPT_FUNCTION(dlerror);
INTERCEPT_FUNCTION(dl_iterate_phdr);
#else
INTERCEPT_FUNCTION(pthread_create);
#endif
+ INTERCEPT_FUNCTION(pthread_join);
INTERCEPT_FUNCTION(pthread_key_create);
#if SANITIZER_NETBSD
void __msan_warning();
// Print a warning and die.
-// Intrumentation inserts calls to this function when building in "fast" mode
+// Instrumentation inserts calls to this function when building in "fast" mode
// (i.e. -mllvm -msan-keep-going)
SANITIZER_INTERFACE_ATTRIBUTE __attribute__((noreturn))
void __msan_warning_noreturn();
SANITIZER_INTERFACE_ATTRIBUTE
void __msan_set_alloca_origin4(void *a, uptr size, char *descr, uptr pc);
SANITIZER_INTERFACE_ATTRIBUTE
+void __msan_set_alloca_origin_with_descr(void *a, uptr size, u32 *id_ptr,
+ char *descr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __msan_set_alloca_origin_no_descr(void *a, uptr size, u32 *id_ptr);
+SANITIZER_INTERFACE_ATTRIBUTE
u32 __msan_chain_origin(u32 id);
SANITIZER_INTERFACE_ATTRIBUTE
u32 __msan_get_origin(const void *a);
// uninitialized.
SANITIZER_INTERFACE_ATTRIBUTE
void __sanitizer_dtor_callback(const void* data, uptr size);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_dtor_callback_fields(const void *data, uptr size);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_dtor_callback_vptr(const void *data);
SANITIZER_INTERFACE_ATTRIBUTE
u16 __sanitizer_unaligned_load16(const uu16 *p);
void ReportMapRange(const char *descr, uptr beg, uptr size) {
if (size > 0) {
uptr end = beg + size - 1;
- VPrintf(1, "%s : %p - %p\n", descr, beg, end);
+ VPrintf(1, "%s : 0x%zx - 0x%zx\n", descr, beg, end);
}
}
if (size > 0) {
uptr end = beg + size - 1;
if (!MemoryRangeIsAvailable(beg, end)) {
- Printf("FATAL: Memory range %p - %p is not available.\n", beg, end);
+ Printf("FATAL: Memory range 0x%zx - 0x%zx is not available.\n", beg, end);
return false;
}
}
}
if ((uptr)addr != beg) {
uptr end = beg + size - 1;
- Printf("FATAL: Cannot protect memory range %p - %p (%s).\n", beg, end,
- name);
+ Printf("FATAL: Cannot protect memory range 0x%zx - 0x%zx (%s).\n", beg,
+ end, name);
return false;
}
}
bool InitShadow(bool init_origins) {
// Let user know mapping parameters first.
- VPrintf(1, "__msan_init %p\n", &__msan_init);
+ VPrintf(1, "__msan_init %p\n", reinterpret_cast<void *>(&__msan_init));
for (unsigned i = 0; i < kMemoryLayoutSize; ++i)
VPrintf(1, "%s: %zx - %zx\n", kMemoryLayout[i].name, kMemoryLayout[i].start,
kMemoryLayout[i].end - 1);
if (!MEM_IS_APP(&__msan_init)) {
Printf("FATAL: Code %p is out of application range. Non-PIE build?\n",
- (uptr)&__msan_init);
+ reinterpret_cast<void *>(&__msan_init));
return false;
}
#include "interception/interception.h"
#include "msan_origin.h"
+#include "msan_thread.h"
#include "sanitizer_common/sanitizer_common.h"
DECLARE_REAL(void *, memset, void *dest, int c, uptr n)
SetShadow(dst, size, (u8)-1);
if (__msan_get_track_origins()) {
+ MsanThread *t = GetCurrentThread();
+ if (t && t->InSignalHandler())
+ return;
Origin o = Origin::CreateHeapOrigin(stack);
SetOrigin(dst, size, o.raw_id());
}
static void DescribeStackOrigin(const char *so, uptr pc) {
Decorator d;
- char *s = internal_strdup(so);
- char *sep = internal_strchr(s, '@');
- CHECK(sep);
- *sep = '\0';
Printf("%s", d.Origin());
- Printf(
- " %sUninitialized value was created by an allocation of '%s%s%s'"
- " in the stack frame of function '%s%s%s'%s\n",
- d.Origin(), d.Name(), s, d.Origin(), d.Name(), sep + 1, d.Origin(),
- d.Default());
- InternalFree(s);
+ if (so) {
+ Printf(
+ " %sUninitialized value was created by an allocation of '%s%s%s'"
+ " in the stack frame%s\n",
+ d.Origin(), d.Name(), so, d.Origin(), d.Default());
+ } else {
+ Printf(" %sUninitialized value was created in the stack frame%s\n",
+ d.Origin(), d.Default());
+ }
- if (pc) {
- // For some reason function address in LLVM IR is 1 less then the address
- // of the first instruction.
- pc = StackTrace::GetNextInstructionPc(pc);
+ if (pc)
StackTrace(&pc, 1).Print();
- }
}
static void DescribeOrigin(u32 id) {
Printf(" %sMemory was marked as uninitialized%s\n", d.Origin(),
d.Default());
break;
+ case STACK_TRACE_TAG_FIELDS:
+ Printf(" %sMember fields were destroyed%s\n", d.Origin(), d.Default());
+ break;
+ case STACK_TRACE_TAG_VPTR:
+ Printf(" %sVirtual table ptr was destroyed%s\n", d.Origin(),
+ d.Default());
+ break;
default:
Printf(" %sUninitialized value was created%s\n", d.Origin(),
d.Default());
ScopedErrorReportLock l;
if (__msan_get_track_origins() > 0) {
- StackDepotStats *stack_depot_stats = StackDepotGetStats();
+ StackDepotStats stack_depot_stats = StackDepotGetStats();
// FIXME: we want this at normal exit, too!
// FIXME: but only with verbosity=1 or something
- Printf("Unique heap origins: %zu\n", stack_depot_stats->n_uniq_ids);
- Printf("Stack depot allocated bytes: %zu\n", stack_depot_stats->allocated);
+ Printf("Unique heap origins: %zu\n", stack_depot_stats.n_uniq_ids);
+ Printf("Stack depot allocated bytes: %zu\n", stack_depot_stats.allocated);
- StackDepotStats *chained_origin_depot_stats = ChainedOriginDepotGetStats();
+ StackDepotStats chained_origin_depot_stats = ChainedOriginDepotGetStats();
Printf("Unique origin histories: %zu\n",
- chained_origin_depot_stats->n_uniq_ids);
+ chained_origin_depot_stats.n_uniq_ids);
Printf("History depot allocated bytes: %zu\n",
- chained_origin_depot_stats->allocated);
+ chained_origin_depot_stats.allocated);
}
}
Decorator d;
Printf("%s", d.Warning());
- Printf("Shadow map of [%p, %p), %zu bytes:\n", start, end, end - start);
+ uptr start_x = reinterpret_cast<uptr>(x);
+ Printf("Shadow map [%p, %p) of [%p, %p), %zu bytes:\n",
+ reinterpret_cast<void *>(start), reinterpret_cast<void *>(end),
+ reinterpret_cast<void *>(start_x),
+ reinterpret_cast<void *>(start_x + end - start), end - start);
Printf("%s", d.Default());
while (s < e) {
// Line start.
if (pos % 16 == 0) {
for (int i = 0; i < 4; ++i) origin_ids[i] = -1;
- Printf("%p:", s);
+ Printf("%p[%p]:", reinterpret_cast<void *>(s),
+ reinterpret_cast<void *>(start_x - start + s));
}
// Group start.
if (pos % 4 == 0) {
}
thread_return_t MsanThread::ThreadStart() {
- Init();
-
if (!start_routine_) {
// start_routine_ == 0 if we're on the main thread or on one of the
// OS X libdispatch worker threads. But nobody is supposed to call
#include "msan_allocator.h"
#include "sanitizer_common/sanitizer_common.h"
-
+#include "sanitizer_common/sanitizer_posix.h"
namespace __msan {
class MsanThread {
MsanThreadLocalMallocStorage &malloc_storage() { return malloc_storage_; }
int destructor_iterations_;
+ __sanitizer_sigset_t starting_sigset_;
private:
// NOTE: There is no MsanThread constructor. It is allocated
set(MSAN_LIBCXX_CFLAGS
-fsanitize=memory
-fsanitize-memory-track-origins
+ -fno-sanitize-memory-param-retval # unittests test mostly this mode.
-Wno-pedantic
-Xclang -fdepfile-entry=${COMPILER_RT_OUTPUT_DIR}/share/msan_ignorelist.txt
)
-Wno-uninitialized
-Werror=sign-compare
-Wno-gnu-zero-variadic-macro-arguments
+ -fno-sanitize-memory-param-retval # unittests test mostly this mode.
)
# Remove -stdlib= which is unused when passing -nostdinc++.
string(REGEX REPLACE "-stdlib=[a-zA-Z+]*" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
-mllvm -msan-keep-going=1
)
set(MSAN_UNITTEST_LINK_FLAGS
+ -nostdlib++
${COMPILER_RT_UNITTEST_LINK_FLAGS}
+ ${COMPILER_RT_UNWINDER_LINK_LIBS}
-fsanitize=memory
# Don't need -stdlib=libc++ because we explicitly list libc++.a in the linker
# inputs.
${obj_list} ${source} ${arch}
KIND ${kind}
COMPILE_DEPS ${MSAN_UNITTEST_HEADERS}
- DEPS gtest msan
+ DEPS llvm_gtest msan
CFLAGS -isystem ${CMAKE_CURRENT_BINARY_DIR}/../libcxx_msan_${arch}/include/c++/v1
${MSAN_UNITTEST_INSTRUMENTED_CFLAGS} ${cflags}
)
EXPECT_POISONED(*p | 0x0000ffff);
EXPECT_POISONED(*p | 0xffff0000);
- EXPECT_POISONED(*GetPoisoned<bool>() & *GetPoisoned<bool>());
+ EXPECT_POISONED((int)*GetPoisoned<bool>() & (int)*GetPoisoned<bool>());
}
template<class T>
char* y = new char[5];
strcpy(x, "abc");
EXPECT_UMR(memccpy(y, x, 'd', 5));
+ break_optimization(y);
delete[] x;
delete[] y;
}
x[0] = 'a';
x[2] = 'b';
EXPECT_UMR(memccpy(y, x, 'b', 5));
+ break_optimization(y);
delete[] x;
delete[] y;
}
EXPECT_NOT_POISONED(y[2]);
}
+TEST(MemorySanitizer, stpncpy) {
+ char *x = new char[3];
+ char *y = new char[5];
+ x[0] = 'a';
+ x[1] = *GetPoisoned<char>(1, 1);
+ x[2] = '\0';
+ char *res = stpncpy(y, x, 4);
+ ASSERT_EQ(res, y + 2);
+ EXPECT_NOT_POISONED(y[0]);
+ EXPECT_POISONED(y[1]);
+ EXPECT_NOT_POISONED(y[2]);
+ EXPECT_NOT_POISONED(y[3]);
+ EXPECT_POISONED(y[4]);
+}
+
TEST(MemorySanitizer, strcat) {
char a[10];
char b[] = "def";
return 0;
}
+static int GetThreadStackMin() {
#ifdef PTHREAD_STACK_MIN
-constexpr int kThreadStackMin = PTHREAD_STACK_MIN;
+ return PTHREAD_STACK_MIN;
#else
-constexpr int kThreadStackMin = 0;
+ return 0;
#endif
+}
TEST(MemorySanitizer, SmallStackThread) {
pthread_attr_t attr;
int res;
res = pthread_attr_init(&attr);
ASSERT_EQ(0, res);
- res = pthread_attr_setstacksize(&attr, std::max(kThreadStackMin, 64 * 1024));
+ res = pthread_attr_setstacksize(&attr,
+ std::max(GetThreadStackMin(), 64 * 1024));
ASSERT_EQ(0, res);
res = pthread_create(&t, &attr, SmallStackThread_threadfn, NULL);
ASSERT_EQ(0, res);
res = pthread_attr_init(&attr);
ASSERT_EQ(0, res);
void *stack;
- const size_t kStackSize = std::max(kThreadStackMin, 32 * 1024);
+ const size_t kStackSize = std::max(GetThreadStackMin(), 32 * 1024);
res = posix_memalign(&stack, 4096, kStackSize);
ASSERT_EQ(0, res);
res = pthread_attr_setstack(&attr, stack, kStackSize);
ASSERT_EQ(-1, n);
}
+TEST(MemorySanitizer, wordexp_empty) {
+ wordexp_t w;
+ int res = wordexp("", &w, 0);
+ ASSERT_EQ(0, res);
+ ASSERT_EQ(0U, w.we_wordc);
+ ASSERT_STREQ(nullptr, w.we_wordv[0]);
+}
+
TEST(MemorySanitizer, wordexp) {
wordexp_t w;
int res = wordexp("a b c", &w, 0);
ASSERT_STREQ("c", w.we_wordv[2]);
}
+TEST(MemorySanitizer, wordexp_initial_offset) {
+ wordexp_t w;
+ w.we_offs = 1;
+ int res = wordexp("a b c", &w, WRDE_DOOFFS);
+ ASSERT_EQ(0, res);
+ ASSERT_EQ(3U, w.we_wordc);
+ ASSERT_EQ(nullptr, w.we_wordv[0]);
+ ASSERT_STREQ("a", w.we_wordv[1]);
+ ASSERT_STREQ("b", w.we_wordv[2]);
+ ASSERT_STREQ("c", w.we_wordv[3]);
+}
+
template<class T>
static bool applySlt(T value, T shadow) {
__msan_partial_poison(&value, &shadow, sizeof(T));
template<class T, class BinaryOp>
ALWAYS_INLINE
void BinaryOpOriginTest(BinaryOp op) {
- U4 ox = rand(); //NOLINT
- U4 oy = rand(); //NOLINT
+ U4 ox = rand();
+ U4 oy = rand();
T *x = GetPoisonedO<T>(0, ox, 0);
T *y = GetPoisonedO<T>(1, oy, 0);
T *z = GetPoisonedO<T>(2, 0, 0);
if (!TrackingOrigins()) return;
EXPECT_POISONED_O(*GetPoisonedO<S4>(0, __LINE__) <= 11, __LINE__);
EXPECT_POISONED_O(*GetPoisonedO<S4>(0, __LINE__) == 11, __LINE__);
- EXPECT_POISONED_O(*GetPoisonedO<float>(0, __LINE__) == 1.1, __LINE__);
+ EXPECT_POISONED_O(*GetPoisonedO<float>(0, __LINE__) == 1.1f, __LINE__);
+ EXPECT_POISONED_O(*GetPoisonedO<double>(0, __LINE__) == 1.1, __LINE__);
}
TEST(MemorySanitizerOrigins, DIV) {
# Build for all components of the ORC runtime support library.
-# ORC runtime library implementation files.
-set(ORC_SOURCES
+# ORC runtime library common implementation files.
+set(ORC_COMMON_SOURCES
+ debug.cpp
extensible_rtti.cpp
log_error_to_stderr.cpp
- macho_platform.cpp
run_program_wrapper.cpp
+ dlfcn_wrapper.cpp
+ )
+
+# ORC runtime library implementation files for all ORC architectures.s
+set(ALL_ORC_SOURCES
+ ${ORC_COMMON_SOURCES}
+ coff_platform.cpp
+ coff_platform.per_jd.cpp
+ elfnix_platform.cpp
+ macho_platform.cpp
)
# Implementation files for all ORC architectures.
-set(x86_64_SOURCES
-# x86-64 specific assembly files will go here.
+set(ALL_ORC_ASM_SOURCES
macho_tlv.x86-64.S
-)
+ macho_tlv.arm64.S
+ elfnix_tls.x86-64.S
+ elfnix_tls.aarch64.S
+ )
-set(ORC_IMPL_HEADERS
-# Implementation headers will go here.
+# Common implementation headers will go here.
+set(ORC_COMMON_IMPL_HEADERS
adt.h
- c_api.h
common.h
compiler.h
endianness.h
error.h
executor_address.h
extensible_rtti.h
- macho_platform.h
simple_packed_serialization.h
stl_extras.h
wrapper_function_utils.h
-)
+ )
+
+# Implementation headers for all ORC architectures.
+set(ALL_ORC_IMPL_HEADERS
+ ${ORC_COMMON_IMPL_HEADERS}
+ macho_platform.h
+ coff_platform.h
+ elfnix_platform.h
+ )
# Create list of all source files for
# consumption by tests.
set(ORC_ALL_SOURCE_FILES
- ${ORC_SOURCES}
- ${x86_64_SOURCES}
- ${ORC_IMPL_HEADERS}
+ ${ALL_ORC_SOURCES}
+ ${ALL_ORC_ASM_SOURCES}
+ ${ALL_ORC_IMPL_HEADERS}
)
list(REMOVE_DUPLICATES ORC_ALL_SOURCE_FILES)
include_directories(..)
include_directories(../../include)
-set(ORC_CFLAGS ${COMPILER_RT_COMMON_CFLAGS})
+set(ORC_CFLAGS
+ ${COMPILER_RT_COMMON_CFLAGS}
+ ${COMPILER_RT_CXX_CFLAGS})
+set(ORC_LINK_FLAGS ${COMPILER_RT_COMMON_LINK_FLAGS})
+set(ORC_LINK_LIBS
+ ${COMPILER_RT_UNWINDER_LINK_LIBS}
+ ${COMPILER_RT_CXX_LINK_LIBS})
# Allow the ORC runtime to reference LLVM headers.
foreach (DIR ${LLVM_INCLUDE_DIR} ${LLVM_MAIN_INCLUDE_DIR})
endif()
if (APPLE)
+ add_asm_sources(ORC_ASM_SOURCES
+ macho_tlv.x86-64.S
+ macho_tlv.arm64.S
+ )
+
+ set(ORC_IMPL_HEADERS
+ ${ORC_COMMON_IMPL_HEADERS}
+ macho_platform.h
+ )
+
+ set(ORC_SOURCES
+ ${ORC_COMMON_SOURCES}
+ macho_platform.cpp
+ )
+
add_compiler_rt_object_libraries(RTOrc
OS ${ORC_SUPPORTED_OS}
ARCHS ${ORC_SUPPORTED_ARCH}
- SOURCES ${ORC_SOURCES} ${x86_64_SOURCES}
+ SOURCES ${ORC_SOURCES} ${ORC_ASM_SOURCES}
ADDITIONAL_HEADERS ${ORC_IMPL_HEADERS}
CFLAGS ${ORC_CFLAGS}
DEPS ${ORC_DEPS})
- # We only support running on osx for now.
- add_compiler_rt_runtime(clang_rt.orc
+ add_compiler_rt_runtime(orc_rt
STATIC
OS ${ORC_SUPPORTED_OS}
ARCHS ${ORC_SUPPORTED_ARCH}
OBJECT_LIBS RTOrc
CFLAGS ${ORC_CFLAGS}
- LINK_FLAGS ${SANITIZER_COMMON_LINK_FLAGS} ${WEAK_SYMBOL_LINK_FLAGS}
+ LINK_FLAGS ${ORC_LINK_FLAGS} ${WEAK_SYMBOL_LINK_FLAGS}
LINK_LIBS ${ORC_LINK_LIBS}
PARENT_TARGET orc)
else() # not Apple
+ if (WIN32)
+ set(ORC_BUILD_TYPE STATIC)
+
+ set(ORC_IMPL_HEADERS
+ ${ORC_COMMON_IMPL_HEADERS}
+ coff_platform.h
+ )
+
+ set(ORC_SOURCES
+ ${ORC_COMMON_SOURCES}
+ coff_platform.cpp
+ coff_platform.per_jd.cpp
+ )
+
+ if (MSVC)
+ set(ORC_CFLAGS "${ORC_CFLAGS} /MD")
+ endif()
+ else()
+ set(ORC_BUILD_TYPE STATIC)
+
+ set(ORC_IMPL_HEADERS
+ ${ORC_COMMON_IMPL_HEADERS}
+ elfnix_platform.h
+ )
+
+ set(ORC_SOURCES
+ ${ORC_COMMON_SOURCES}
+ elfnix_platform.cpp
+ )
+
+ add_asm_sources(ORC_ASM_SOURCES
+ elfnix_tls.x86-64.S
+ elfnix_tls.aarch64.S
+ )
+ endif()
+
foreach(arch ${ORC_SUPPORTED_ARCH})
if(NOT CAN_TARGET_${arch})
continue()
endif()
+
add_compiler_rt_object_libraries(RTOrc
ARCHS ${arch}
- SOURCES ${ORC_SOURCES} ${${arch}_SOURCES}
+ SOURCES ${ORC_SOURCES} ${ORC_ASM_SOURCES}
ADDITIONAL_HEADERS ${ORC_IMPL_HEADERS}
CFLAGS ${ORC_CFLAGS}
DEPS ${ORC_DEPS})
# Common ORC archive for instrumented binaries.
- add_compiler_rt_runtime(clang_rt.orc
- STATIC
- ARCHS ${arch}
- CFLAGS ${ORC_CFLAGS}
- OBJECT_LIBS ${ORC_COMMON_RUNTIME_OBJECT_LIBS} RTOrc
- PARENT_TARGET orc)
+ add_compiler_rt_runtime(orc_rt
+ ${ORC_BUILD_TYPE}
+ ARCHS ${arch}
+ CFLAGS ${ORC_CFLAGS}
+ LINK_FLAGS ${ORC_LINK_FLAGS}
+ LINK_LIBS ${ORC_LINK_LIBS}
+ OBJECT_LIBS ${ORC_COMMON_RUNTIME_OBJECT_LIBS} RTOrc
+ PARENT_TARGET orc)
endforeach()
endif() # not Apple
-if(COMPILER_RT_INCLUDE_TESTS)
- add_subdirectory(unittests)
-endif()
+add_subdirectory(tests)
#include <cstring>
#include <limits>
+#include <ostream>
#include <string>
namespace __orc_rt {
size_type Size = 0;
};
-/// A substitue for std::string_view (and llvm::StringRef).
-/// FIXME: Remove in favor of std::string_view once we have c++17.
-class string_view {
-public:
- typedef char value_type;
- typedef char *pointer;
- typedef const char *const_pointer;
- typedef char &reference;
- typedef const char &const_reference;
- typedef std::size_t size_type;
- typedef std::ptrdiff_t difference_type;
-
- typedef const_pointer const_iterator;
- typedef const_iterator iterator;
-
- constexpr string_view() noexcept = default;
- constexpr string_view(const char *S, size_type Count)
- : Data(S), Size(Count) {}
- string_view(const char *S) : Data(S), Size(strlen(S)) {}
-
- constexpr const_iterator begin() const noexcept { return Data; }
- constexpr const_iterator end() const noexcept { return Data + Size; }
- constexpr const_pointer data() const noexcept { return Data; }
- constexpr const_reference operator[](size_type idx) { return Data[idx]; }
- constexpr size_type size() const noexcept { return Size; }
- constexpr bool empty() const noexcept { return Size == 0; }
-
- friend bool operator==(const string_view &LHS, const string_view &RHS) {
- if (LHS.Size != RHS.Size)
- return false;
- if (LHS.Data == RHS.Data)
- return true;
- for (size_t I = 0; I != LHS.Size; ++I)
- if (LHS.Data[I] != RHS.Data[I])
- return false;
- return true;
- }
-
- friend bool operator!=(const string_view &LHS, const string_view &RHS) {
- return !(LHS == RHS);
- }
-
-private:
- const char *Data = nullptr;
- size_type Size = 0;
-};
-
-inline std::string to_string(string_view SV) {
- return std::string(SV.data(), SV.size());
-}
-
} // end namespace __orc_rt
-#endif // ORC_RT_COMMON_H
+#endif // ORC_RT_ADT_H
--- /dev/null
+//===- coff_platform.cpp --------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains code required to load the rest of the COFF runtime.
+//
+//===----------------------------------------------------------------------===//
+
+#define NOMINMAX
+#include <windows.h>
+
+#include "coff_platform.h"
+
+#include "debug.h"
+#include "error.h"
+#include "wrapper_function_utils.h"
+
+#include <array>
+#include <list>
+#include <map>
+#include <mutex>
+#include <sstream>
+#include <string_view>
+#include <vector>
+
+#define DEBUG_TYPE "coff_platform"
+
+using namespace __orc_rt;
+
+namespace __orc_rt {
+
+using COFFJITDylibDepInfo = std::vector<ExecutorAddr>;
+using COFFJITDylibDepInfoMap =
+ std::unordered_map<ExecutorAddr, COFFJITDylibDepInfo>;
+
+using SPSCOFFObjectSectionsMap =
+ SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRange>>;
+
+using SPSCOFFJITDylibDepInfo = SPSSequence<SPSExecutorAddr>;
+
+using SPSCOFFJITDylibDepInfoMap =
+ SPSSequence<SPSTuple<SPSExecutorAddr, SPSCOFFJITDylibDepInfo>>;
+
+} // namespace __orc_rt
+
+ORC_RT_JIT_DISPATCH_TAG(__orc_rt_coff_symbol_lookup_tag)
+ORC_RT_JIT_DISPATCH_TAG(__orc_rt_coff_push_initializers_tag)
+
+namespace {
+class COFFPlatformRuntimeState {
+private:
+ // Ctor/dtor section.
+ // Manage lists of *tor functions sorted by the last character of subsection
+ // name.
+ struct XtorSection {
+ void Register(char SubsectionChar, span<void (*)(void)> Xtors) {
+ Subsections[SubsectionChar - 'A'].push_back(Xtors);
+ SubsectionsNew[SubsectionChar - 'A'].push_back(Xtors);
+ }
+
+ void RegisterNoRun(char SubsectionChar, span<void (*)(void)> Xtors) {
+ Subsections[SubsectionChar - 'A'].push_back(Xtors);
+ }
+
+ void Reset() { SubsectionsNew = Subsections; }
+
+ void RunAllNewAndFlush();
+
+ private:
+ std::array<std::vector<span<void (*)(void)>>, 26> Subsections;
+ std::array<std::vector<span<void (*)(void)>>, 26> SubsectionsNew;
+ };
+
+ struct JITDylibState {
+ std::string Name;
+ void *Header = nullptr;
+ size_t LinkedAgainstRefCount = 0;
+ size_t DlRefCount = 0;
+ std::vector<JITDylibState *> Deps;
+ std::vector<void (*)(void)> AtExits;
+ XtorSection CInitSection; // XIA~XIZ
+ XtorSection CXXInitSection; // XCA~XCZ
+ XtorSection CPreTermSection; // XPA~XPZ
+ XtorSection CTermSection; // XTA~XTZ
+
+ bool referenced() const {
+ return LinkedAgainstRefCount != 0 || DlRefCount != 0;
+ }
+ };
+
+public:
+ static void initialize();
+ static COFFPlatformRuntimeState &get();
+ static bool isInitialized() { return CPS; }
+ static void destroy();
+
+ COFFPlatformRuntimeState() = default;
+
+ // Delete copy and move constructors.
+ COFFPlatformRuntimeState(const COFFPlatformRuntimeState &) = delete;
+ COFFPlatformRuntimeState &
+ operator=(const COFFPlatformRuntimeState &) = delete;
+ COFFPlatformRuntimeState(COFFPlatformRuntimeState &&) = delete;
+ COFFPlatformRuntimeState &operator=(COFFPlatformRuntimeState &&) = delete;
+
+ const char *dlerror();
+ void *dlopen(std::string_view Name, int Mode);
+ int dlclose(void *Header);
+ void *dlsym(void *Header, std::string_view Symbol);
+
+ Error registerJITDylib(std::string Name, void *Header);
+ Error deregisterJITDylib(void *Header);
+
+ Error registerAtExit(ExecutorAddr HeaderAddr, void (*AtExit)(void));
+
+ Error registerObjectSections(
+ ExecutorAddr HeaderAddr,
+ std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs,
+ bool RunInitializers);
+ Error deregisterObjectSections(
+ ExecutorAddr HeaderAddr,
+ std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs);
+
+ void *findJITDylibBaseByPC(uint64_t PC);
+
+private:
+ Error registerBlockRange(ExecutorAddr HeaderAddr, ExecutorAddrRange Range);
+ Error deregisterBlockRange(ExecutorAddr HeaderAddr, ExecutorAddrRange Range);
+
+ Error registerSEHFrames(ExecutorAddr HeaderAddr,
+ ExecutorAddrRange SEHFrameRange);
+ Error deregisterSEHFrames(ExecutorAddr HeaderAddr,
+ ExecutorAddrRange SEHFrameRange);
+
+ Expected<void *> dlopenImpl(std::string_view Path, int Mode);
+ Error dlopenFull(JITDylibState &JDS);
+ Error dlopenInitialize(JITDylibState &JDS, COFFJITDylibDepInfoMap &DepInfo);
+
+ Error dlcloseImpl(void *DSOHandle);
+ Error dlcloseDeinitialize(JITDylibState &JDS);
+
+ JITDylibState *getJITDylibStateByHeader(void *DSOHandle);
+ JITDylibState *getJITDylibStateByName(std::string_view Path);
+ Expected<ExecutorAddr> lookupSymbolInJITDylib(void *DSOHandle,
+ std::string_view Symbol);
+
+ static COFFPlatformRuntimeState *CPS;
+
+ std::recursive_mutex JDStatesMutex;
+ std::map<void *, JITDylibState> JDStates;
+ struct BlockRange {
+ void *Header;
+ size_t Size;
+ };
+ std::map<void *, BlockRange> BlockRanges;
+ std::unordered_map<std::string_view, void *> JDNameToHeader;
+ std::string DLFcnError;
+};
+
+} // namespace
+
+COFFPlatformRuntimeState *COFFPlatformRuntimeState::CPS = nullptr;
+
+COFFPlatformRuntimeState::JITDylibState *
+COFFPlatformRuntimeState::getJITDylibStateByHeader(void *Header) {
+ auto I = JDStates.find(Header);
+ if (I == JDStates.end())
+ return nullptr;
+ return &I->second;
+}
+
+COFFPlatformRuntimeState::JITDylibState *
+COFFPlatformRuntimeState::getJITDylibStateByName(std::string_view Name) {
+ // FIXME: Avoid creating string copy here.
+ auto I = JDNameToHeader.find(std::string(Name.data(), Name.size()));
+ if (I == JDNameToHeader.end())
+ return nullptr;
+ void *H = I->second;
+ auto J = JDStates.find(H);
+ assert(J != JDStates.end() &&
+ "JITDylib has name map entry but no header map entry");
+ return &J->second;
+}
+
+Error COFFPlatformRuntimeState::registerJITDylib(std::string Name,
+ void *Header) {
+ ORC_RT_DEBUG({
+ printdbg("Registering JITDylib %s: Header = %p\n", Name.c_str(), Header);
+ });
+ std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
+ if (JDStates.count(Header)) {
+ std::ostringstream ErrStream;
+ ErrStream << "Duplicate JITDylib registration for header " << Header
+ << " (name = " << Name << ")";
+ return make_error<StringError>(ErrStream.str());
+ }
+ if (JDNameToHeader.count(Name)) {
+ std::ostringstream ErrStream;
+ ErrStream << "Duplicate JITDylib registration for header " << Header
+ << " (header = " << Header << ")";
+ return make_error<StringError>(ErrStream.str());
+ }
+
+ auto &JDS = JDStates[Header];
+ JDS.Name = std::move(Name);
+ JDS.Header = Header;
+ JDNameToHeader[JDS.Name] = Header;
+ return Error::success();
+}
+
+Error COFFPlatformRuntimeState::deregisterJITDylib(void *Header) {
+ std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
+ auto I = JDStates.find(Header);
+ if (I == JDStates.end()) {
+ std::ostringstream ErrStream;
+ ErrStream << "Attempted to deregister unrecognized header " << Header;
+ return make_error<StringError>(ErrStream.str());
+ }
+
+ // Remove std::string construction once we can use C++20.
+ auto J = JDNameToHeader.find(
+ std::string(I->second.Name.data(), I->second.Name.size()));
+ assert(J != JDNameToHeader.end() &&
+ "Missing JDNameToHeader entry for JITDylib");
+
+ ORC_RT_DEBUG({
+ printdbg("Deregistering JITDylib %s: Header = %p\n", I->second.Name.c_str(),
+ Header);
+ });
+
+ JDNameToHeader.erase(J);
+ JDStates.erase(I);
+ return Error::success();
+}
+
+void COFFPlatformRuntimeState::XtorSection::RunAllNewAndFlush() {
+ for (auto &Subsection : SubsectionsNew) {
+ for (auto &XtorGroup : Subsection)
+ for (auto &Xtor : XtorGroup)
+ if (Xtor)
+ Xtor();
+ Subsection.clear();
+ }
+}
+
+const char *COFFPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); }
+
+void *COFFPlatformRuntimeState::dlopen(std::string_view Path, int Mode) {
+ ORC_RT_DEBUG({
+ std::string S(Path.data(), Path.size());
+ printdbg("COFFPlatform::dlopen(\"%s\")\n", S.c_str());
+ });
+ std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
+ if (auto H = dlopenImpl(Path, Mode))
+ return *H;
+ else {
+ // FIXME: Make dlerror thread safe.
+ DLFcnError = toString(H.takeError());
+ return nullptr;
+ }
+}
+
+int COFFPlatformRuntimeState::dlclose(void *DSOHandle) {
+ ORC_RT_DEBUG({
+ auto *JDS = getJITDylibStateByHeader(DSOHandle);
+ std::string DylibName;
+ if (JDS) {
+ std::string S;
+ printdbg("COFFPlatform::dlclose(%p) (%s)\n", DSOHandle, S.c_str());
+ } else
+ printdbg("COFFPlatform::dlclose(%p) (%s)\n", DSOHandle, "invalid handle");
+ });
+ std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
+ if (auto Err = dlcloseImpl(DSOHandle)) {
+ // FIXME: Make dlerror thread safe.
+ DLFcnError = toString(std::move(Err));
+ return -1;
+ }
+ return 0;
+}
+
+void *COFFPlatformRuntimeState::dlsym(void *Header, std::string_view Symbol) {
+ auto Addr = lookupSymbolInJITDylib(Header, Symbol);
+ if (!Addr) {
+ return 0;
+ }
+
+ return Addr->toPtr<void *>();
+}
+
+Expected<void *> COFFPlatformRuntimeState::dlopenImpl(std::string_view Path,
+ int Mode) {
+ // Try to find JITDylib state by name.
+ auto *JDS = getJITDylibStateByName(Path);
+
+ if (!JDS)
+ return make_error<StringError>("No registered JTIDylib for path " +
+ std::string(Path.data(), Path.size()));
+
+ if (auto Err = dlopenFull(*JDS))
+ return std::move(Err);
+
+ // Bump the ref-count on this dylib.
+ ++JDS->DlRefCount;
+
+ // Return the header address.
+ return JDS->Header;
+}
+
+Error COFFPlatformRuntimeState::dlopenFull(JITDylibState &JDS) {
+ // Call back to the JIT to push the initializers.
+ Expected<COFFJITDylibDepInfoMap> DepInfoMap((COFFJITDylibDepInfoMap()));
+ if (auto Err = WrapperFunction<SPSExpected<SPSCOFFJITDylibDepInfoMap>(
+ SPSExecutorAddr)>::call(&__orc_rt_coff_push_initializers_tag,
+ DepInfoMap,
+ ExecutorAddr::fromPtr(JDS.Header)))
+ return Err;
+ if (!DepInfoMap)
+ return DepInfoMap.takeError();
+
+ if (auto Err = dlopenInitialize(JDS, *DepInfoMap))
+ return Err;
+
+ if (!DepInfoMap->empty()) {
+ ORC_RT_DEBUG({
+ printdbg("Unrecognized dep-info key headers in dlopen of %s\n",
+ JDS.Name.c_str());
+ });
+ std::ostringstream ErrStream;
+ ErrStream << "Encountered unrecognized dep-info key headers "
+ "while processing dlopen of "
+ << JDS.Name;
+ return make_error<StringError>(ErrStream.str());
+ }
+
+ return Error::success();
+}
+
+Error COFFPlatformRuntimeState::dlopenInitialize(
+ JITDylibState &JDS, COFFJITDylibDepInfoMap &DepInfo) {
+ ORC_RT_DEBUG({
+ printdbg("COFFPlatformRuntimeState::dlopenInitialize(\"%s\")\n",
+ JDS.Name.c_str());
+ });
+
+ // Skip visited dependency.
+ auto I = DepInfo.find(ExecutorAddr::fromPtr(JDS.Header));
+ if (I == DepInfo.end())
+ return Error::success();
+
+ auto DI = std::move(I->second);
+ DepInfo.erase(I);
+
+ // Run initializers of dependencies in proper order by depth-first traversal
+ // of dependency graph.
+ std::vector<JITDylibState *> OldDeps;
+ std::swap(JDS.Deps, OldDeps);
+ JDS.Deps.reserve(DI.size());
+ for (auto DepHeaderAddr : DI) {
+ auto *DepJDS = getJITDylibStateByHeader(DepHeaderAddr.toPtr<void *>());
+ if (!DepJDS) {
+ std::ostringstream ErrStream;
+ ErrStream << "Encountered unrecognized dep header "
+ << DepHeaderAddr.toPtr<void *>() << " while initializing "
+ << JDS.Name;
+ return make_error<StringError>(ErrStream.str());
+ }
+ ++DepJDS->LinkedAgainstRefCount;
+ if (auto Err = dlopenInitialize(*DepJDS, DepInfo))
+ return Err;
+ }
+
+ // Run static initializers.
+ JDS.CInitSection.RunAllNewAndFlush();
+ JDS.CXXInitSection.RunAllNewAndFlush();
+
+ // Decrement old deps.
+ for (auto *DepJDS : OldDeps) {
+ --DepJDS->LinkedAgainstRefCount;
+ if (!DepJDS->referenced())
+ if (auto Err = dlcloseDeinitialize(*DepJDS))
+ return Err;
+ }
+
+ return Error::success();
+}
+
+Error COFFPlatformRuntimeState::dlcloseImpl(void *DSOHandle) {
+ // Try to find JITDylib state by header.
+ auto *JDS = getJITDylibStateByHeader(DSOHandle);
+
+ if (!JDS) {
+ std::ostringstream ErrStream;
+ ErrStream << "No registered JITDylib for " << DSOHandle;
+ return make_error<StringError>(ErrStream.str());
+ }
+
+ // Bump the ref-count.
+ --JDS->DlRefCount;
+
+ if (!JDS->referenced())
+ return dlcloseDeinitialize(*JDS);
+
+ return Error::success();
+}
+
+Error COFFPlatformRuntimeState::dlcloseDeinitialize(JITDylibState &JDS) {
+ ORC_RT_DEBUG({
+ printdbg("COFFPlatformRuntimeState::dlcloseDeinitialize(\"%s\")\n",
+ JDS.Name.c_str());
+ });
+
+ // Run atexits
+ for (auto AtExit : JDS.AtExits)
+ AtExit();
+ JDS.AtExits.clear();
+
+ // Run static terminators.
+ JDS.CPreTermSection.RunAllNewAndFlush();
+ JDS.CTermSection.RunAllNewAndFlush();
+
+ // Queue all xtors as new again.
+ JDS.CInitSection.Reset();
+ JDS.CXXInitSection.Reset();
+ JDS.CPreTermSection.Reset();
+ JDS.CTermSection.Reset();
+
+ // Deinitialize any dependencies.
+ for (auto *DepJDS : JDS.Deps) {
+ --DepJDS->LinkedAgainstRefCount;
+ if (!DepJDS->referenced())
+ if (auto Err = dlcloseDeinitialize(*DepJDS))
+ return Err;
+ }
+
+ return Error::success();
+}
+
+Expected<ExecutorAddr>
+COFFPlatformRuntimeState::lookupSymbolInJITDylib(void *header,
+ std::string_view Sym) {
+ Expected<ExecutorAddr> Result((ExecutorAddr()));
+ if (auto Err = WrapperFunction<SPSExpected<SPSExecutorAddr>(
+ SPSExecutorAddr, SPSString)>::call(&__orc_rt_coff_symbol_lookup_tag,
+ Result,
+ ExecutorAddr::fromPtr(header),
+ Sym))
+ return std::move(Err);
+ return Result;
+}
+
+Error COFFPlatformRuntimeState::registerObjectSections(
+ ExecutorAddr HeaderAddr,
+ std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs,
+ bool RunInitializers) {
+ std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
+ auto I = JDStates.find(HeaderAddr.toPtr<void *>());
+ if (I == JDStates.end()) {
+ std::ostringstream ErrStream;
+ ErrStream << "Unrecognized header " << HeaderAddr.getValue();
+ return make_error<StringError>(ErrStream.str());
+ }
+ auto &JDState = I->second;
+ for (auto &KV : Secs) {
+ if (auto Err = registerBlockRange(HeaderAddr, KV.second))
+ return Err;
+ if (KV.first.empty())
+ continue;
+ char LastChar = KV.first.data()[KV.first.size() - 1];
+ if (KV.first == ".pdata") {
+ if (auto Err = registerSEHFrames(HeaderAddr, KV.second))
+ return Err;
+ } else if (KV.first >= ".CRT$XIA" && KV.first <= ".CRT$XIZ") {
+ if (RunInitializers)
+ JDState.CInitSection.Register(LastChar,
+ KV.second.toSpan<void (*)(void)>());
+ else
+ JDState.CInitSection.RegisterNoRun(LastChar,
+ KV.second.toSpan<void (*)(void)>());
+ } else if (KV.first >= ".CRT$XCA" && KV.first <= ".CRT$XCZ") {
+ if (RunInitializers)
+ JDState.CXXInitSection.Register(LastChar,
+ KV.second.toSpan<void (*)(void)>());
+ else
+ JDState.CXXInitSection.RegisterNoRun(
+ LastChar, KV.second.toSpan<void (*)(void)>());
+ } else if (KV.first >= ".CRT$XPA" && KV.first <= ".CRT$XPZ")
+ JDState.CPreTermSection.Register(LastChar,
+ KV.second.toSpan<void (*)(void)>());
+ else if (KV.first >= ".CRT$XTA" && KV.first <= ".CRT$XTZ")
+ JDState.CTermSection.Register(LastChar,
+ KV.second.toSpan<void (*)(void)>());
+ }
+ return Error::success();
+}
+
+Error COFFPlatformRuntimeState::deregisterObjectSections(
+ ExecutorAddr HeaderAddr,
+ std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs) {
+ std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
+ auto I = JDStates.find(HeaderAddr.toPtr<void *>());
+ if (I == JDStates.end()) {
+ std::ostringstream ErrStream;
+ ErrStream << "Attempted to deregister unrecognized header "
+ << HeaderAddr.getValue();
+ return make_error<StringError>(ErrStream.str());
+ }
+ auto &JDState = I->second;
+ for (auto &KV : Secs) {
+ if (auto Err = deregisterBlockRange(HeaderAddr, KV.second))
+ return Err;
+ if (KV.first == ".pdata")
+ if (auto Err = deregisterSEHFrames(HeaderAddr, KV.second))
+ return Err;
+ }
+ return Error::success();
+}
+
+Error COFFPlatformRuntimeState::registerSEHFrames(
+ ExecutorAddr HeaderAddr, ExecutorAddrRange SEHFrameRange) {
+ int N = (SEHFrameRange.End.getValue() - SEHFrameRange.Start.getValue()) /
+ sizeof(RUNTIME_FUNCTION);
+ auto Func = SEHFrameRange.Start.toPtr<PRUNTIME_FUNCTION>();
+ if (!RtlAddFunctionTable(Func, N,
+ static_cast<DWORD64>(HeaderAddr.getValue())))
+ return make_error<StringError>("Failed to register SEH frames");
+ return Error::success();
+}
+
+Error COFFPlatformRuntimeState::deregisterSEHFrames(
+ ExecutorAddr HeaderAddr, ExecutorAddrRange SEHFrameRange) {
+ if (!RtlDeleteFunctionTable(SEHFrameRange.Start.toPtr<PRUNTIME_FUNCTION>()))
+ return make_error<StringError>("Failed to deregister SEH frames");
+ return Error::success();
+}
+
+Error COFFPlatformRuntimeState::registerBlockRange(ExecutorAddr HeaderAddr,
+ ExecutorAddrRange Range) {
+ assert(!BlockRanges.count(Range.Start.toPtr<void *>()) &&
+ "Block range address already registered");
+ BlockRange B = {HeaderAddr.toPtr<void *>(), Range.size()};
+ BlockRanges.emplace(Range.Start.toPtr<void *>(), B);
+ return Error::success();
+}
+
+Error COFFPlatformRuntimeState::deregisterBlockRange(ExecutorAddr HeaderAddr,
+ ExecutorAddrRange Range) {
+ assert(BlockRanges.count(Range.Start.toPtr<void *>()) &&
+ "Block range address not registered");
+ BlockRanges.erase(Range.Start.toPtr<void *>());
+ return Error::success();
+}
+
+Error COFFPlatformRuntimeState::registerAtExit(ExecutorAddr HeaderAddr,
+ void (*AtExit)(void)) {
+ std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
+ auto I = JDStates.find(HeaderAddr.toPtr<void *>());
+ if (I == JDStates.end()) {
+ std::ostringstream ErrStream;
+ ErrStream << "Unrecognized header " << HeaderAddr.getValue();
+ return make_error<StringError>(ErrStream.str());
+ }
+ I->second.AtExits.push_back(AtExit);
+ return Error::success();
+}
+
+void COFFPlatformRuntimeState::initialize() {
+ assert(!CPS && "COFFPlatformRuntimeState should be null");
+ CPS = new COFFPlatformRuntimeState();
+}
+
+COFFPlatformRuntimeState &COFFPlatformRuntimeState::get() {
+ assert(CPS && "COFFPlatformRuntimeState not initialized");
+ return *CPS;
+}
+
+void COFFPlatformRuntimeState::destroy() {
+ assert(CPS && "COFFPlatformRuntimeState not initialized");
+ delete CPS;
+}
+
+void *COFFPlatformRuntimeState::findJITDylibBaseByPC(uint64_t PC) {
+ std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
+ auto It = BlockRanges.upper_bound(reinterpret_cast<void *>(PC));
+ if (It == BlockRanges.begin())
+ return nullptr;
+ --It;
+ auto &Range = It->second;
+ if (PC >= reinterpret_cast<uint64_t>(It->first) + Range.Size)
+ return nullptr;
+ return Range.Header;
+}
+
+ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
+__orc_rt_coff_platform_bootstrap(char *ArgData, size_t ArgSize) {
+ COFFPlatformRuntimeState::initialize();
+ return WrapperFunctionResult().release();
+}
+
+ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
+__orc_rt_coff_platform_shutdown(char *ArgData, size_t ArgSize) {
+ COFFPlatformRuntimeState::destroy();
+ return WrapperFunctionResult().release();
+}
+
+ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
+__orc_rt_coff_register_jitdylib(char *ArgData, size_t ArgSize) {
+ return WrapperFunction<SPSError(SPSString, SPSExecutorAddr)>::handle(
+ ArgData, ArgSize,
+ [](std::string &Name, ExecutorAddr HeaderAddr) {
+ return COFFPlatformRuntimeState::get().registerJITDylib(
+ std::move(Name), HeaderAddr.toPtr<void *>());
+ })
+ .release();
+}
+
+ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
+__orc_rt_coff_deregister_jitdylib(char *ArgData, size_t ArgSize) {
+ return WrapperFunction<SPSError(SPSExecutorAddr)>::handle(
+ ArgData, ArgSize,
+ [](ExecutorAddr HeaderAddr) {
+ return COFFPlatformRuntimeState::get().deregisterJITDylib(
+ HeaderAddr.toPtr<void *>());
+ })
+ .release();
+}
+
+ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
+__orc_rt_coff_register_object_sections(char *ArgData, size_t ArgSize) {
+ return WrapperFunction<SPSError(SPSExecutorAddr, SPSCOFFObjectSectionsMap,
+ bool)>::
+ handle(ArgData, ArgSize,
+ [](ExecutorAddr HeaderAddr,
+ std::vector<std::pair<std::string_view, ExecutorAddrRange>>
+ &Secs,
+ bool RunInitializers) {
+ return COFFPlatformRuntimeState::get().registerObjectSections(
+ HeaderAddr, std::move(Secs), RunInitializers);
+ })
+ .release();
+}
+
+ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
+__orc_rt_coff_deregister_object_sections(char *ArgData, size_t ArgSize) {
+ return WrapperFunction<SPSError(SPSExecutorAddr, SPSCOFFObjectSectionsMap)>::
+ handle(ArgData, ArgSize,
+ [](ExecutorAddr HeaderAddr,
+ std::vector<std::pair<std::string_view, ExecutorAddrRange>>
+ &Secs) {
+ return COFFPlatformRuntimeState::get().deregisterObjectSections(
+ HeaderAddr, std::move(Secs));
+ })
+ .release();
+}
+//------------------------------------------------------------------------------
+// JIT'd dlfcn alternatives.
+//------------------------------------------------------------------------------
+
+const char *__orc_rt_coff_jit_dlerror() {
+ return COFFPlatformRuntimeState::get().dlerror();
+}
+
+void *__orc_rt_coff_jit_dlopen(const char *path, int mode) {
+ return COFFPlatformRuntimeState::get().dlopen(path, mode);
+}
+
+int __orc_rt_coff_jit_dlclose(void *header) {
+ return COFFPlatformRuntimeState::get().dlclose(header);
+}
+
+void *__orc_rt_coff_jit_dlsym(void *header, const char *symbol) {
+ return COFFPlatformRuntimeState::get().dlsym(header, symbol);
+}
+
+//------------------------------------------------------------------------------
+// COFF SEH exception support
+//------------------------------------------------------------------------------
+
+struct ThrowInfo {
+ uint32_t attributes;
+ void *data;
+};
+
+ORC_RT_INTERFACE void __stdcall __orc_rt_coff_cxx_throw_exception(
+ void *pExceptionObject, ThrowInfo *pThrowInfo) {
+ constexpr uint32_t EH_EXCEPTION_NUMBER = 'msc' | 0xE0000000;
+ constexpr uint32_t EH_MAGIC_NUMBER1 = 0x19930520;
+ auto BaseAddr = COFFPlatformRuntimeState::get().findJITDylibBaseByPC(
+ reinterpret_cast<uint64_t>(pThrowInfo));
+ if (!BaseAddr) {
+ // This is not from JIT'd region.
+ // FIXME: Use the default implementation like below when alias api is
+ // capable. _CxxThrowException(pExceptionObject, pThrowInfo);
+ fprintf(stderr, "Throwing exception from compiled callback into JIT'd "
+ "exception handler not supported yet.\n");
+ abort();
+ return;
+ }
+ const ULONG_PTR parameters[] = {
+ EH_MAGIC_NUMBER1,
+ reinterpret_cast<ULONG_PTR>(pExceptionObject),
+ reinterpret_cast<ULONG_PTR>(pThrowInfo),
+ reinterpret_cast<ULONG_PTR>(BaseAddr),
+ };
+ RaiseException(EH_EXCEPTION_NUMBER, EXCEPTION_NONCONTINUABLE,
+ _countof(parameters), parameters);
+}
+
+//------------------------------------------------------------------------------
+// COFF atexits
+//------------------------------------------------------------------------------
+
+typedef int (*OnExitFunction)(void);
+typedef void (*AtExitFunction)(void);
+
+ORC_RT_INTERFACE OnExitFunction __orc_rt_coff_onexit(void *Header,
+ OnExitFunction Func) {
+ if (auto Err = COFFPlatformRuntimeState::get().registerAtExit(
+ ExecutorAddr::fromPtr(Header), (void (*)(void))Func)) {
+ consumeError(std::move(Err));
+ return nullptr;
+ }
+ return Func;
+}
+
+ORC_RT_INTERFACE int __orc_rt_coff_atexit(void *Header, AtExitFunction Func) {
+ if (auto Err = COFFPlatformRuntimeState::get().registerAtExit(
+ ExecutorAddr::fromPtr(Header), (void (*)(void))Func)) {
+ consumeError(std::move(Err));
+ return -1;
+ }
+ return 0;
+}
+
+//------------------------------------------------------------------------------
+// COFF Run Program
+//------------------------------------------------------------------------------
+
+ORC_RT_INTERFACE int64_t __orc_rt_coff_run_program(const char *JITDylibName,
+ const char *EntrySymbolName,
+ int argc, char *argv[]) {
+ using MainTy = int (*)(int, char *[]);
+
+ void *H =
+ __orc_rt_coff_jit_dlopen(JITDylibName, __orc_rt::coff::ORC_RT_RTLD_LAZY);
+ if (!H) {
+ __orc_rt_log_error(__orc_rt_coff_jit_dlerror());
+ return -1;
+ }
+
+ auto *Main =
+ reinterpret_cast<MainTy>(__orc_rt_coff_jit_dlsym(H, EntrySymbolName));
+
+ if (!Main) {
+ __orc_rt_log_error(__orc_rt_coff_jit_dlerror());
+ return -1;
+ }
+
+ int Result = Main(argc, argv);
+
+ if (__orc_rt_coff_jit_dlclose(H) == -1)
+ __orc_rt_log_error(__orc_rt_coff_jit_dlerror());
+
+ return Result;
+}
--- /dev/null
+//===- coff_platform.h -------------------------------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// ORC Runtime support for dynamic loading features on COFF-based platforms.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef ORC_RT_COFF_PLATFORM_H
+#define ORC_RT_COFF_PLATFORM_H
+
+#include "common.h"
+#include "executor_address.h"
+
+// dlfcn functions.
+ORC_RT_INTERFACE const char *__orc_rt_coff_jit_dlerror();
+ORC_RT_INTERFACE void *__orc_rt_coff_jit_dlopen(const char *path, int mode);
+ORC_RT_INTERFACE int __orc_rt_coff_jit_dlclose(void *header);
+ORC_RT_INTERFACE void *__orc_rt_coff_jit_dlsym(void *header,
+ const char *symbol);
+
+namespace __orc_rt {
+namespace coff {
+
+enum dlopen_mode : int {
+ ORC_RT_RTLD_LAZY = 0x1,
+ ORC_RT_RTLD_NOW = 0x2,
+ ORC_RT_RTLD_LOCAL = 0x4,
+ ORC_RT_RTLD_GLOBAL = 0x8
+};
+
+} // end namespace coff
+} // end namespace __orc_rt
+
+#endif
--- /dev/null
+//===- coff_platform.per_jd.cpp -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains code that will be loaded per each JITDylib.
+//
+//===----------------------------------------------------------------------===//
+#include "compiler.h"
+
+ORC_RT_INTERFACE void __orc_rt_coff_per_jd_marker() {}
+
+typedef int (*OnExitFunction)(void);
+typedef void (*AtExitFunction)(void);
+
+extern "C" void *__ImageBase;
+ORC_RT_INTERFACE OnExitFunction __orc_rt_coff_onexit(void *Header,
+ OnExitFunction Func);
+ORC_RT_INTERFACE int __orc_rt_coff_atexit(void *Header, AtExitFunction Func);
+
+ORC_RT_INTERFACE OnExitFunction
+__orc_rt_coff_onexit_per_jd(OnExitFunction Func) {
+ return __orc_rt_coff_onexit(&__ImageBase, Func);
+}
+
+ORC_RT_INTERFACE int __orc_rt_coff_atexit_per_jd(AtExitFunction Func) {
+ return __orc_rt_coff_atexit(&__ImageBase, Func);
+}
#ifndef ORC_RT_COMMON_H
#define ORC_RT_COMMON_H
-#include "c_api.h"
#include "compiler.h"
+#include "orc_rt/c_api.h"
#include <type_traits>
/// This macro should be used to define tags that will be associated with
/// This is declared for use by the runtime, but should be implemented in the
/// executor or provided by a definition added to the JIT before the runtime
/// is loaded.
-extern "C" __orc_rt_Opaque __orc_rt_jit_dispatch_ctx ORC_RT_WEAK_IMPORT;
+ORC_RT_IMPORT __orc_rt_Opaque __orc_rt_jit_dispatch_ctx ORC_RT_WEAK_IMPORT;
/// For dispatching calls to the JIT object.
///
/// This is declared for use by the runtime, but should be implemented in the
/// executor or provided by a definition added to the JIT before the runtime
/// is loaded.
-extern "C" __orc_rt_CWrapperFunctionResult
+ORC_RT_IMPORT __orc_rt_CWrapperFunctionResult
__orc_rt_jit_dispatch(__orc_rt_Opaque *DispatchCtx, const void *FnTag,
const char *Data, size_t Size) ORC_RT_WEAK_IMPORT;
#ifndef ORC_RT_COMPILER_H
#define ORC_RT_COMPILER_H
+#if defined(_WIN32)
+#define ORC_RT_INTERFACE extern "C"
+#define ORC_RT_HIDDEN
+#define ORC_RT_IMPORT extern "C" __declspec(dllimport)
+#else
#define ORC_RT_INTERFACE extern "C" __attribute__((visibility("default")))
#define ORC_RT_HIDDEN __attribute__((visibility("hidden")))
+#define ORC_RT_IMPORT extern "C"
+#endif
#ifndef __has_builtin
# define __has_builtin(x) 0
#define ORC_RT_UNLIKELY(EXPR) (EXPR)
#endif
-#ifdef __APPLE__
+#if defined(__APPLE__)
#define ORC_RT_WEAK_IMPORT __attribute__((weak_import))
+#elif defined(_WIN32)
+#define ORC_RT_WEAK_IMPORT
#else
#define ORC_RT_WEAK_IMPORT __attribute__((weak))
#endif
--- /dev/null
+//===- debug.cpp ----------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime support library.
+//
+//===----------------------------------------------------------------------===//
+
+#include "debug.h"
+
+#include <cassert>
+#include <cstdarg>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+
+
+namespace __orc_rt {
+
+#ifndef NDEBUG
+
+std::atomic<const char *> DebugTypes;
+char DebugTypesAll;
+char DebugTypesNone;
+
+/// Sets the DebugState and DebugTypes values -- this function may be called
+/// concurrently on multiple threads, but will always assign the same values so
+/// this should be safe.
+const char *initializeDebug() {
+ if (const char *DT = getenv("ORC_RT_DEBUG")) {
+ // If ORC_RT_DEBUG=1 then log everything.
+ if (strcmp(DT, "1") == 0) {
+ DebugTypes.store(&DebugTypesAll, std::memory_order_relaxed);
+ return &DebugTypesAll;
+ }
+
+ // If ORC_RT_DEBUG is non-empty then record the string for use in
+ // debugTypeEnabled.
+ if (strcmp(DT, "") != 0) {
+ DebugTypes.store(DT, std::memory_order_relaxed);
+ return DT;
+ }
+ }
+
+ // If ORT_RT_DEBUG is undefined or defined as empty then log nothing.
+ DebugTypes.store(&DebugTypesNone, std::memory_order_relaxed);
+ return &DebugTypesNone;
+}
+
+bool debugTypeEnabled(const char *Type, const char *Types) {
+ assert(Types && Types != &DebugTypesAll && Types != &DebugTypesNone &&
+ "Invalid Types value");
+ size_t TypeLen = strlen(Type);
+ const char *Start = Types;
+ const char *End = Start;
+
+ do {
+ if (*End == '\0' || *End == ',') {
+ size_t ItemLen = End - Start;
+ if (ItemLen == TypeLen && memcmp(Type, Start, TypeLen) == 0)
+ return true;
+ if (*End == '\0')
+ return false;
+ Start = End + 1;
+ }
+ ++End;
+ } while (true);
+}
+
+void printdbg(const char *format, ...) {
+ va_list Args;
+ va_start(Args, format);
+ vfprintf(stderr, format, Args);
+ va_end(Args);
+}
+
+#endif // !NDEBUG
+
+} // end namespace __orc_rt
--- /dev/null
+//===- debug.h - Debugging output utilities ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime support library.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef ORC_RT_DEBUG_H
+#define ORC_RT_DEBUG_H
+
+#include <atomic>
+
+#ifndef NDEBUG
+
+namespace __orc_rt {
+
+extern std::atomic<const char *> DebugTypes;
+extern char DebugTypesAll;
+extern char DebugTypesNone;
+
+const char *initializeDebug();
+bool debugTypeEnabled(const char *Type, const char *Types);
+void printdbg(const char *format, ...);
+
+} // namespace __orc_rt
+
+#define ORC_RT_DEBUG_WITH_TYPE(TYPE, X) \
+ do { \
+ const char *Types = \
+ ::__orc_rt::DebugTypes.load(std::memory_order_relaxed); \
+ if (!Types) \
+ Types = initializeDebug(); \
+ if (Types == &DebugTypesNone) \
+ break; \
+ if (Types == &DebugTypesAll || \
+ ::__orc_rt::debugTypeEnabled(TYPE, Types)) { \
+ X; \
+ } \
+ } while (false)
+
+#else
+
+#define ORC_RT_DEBUG_WITH_TYPE(TYPE, X) \
+ do { \
+ } while (false)
+
+#endif // !NDEBUG
+
+#define ORC_RT_DEBUG(X) ORC_RT_DEBUG_WITH_TYPE(DEBUG_TYPE, X)
+
+#endif // ORC_RT_COMMON_H
--- /dev/null
+//===- dlfcn_wrapper.cpp --------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime support library.
+//
+//===----------------------------------------------------------------------===//
+
+#include "adt.h"
+#include "common.h"
+#include "wrapper_function_utils.h"
+
+#include <vector>
+
+using namespace __orc_rt;
+
+extern "C" const char *__orc_rt_jit_dlerror();
+extern "C" void *__orc_rt_jit_dlopen(const char *path, int mode);
+extern "C" int __orc_rt_jit_dlclose(void *dso_handle);
+
+ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
+__orc_rt_jit_dlerror_wrapper(const char *ArgData, size_t ArgSize) {
+ return WrapperFunction<SPSString()>::handle(
+ ArgData, ArgSize,
+ []() { return std::string(__orc_rt_jit_dlerror()); })
+ .release();
+}
+
+ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
+__orc_rt_jit_dlopen_wrapper(const char *ArgData, size_t ArgSize) {
+ return WrapperFunction<SPSExecutorAddr(SPSString, int32_t)>::handle(
+ ArgData, ArgSize,
+ [](const std::string &Path, int32_t mode) {
+ return ExecutorAddr::fromPtr(
+ __orc_rt_jit_dlopen(Path.c_str(), mode));
+ })
+ .release();
+}
+
+ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
+__orc_rt_jit_dlclose_wrapper(const char *ArgData, size_t ArgSize) {
+ return WrapperFunction<int32_t(SPSExecutorAddr)>::handle(
+ ArgData, ArgSize,
+ [](ExecutorAddr &DSOHandle) {
+ return __orc_rt_jit_dlclose(DSOHandle.toPtr<void *>());
+ })
+ .release();
+}
--- /dev/null
+//===- elfnix_platform.cpp ------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains code required to load the rest of the ELF-on-*IX runtime.
+//
+//===----------------------------------------------------------------------===//
+
+#include "elfnix_platform.h"
+#include "common.h"
+#include "error.h"
+#include "wrapper_function_utils.h"
+
+#include <algorithm>
+#include <map>
+#include <mutex>
+#include <sstream>
+#include <string_view>
+#include <unordered_map>
+#include <vector>
+
+using namespace __orc_rt;
+using namespace __orc_rt::elfnix;
+
+// Declare function tags for functions in the JIT process.
+ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_get_initializers_tag)
+ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_get_deinitializers_tag)
+ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_symbol_lookup_tag)
+
+// eh-frame registration functions, made available via aliases
+// installed by the Platform
+extern "C" void __orc_rt_register_eh_frame_section(const void *);
+extern "C" void __orc_rt_deregister_eh_frame_section(const void *);
+
+namespace {
+
+Error validatePointerSectionExtent(const char *SectionName,
+ const ExecutorAddrRange &SE) {
+ if (SE.size() % sizeof(uintptr_t)) {
+ std::ostringstream ErrMsg;
+ ErrMsg << std::hex << "Size of " << SectionName << " 0x"
+ << SE.Start.getValue() << " -- 0x" << SE.End.getValue()
+ << " is not a pointer multiple";
+ return make_error<StringError>(ErrMsg.str());
+ }
+ return Error::success();
+}
+
+Error runInitArray(const std::vector<ExecutorAddrRange> &InitArraySections,
+ const ELFNixJITDylibInitializers &MOJDIs) {
+
+ for (const auto &ModInits : InitArraySections) {
+ if (auto Err = validatePointerSectionExtent(".init_array", ModInits))
+ return Err;
+
+ using InitFunc = void (*)();
+ for (auto *Init : ModInits.toSpan<InitFunc>())
+ (*Init)();
+ }
+
+ return Error::success();
+}
+
+struct TLSInfoEntry {
+ unsigned long Key = 0;
+ unsigned long DataAddress = 0;
+};
+
+struct TLSDescriptor {
+ void (*Resolver)(void *);
+ TLSInfoEntry *InfoEntry;
+};
+
+class ELFNixPlatformRuntimeState {
+private:
+ struct AtExitEntry {
+ void (*Func)(void *);
+ void *Arg;
+ };
+
+ using AtExitsVector = std::vector<AtExitEntry>;
+
+ struct PerJITDylibState {
+ void *Header = nullptr;
+ size_t RefCount = 0;
+ bool AllowReinitialization = false;
+ AtExitsVector AtExits;
+ };
+
+public:
+ static void initialize(void *DSOHandle);
+ static ELFNixPlatformRuntimeState &get();
+ static void destroy();
+
+ ELFNixPlatformRuntimeState(void *DSOHandle)
+ : PlatformJDDSOHandle(DSOHandle) {}
+
+ // Delete copy and move constructors.
+ ELFNixPlatformRuntimeState(const ELFNixPlatformRuntimeState &) = delete;
+ ELFNixPlatformRuntimeState &
+ operator=(const ELFNixPlatformRuntimeState &) = delete;
+ ELFNixPlatformRuntimeState(ELFNixPlatformRuntimeState &&) = delete;
+ ELFNixPlatformRuntimeState &operator=(ELFNixPlatformRuntimeState &&) = delete;
+
+ Error registerObjectSections(ELFNixPerObjectSectionsToRegister POSR);
+ Error deregisterObjectSections(ELFNixPerObjectSectionsToRegister POSR);
+
+ const char *dlerror();
+ void *dlopen(std::string_view Name, int Mode);
+ int dlclose(void *DSOHandle);
+ void *dlsym(void *DSOHandle, std::string_view Symbol);
+
+ int registerAtExit(void (*F)(void *), void *Arg, void *DSOHandle);
+ void runAtExits(void *DSOHandle);
+
+ /// Returns the base address of the section containing ThreadData.
+ Expected<std::pair<const char *, size_t>>
+ getThreadDataSectionFor(const char *ThreadData);
+
+ void *getPlatformJDDSOHandle() { return PlatformJDDSOHandle; }
+
+private:
+ PerJITDylibState *getJITDylibStateByHeaderAddr(void *DSOHandle);
+ PerJITDylibState *getJITDylibStateByName(std::string_view Path);
+ PerJITDylibState &
+ getOrCreateJITDylibState(ELFNixJITDylibInitializers &MOJDIs);
+
+ Error registerThreadDataSection(span<const char> ThreadDataSection);
+
+ Expected<ExecutorAddr> lookupSymbolInJITDylib(void *DSOHandle,
+ std::string_view Symbol);
+
+ Expected<ELFNixJITDylibInitializerSequence>
+ getJITDylibInitializersByName(std::string_view Path);
+ Expected<void *> dlopenInitialize(std::string_view Path, int Mode);
+ Error initializeJITDylib(ELFNixJITDylibInitializers &MOJDIs);
+
+ static ELFNixPlatformRuntimeState *MOPS;
+
+ void *PlatformJDDSOHandle;
+
+ // FIXME: Move to thread-state.
+ std::string DLFcnError;
+
+ std::recursive_mutex JDStatesMutex;
+ std::unordered_map<void *, PerJITDylibState> JDStates;
+ std::unordered_map<std::string, void *> JDNameToHeader;
+
+ std::mutex ThreadDataSectionsMutex;
+ std::map<const char *, size_t> ThreadDataSections;
+};
+
+ELFNixPlatformRuntimeState *ELFNixPlatformRuntimeState::MOPS = nullptr;
+
+void ELFNixPlatformRuntimeState::initialize(void *DSOHandle) {
+ assert(!MOPS && "ELFNixPlatformRuntimeState should be null");
+ MOPS = new ELFNixPlatformRuntimeState(DSOHandle);
+}
+
+ELFNixPlatformRuntimeState &ELFNixPlatformRuntimeState::get() {
+ assert(MOPS && "ELFNixPlatformRuntimeState not initialized");
+ return *MOPS;
+}
+
+void ELFNixPlatformRuntimeState::destroy() {
+ assert(MOPS && "ELFNixPlatformRuntimeState not initialized");
+ delete MOPS;
+}
+
+Error ELFNixPlatformRuntimeState::registerObjectSections(
+ ELFNixPerObjectSectionsToRegister POSR) {
+ if (POSR.EHFrameSection.Start)
+ __orc_rt_register_eh_frame_section(
+ POSR.EHFrameSection.Start.toPtr<const char *>());
+
+ if (POSR.ThreadDataSection.Start) {
+ if (auto Err = registerThreadDataSection(
+ POSR.ThreadDataSection.toSpan<const char>()))
+ return Err;
+ }
+
+ return Error::success();
+}
+
+Error ELFNixPlatformRuntimeState::deregisterObjectSections(
+ ELFNixPerObjectSectionsToRegister POSR) {
+ if (POSR.EHFrameSection.Start)
+ __orc_rt_deregister_eh_frame_section(
+ POSR.EHFrameSection.Start.toPtr<const char *>());
+
+ return Error::success();
+}
+
+const char *ELFNixPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); }
+
+void *ELFNixPlatformRuntimeState::dlopen(std::string_view Path, int Mode) {
+ std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
+
+ // Use fast path if all JITDylibs are already loaded and don't require
+ // re-running initializers.
+ if (auto *JDS = getJITDylibStateByName(Path)) {
+ if (!JDS->AllowReinitialization) {
+ ++JDS->RefCount;
+ return JDS->Header;
+ }
+ }
+
+ auto H = dlopenInitialize(Path, Mode);
+ if (!H) {
+ DLFcnError = toString(H.takeError());
+ return nullptr;
+ }
+
+ return *H;
+}
+
+int ELFNixPlatformRuntimeState::dlclose(void *DSOHandle) {
+ runAtExits(DSOHandle);
+ return 0;
+}
+
+void *ELFNixPlatformRuntimeState::dlsym(void *DSOHandle,
+ std::string_view Symbol) {
+ auto Addr = lookupSymbolInJITDylib(DSOHandle, Symbol);
+ if (!Addr) {
+ DLFcnError = toString(Addr.takeError());
+ return 0;
+ }
+
+ return Addr->toPtr<void *>();
+}
+
+int ELFNixPlatformRuntimeState::registerAtExit(void (*F)(void *), void *Arg,
+ void *DSOHandle) {
+ // FIXME: Handle out-of-memory errors, returning -1 if OOM.
+ std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
+ auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
+ assert(JDS && "JITDylib state not initialized");
+ JDS->AtExits.push_back({F, Arg});
+ return 0;
+}
+
+void ELFNixPlatformRuntimeState::runAtExits(void *DSOHandle) {
+ // FIXME: Should atexits be allowed to run concurrently with access to
+ // JDState?
+ AtExitsVector V;
+ {
+ std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
+ auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
+ assert(JDS && "JITDlybi state not initialized");
+ std::swap(V, JDS->AtExits);
+ }
+
+ while (!V.empty()) {
+ auto &AE = V.back();
+ AE.Func(AE.Arg);
+ V.pop_back();
+ }
+}
+
+Expected<std::pair<const char *, size_t>>
+ELFNixPlatformRuntimeState::getThreadDataSectionFor(const char *ThreadData) {
+ std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
+ auto I = ThreadDataSections.upper_bound(ThreadData);
+ // Check that we have a valid entry conovering this address.
+ if (I == ThreadDataSections.begin())
+ return make_error<StringError>("No thread local data section for key");
+ I = std::prev(I);
+ if (ThreadData >= I->first + I->second)
+ return make_error<StringError>("No thread local data section for key");
+ return *I;
+}
+
+ELFNixPlatformRuntimeState::PerJITDylibState *
+ELFNixPlatformRuntimeState::getJITDylibStateByHeaderAddr(void *DSOHandle) {
+ auto I = JDStates.find(DSOHandle);
+ if (I == JDStates.end())
+ return nullptr;
+ return &I->second;
+}
+
+ELFNixPlatformRuntimeState::PerJITDylibState *
+ELFNixPlatformRuntimeState::getJITDylibStateByName(std::string_view Name) {
+ // FIXME: Avoid creating string copy here.
+ auto I = JDNameToHeader.find(std::string(Name.data(), Name.size()));
+ if (I == JDNameToHeader.end())
+ return nullptr;
+ void *H = I->second;
+ auto J = JDStates.find(H);
+ assert(J != JDStates.end() &&
+ "JITDylib has name map entry but no header map entry");
+ return &J->second;
+}
+
+ELFNixPlatformRuntimeState::PerJITDylibState &
+ELFNixPlatformRuntimeState::getOrCreateJITDylibState(
+ ELFNixJITDylibInitializers &MOJDIs) {
+ void *Header = MOJDIs.DSOHandleAddress.toPtr<void *>();
+
+ auto &JDS = JDStates[Header];
+
+ // If this entry hasn't been created yet.
+ if (!JDS.Header) {
+ assert(!JDNameToHeader.count(MOJDIs.Name) &&
+ "JITDylib has header map entry but no name map entry");
+ JDNameToHeader[MOJDIs.Name] = Header;
+ JDS.Header = Header;
+ }
+
+ return JDS;
+}
+
+Error ELFNixPlatformRuntimeState::registerThreadDataSection(
+ span<const char> ThreadDataSection) {
+ std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
+ auto I = ThreadDataSections.upper_bound(ThreadDataSection.data());
+ if (I != ThreadDataSections.begin()) {
+ auto J = std::prev(I);
+ if (J->first + J->second > ThreadDataSection.data())
+ return make_error<StringError>("Overlapping .tdata sections");
+ }
+ ThreadDataSections.insert(
+ I, std::make_pair(ThreadDataSection.data(), ThreadDataSection.size()));
+ return Error::success();
+}
+
+Expected<ExecutorAddr>
+ELFNixPlatformRuntimeState::lookupSymbolInJITDylib(void *DSOHandle,
+ std::string_view Sym) {
+ Expected<ExecutorAddr> Result((ExecutorAddr()));
+ if (auto Err = WrapperFunction<SPSExpected<SPSExecutorAddr>(
+ SPSExecutorAddr, SPSString)>::call(&__orc_rt_elfnix_symbol_lookup_tag,
+ Result,
+ ExecutorAddr::fromPtr(DSOHandle),
+ Sym))
+ return std::move(Err);
+ return Result;
+}
+
+Expected<ELFNixJITDylibInitializerSequence>
+ELFNixPlatformRuntimeState::getJITDylibInitializersByName(
+ std::string_view Path) {
+ Expected<ELFNixJITDylibInitializerSequence> Result(
+ (ELFNixJITDylibInitializerSequence()));
+ std::string PathStr(Path.data(), Path.size());
+ if (auto Err =
+ WrapperFunction<SPSExpected<SPSELFNixJITDylibInitializerSequence>(
+ SPSString)>::call(&__orc_rt_elfnix_get_initializers_tag, Result,
+ Path))
+ return std::move(Err);
+ return Result;
+}
+
+Expected<void *>
+ELFNixPlatformRuntimeState::dlopenInitialize(std::string_view Path, int Mode) {
+ // Either our JITDylib wasn't loaded, or it or one of its dependencies allows
+ // reinitialization. We need to call in to the JIT to see if there's any new
+ // work pending.
+ auto InitSeq = getJITDylibInitializersByName(Path);
+ if (!InitSeq)
+ return InitSeq.takeError();
+
+ // Init sequences should be non-empty.
+ if (InitSeq->empty())
+ return make_error<StringError>(
+ "__orc_rt_elfnix_get_initializers returned an "
+ "empty init sequence");
+
+ // Otherwise register and run initializers for each JITDylib.
+ for (auto &MOJDIs : *InitSeq)
+ if (auto Err = initializeJITDylib(MOJDIs))
+ return std::move(Err);
+
+ // Return the header for the last item in the list.
+ auto *JDS = getJITDylibStateByHeaderAddr(
+ InitSeq->back().DSOHandleAddress.toPtr<void *>());
+ assert(JDS && "Missing state entry for JD");
+ return JDS->Header;
+}
+
+long getPriority(const std::string &name) {
+ auto pos = name.find_last_not_of("0123456789");
+ if (pos == name.size() - 1)
+ return 65535;
+ else
+ return std::strtol(name.c_str() + pos + 1, nullptr, 10);
+}
+
+Error ELFNixPlatformRuntimeState::initializeJITDylib(
+ ELFNixJITDylibInitializers &MOJDIs) {
+
+ auto &JDS = getOrCreateJITDylibState(MOJDIs);
+ ++JDS.RefCount;
+
+ using SectionList = std::vector<ExecutorAddrRange>;
+ std::sort(MOJDIs.InitSections.begin(), MOJDIs.InitSections.end(),
+ [](const std::pair<std::string, SectionList> &LHS,
+ const std::pair<std::string, SectionList> &RHS) -> bool {
+ return getPriority(LHS.first) < getPriority(RHS.first);
+ });
+ for (auto &Entry : MOJDIs.InitSections)
+ if (auto Err = runInitArray(Entry.second, MOJDIs))
+ return Err;
+
+ return Error::success();
+}
+class ELFNixPlatformRuntimeTLVManager {
+public:
+ void *getInstance(const char *ThreadData);
+
+private:
+ std::unordered_map<const char *, char *> Instances;
+ std::unordered_map<const char *, std::unique_ptr<char[]>> AllocatedSections;
+};
+
+void *ELFNixPlatformRuntimeTLVManager::getInstance(const char *ThreadData) {
+ auto I = Instances.find(ThreadData);
+ if (I != Instances.end())
+ return I->second;
+ auto TDS =
+ ELFNixPlatformRuntimeState::get().getThreadDataSectionFor(ThreadData);
+ if (!TDS) {
+ __orc_rt_log_error(toString(TDS.takeError()).c_str());
+ return nullptr;
+ }
+
+ auto &Allocated = AllocatedSections[TDS->first];
+ if (!Allocated) {
+ Allocated = std::make_unique<char[]>(TDS->second);
+ memcpy(Allocated.get(), TDS->first, TDS->second);
+ }
+ size_t ThreadDataDelta = ThreadData - TDS->first;
+ assert(ThreadDataDelta <= TDS->second && "ThreadData outside section bounds");
+
+ char *Instance = Allocated.get() + ThreadDataDelta;
+ Instances[ThreadData] = Instance;
+ return Instance;
+}
+
+void destroyELFNixTLVMgr(void *ELFNixTLVMgr) {
+ delete static_cast<ELFNixPlatformRuntimeTLVManager *>(ELFNixTLVMgr);
+}
+
+} // end anonymous namespace
+
+//------------------------------------------------------------------------------
+// JIT entry points
+//------------------------------------------------------------------------------
+
+ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
+__orc_rt_elfnix_platform_bootstrap(char *ArgData, size_t ArgSize) {
+ return WrapperFunction<void(uint64_t)>::handle(
+ ArgData, ArgSize,
+ [](uint64_t &DSOHandle) {
+ ELFNixPlatformRuntimeState::initialize(
+ reinterpret_cast<void *>(DSOHandle));
+ })
+ .release();
+}
+
+ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
+__orc_rt_elfnix_platform_shutdown(char *ArgData, size_t ArgSize) {
+ ELFNixPlatformRuntimeState::destroy();
+ return WrapperFunctionResult().release();
+}
+
+/// Wrapper function for registering metadata on a per-object basis.
+ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
+__orc_rt_elfnix_register_object_sections(char *ArgData, size_t ArgSize) {
+ return WrapperFunction<SPSError(SPSELFNixPerObjectSectionsToRegister)>::
+ handle(ArgData, ArgSize,
+ [](ELFNixPerObjectSectionsToRegister &POSR) {
+ return ELFNixPlatformRuntimeState::get().registerObjectSections(
+ std::move(POSR));
+ })
+ .release();
+}
+
+/// Wrapper for releasing per-object metadat.
+ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
+__orc_rt_elfnix_deregister_object_sections(char *ArgData, size_t ArgSize) {
+ return WrapperFunction<SPSError(SPSELFNixPerObjectSectionsToRegister)>::
+ handle(ArgData, ArgSize,
+ [](ELFNixPerObjectSectionsToRegister &POSR) {
+ return ELFNixPlatformRuntimeState::get()
+ .deregisterObjectSections(std::move(POSR));
+ })
+ .release();
+}
+
+//------------------------------------------------------------------------------
+// TLV support
+//------------------------------------------------------------------------------
+
+ORC_RT_INTERFACE void *__orc_rt_elfnix_tls_get_addr_impl(TLSInfoEntry *D) {
+ auto *TLVMgr = static_cast<ELFNixPlatformRuntimeTLVManager *>(
+ pthread_getspecific(D->Key));
+ if (!TLVMgr)
+ TLVMgr = new ELFNixPlatformRuntimeTLVManager();
+ if (pthread_setspecific(D->Key, TLVMgr)) {
+ __orc_rt_log_error("Call to pthread_setspecific failed");
+ return nullptr;
+ }
+
+ return TLVMgr->getInstance(
+ reinterpret_cast<char *>(static_cast<uintptr_t>(D->DataAddress)));
+}
+
+ORC_RT_INTERFACE ptrdiff_t ___orc_rt_elfnix_tlsdesc_resolver_impl(
+ TLSDescriptor *D, const char *ThreadPointer) {
+ const char *TLVPtr = reinterpret_cast<const char *>(
+ __orc_rt_elfnix_tls_get_addr_impl(D->InfoEntry));
+ return TLVPtr - ThreadPointer;
+}
+
+ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
+__orc_rt_elfnix_create_pthread_key(char *ArgData, size_t ArgSize) {
+ return WrapperFunction<SPSExpected<uint64_t>(void)>::handle(
+ ArgData, ArgSize,
+ []() -> Expected<uint64_t> {
+ pthread_key_t Key;
+ if (int Err = pthread_key_create(&Key, destroyELFNixTLVMgr)) {
+ __orc_rt_log_error("Call to pthread_key_create failed");
+ return make_error<StringError>(strerror(Err));
+ }
+ return static_cast<uint64_t>(Key);
+ })
+ .release();
+}
+
+//------------------------------------------------------------------------------
+// cxa_atexit support
+//------------------------------------------------------------------------------
+
+int __orc_rt_elfnix_cxa_atexit(void (*func)(void *), void *arg,
+ void *dso_handle) {
+ return ELFNixPlatformRuntimeState::get().registerAtExit(func, arg,
+ dso_handle);
+}
+
+int __orc_rt_elfnix_atexit(void (*func)(void *)) {
+ auto &PlatformRTState = ELFNixPlatformRuntimeState::get();
+ return ELFNixPlatformRuntimeState::get().registerAtExit(
+ func, NULL, PlatformRTState.getPlatformJDDSOHandle());
+}
+
+void __orc_rt_elfnix_cxa_finalize(void *dso_handle) {
+ ELFNixPlatformRuntimeState::get().runAtExits(dso_handle);
+}
+
+//------------------------------------------------------------------------------
+// JIT'd dlfcn alternatives.
+//------------------------------------------------------------------------------
+
+const char *__orc_rt_elfnix_jit_dlerror() {
+ return ELFNixPlatformRuntimeState::get().dlerror();
+}
+
+void *__orc_rt_elfnix_jit_dlopen(const char *path, int mode) {
+ return ELFNixPlatformRuntimeState::get().dlopen(path, mode);
+}
+
+int __orc_rt_elfnix_jit_dlclose(void *dso_handle) {
+ return ELFNixPlatformRuntimeState::get().dlclose(dso_handle);
+}
+
+void *__orc_rt_elfnix_jit_dlsym(void *dso_handle, const char *symbol) {
+ return ELFNixPlatformRuntimeState::get().dlsym(dso_handle, symbol);
+}
+
+//------------------------------------------------------------------------------
+// ELFNix Run Program
+//------------------------------------------------------------------------------
+
+ORC_RT_INTERFACE int64_t __orc_rt_elfnix_run_program(
+ const char *JITDylibName, const char *EntrySymbolName, int argc,
+ char *argv[]) {
+ using MainTy = int (*)(int, char *[]);
+
+ void *H = __orc_rt_elfnix_jit_dlopen(JITDylibName,
+ __orc_rt::elfnix::ORC_RT_RTLD_LAZY);
+ if (!H) {
+ __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror());
+ return -1;
+ }
+
+ auto *Main =
+ reinterpret_cast<MainTy>(__orc_rt_elfnix_jit_dlsym(H, EntrySymbolName));
+
+ if (!Main) {
+ __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror());
+ return -1;
+ }
+
+ int Result = Main(argc, argv);
+
+ if (__orc_rt_elfnix_jit_dlclose(H) == -1)
+ __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror());
+
+ return Result;
+}
--- /dev/null
+//===- elfnix_platform.h ----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// ORC Runtime support for dynamic loading features on ELF-based platforms.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef ORC_RT_ELFNIX_PLATFORM_H
+#define ORC_RT_ELFNIX_PLATFORM_H
+
+#include "common.h"
+#include "executor_address.h"
+
+// Atexit functions.
+ORC_RT_INTERFACE int __orc_rt_elfnix_cxa_atexit(void (*func)(void *), void *arg,
+ void *dso_handle);
+ORC_RT_INTERFACE int __orc_rt_elfnix_atexit(void (*func)(void *));
+ORC_RT_INTERFACE void __orc_rt_elfnix_cxa_finalize(void *dso_handle);
+
+// dlfcn functions.
+ORC_RT_INTERFACE const char *__orc_rt_elfnix_jit_dlerror();
+ORC_RT_INTERFACE void *__orc_rt_elfnix_jit_dlopen(const char *path, int mode);
+ORC_RT_INTERFACE int __orc_rt_elfnix_jit_dlclose(void *dso_handle);
+ORC_RT_INTERFACE void *__orc_rt_elfnix_jit_dlsym(void *dso_handle,
+ const char *symbol);
+
+namespace __orc_rt {
+namespace elfnix {
+
+struct ELFNixPerObjectSectionsToRegister {
+ ExecutorAddrRange EHFrameSection;
+ ExecutorAddrRange ThreadDataSection;
+};
+
+struct ELFNixJITDylibInitializers {
+ using SectionList = std::vector<ExecutorAddrRange>;
+
+ ELFNixJITDylibInitializers() = default;
+ ELFNixJITDylibInitializers(std::string Name, ExecutorAddr DSOHandleAddress)
+ : Name(std::move(Name)), DSOHandleAddress(std::move(DSOHandleAddress)) {}
+
+ std::string Name;
+ ExecutorAddr DSOHandleAddress;
+
+ std::vector<std::pair<std::string, SectionList>> InitSections;
+};
+
+class ELFNixJITDylibDeinitializers {};
+
+using ELFNixJITDylibInitializerSequence =
+ std::vector<ELFNixJITDylibInitializers>;
+
+using ELFNixJITDylibDeinitializerSequence =
+ std::vector<ELFNixJITDylibDeinitializers>;
+
+enum dlopen_mode : int {
+ ORC_RT_RTLD_LAZY = 0x1,
+ ORC_RT_RTLD_NOW = 0x2,
+ ORC_RT_RTLD_LOCAL = 0x4,
+ ORC_RT_RTLD_GLOBAL = 0x8
+};
+
+} // end namespace elfnix
+
+using SPSELFNixPerObjectSectionsToRegister =
+ SPSTuple<SPSExecutorAddrRange, SPSExecutorAddrRange>;
+
+template <>
+class SPSSerializationTraits<SPSELFNixPerObjectSectionsToRegister,
+ elfnix::ELFNixPerObjectSectionsToRegister> {
+
+public:
+ static size_t size(const elfnix::ELFNixPerObjectSectionsToRegister &MOPOSR) {
+ return SPSELFNixPerObjectSectionsToRegister::AsArgList::size(
+ MOPOSR.EHFrameSection, MOPOSR.ThreadDataSection);
+ }
+
+ static bool
+ serialize(SPSOutputBuffer &OB,
+ const elfnix::ELFNixPerObjectSectionsToRegister &MOPOSR) {
+ return SPSELFNixPerObjectSectionsToRegister::AsArgList::serialize(
+ OB, MOPOSR.EHFrameSection, MOPOSR.ThreadDataSection);
+ }
+
+ static bool deserialize(SPSInputBuffer &IB,
+ elfnix::ELFNixPerObjectSectionsToRegister &MOPOSR) {
+ return SPSELFNixPerObjectSectionsToRegister::AsArgList::deserialize(
+ IB, MOPOSR.EHFrameSection, MOPOSR.ThreadDataSection);
+ }
+};
+
+using SPSNamedExecutorAddrRangeSequenceMap =
+ SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRangeSequence>>;
+
+using SPSELFNixJITDylibInitializers =
+ SPSTuple<SPSString, SPSExecutorAddr, SPSNamedExecutorAddrRangeSequenceMap>;
+
+using SPSELFNixJITDylibInitializerSequence =
+ SPSSequence<SPSELFNixJITDylibInitializers>;
+
+/// Serialization traits for ELFNixJITDylibInitializers.
+template <>
+class SPSSerializationTraits<SPSELFNixJITDylibInitializers,
+ elfnix::ELFNixJITDylibInitializers> {
+public:
+ static size_t size(const elfnix::ELFNixJITDylibInitializers &MOJDIs) {
+ return SPSELFNixJITDylibInitializers::AsArgList::size(
+ MOJDIs.Name, MOJDIs.DSOHandleAddress, MOJDIs.InitSections);
+ }
+
+ static bool serialize(SPSOutputBuffer &OB,
+ const elfnix::ELFNixJITDylibInitializers &MOJDIs) {
+ return SPSELFNixJITDylibInitializers::AsArgList::serialize(
+ OB, MOJDIs.Name, MOJDIs.DSOHandleAddress, MOJDIs.InitSections);
+ }
+
+ static bool deserialize(SPSInputBuffer &IB,
+ elfnix::ELFNixJITDylibInitializers &MOJDIs) {
+ return SPSELFNixJITDylibInitializers::AsArgList::deserialize(
+ IB, MOJDIs.Name, MOJDIs.DSOHandleAddress, MOJDIs.InitSections);
+ }
+};
+
+} // end namespace __orc_rt
+
+#endif // ORC_RT_ELFNIX_PLATFORM_H
--- /dev/null
+//===-- elfnix_tlv.aarch64.s ---------------------------------------*- ASM -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime support library.
+//
+//===----------------------------------------------------------------------===//
+
+// The content of this file is aarch64-only
+#if defined(__arm64__) || defined(__aarch64__)
+
+#define REGISTER_SAVE_SPACE_SIZE 32 * 24
+
+ .text
+
+ // returns address of TLV in x0, all other registers preserved
+ // TODO: add fast-path for repeat access
+ .globl ___orc_rt_elfnix_tlsdesc_resolver
+___orc_rt_elfnix_tlsdesc_resolver:
+ sub sp, sp, #REGISTER_SAVE_SPACE_SIZE
+ stp x29, x30, [sp, #16 * 1]
+ stp x27, x28, [sp, #16 * 2]
+ stp x25, x26, [sp, #16 * 3]
+ stp x23, x24, [sp, #16 * 4]
+ stp x21, x22, [sp, #16 * 5]
+ stp x19, x20, [sp, #16 * 6]
+ stp x17, x18, [sp, #16 * 7]
+ stp x15, x16, [sp, #16 * 8]
+ stp x13, x14, [sp, #16 * 9]
+ stp x11, x12, [sp, #16 * 10]
+ stp x9, x10, [sp, #16 * 11]
+ stp x7, x8, [sp, #16 * 12]
+ stp x5, x6, [sp, #16 * 13]
+ stp x3, x4, [sp, #16 * 14]
+ stp x1, x2, [sp, #16 * 15]
+ stp q30, q31, [sp, #32 * 8]
+ stp q28, q29, [sp, #32 * 9]
+ stp q26, q27, [sp, #32 * 10]
+ stp q24, q25, [sp, #32 * 11]
+ stp q22, q23, [sp, #32 * 12]
+ stp q20, q21, [sp, #32 * 13]
+ stp q18, q19, [sp, #32 * 14]
+ stp q16, q17, [sp, #32 * 15]
+ stp q14, q15, [sp, #32 * 16]
+ stp q12, q13, [sp, #32 * 17]
+ stp q10, q11, [sp, #32 * 18]
+ stp q8, q9, [sp, #32 * 19]
+ stp q6, q7, [sp, #32 * 20]
+ stp q4, q5, [sp, #32 * 21]
+ stp q2, q3, [sp, #32 * 22]
+ stp q0, q1, [sp, #32 * 23]
+
+ mrs x1, TPIDR_EL0 // get thread pointer
+ bl ___orc_rt_elfnix_tlsdesc_resolver_impl
+
+ ldp q0, q1, [sp, #32 * 23]
+ ldp q2, q3, [sp, #32 * 22]
+ ldp q4, q5, [sp, #32 * 21]
+ ldp q6, q7, [sp, #32 * 20]
+ ldp q8, q9, [sp, #32 * 19]
+ ldp q10, q11, [sp, #32 * 18]
+ ldp q12, q13, [sp, #32 * 17]
+ ldp q14, q15, [sp, #32 * 16]
+ ldp q16, q17, [sp, #32 * 15]
+ ldp q18, q19, [sp, #32 * 14]
+ ldp q20, q21, [sp, #32 * 13]
+ ldp q22, q23, [sp, #32 * 12]
+ ldp q24, q25, [sp, #32 * 11]
+ ldp q26, q27, [sp, #32 * 10]
+ ldp q28, q29, [sp, #32 * 9]
+ ldp q30, q31, [sp, #32 * 8]
+ ldp x1, x2, [sp, #16 * 15]
+ ldp x3, x4, [sp, #16 * 14]
+ ldp x5, x6, [sp, #16 * 13]
+ ldp x7, x8, [sp, #16 * 12]
+ ldp x9, x10, [sp, #16 * 11]
+ ldp x11, x12, [sp, #16 * 10]
+ ldp x13, x14, [sp, #16 * 9]
+ ldp x15, x16, [sp, #16 * 8]
+ ldp x17, x18, [sp, #16 * 7]
+ ldp x19, x20, [sp, #16 * 6]
+ ldp x21, x22, [sp, #16 * 5]
+ ldp x23, x24, [sp, #16 * 4]
+ ldp x25, x26, [sp, #16 * 3]
+ ldp x27, x28, [sp, #16 * 2]
+ ldp x29, x30, [sp, #16 * 1]
+ add sp, sp, #REGISTER_SAVE_SPACE_SIZE
+ ret
+
+#endif // defined(__arm64__) || defined(__aarch64__)
--- /dev/null
+
+//===-- orc_rt_elfnix_tls_x86-64.s -------------------------------*- ASM -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime support library.
+//
+//===----------------------------------------------------------------------===//
+
+// The content of this file is x86_64-only
+#if defined(__x86_64__)
+
+#define REGISTER_SAVE_SPACE_SIZE 512
+
+ .text
+
+ // returns address of TLV in %rax, all other registers preserved
+ .globl ___orc_rt_elfnix_tls_get_addr
+___orc_rt_elfnix_tls_get_addr:
+ pushq %rbp
+ movq %rsp, %rbp
+ subq $REGISTER_SAVE_SPACE_SIZE, %rsp
+ movq %rcx, -16(%rbp)
+ movq %rdx, -24(%rbp)
+ movq %rsi, -32(%rbp)
+ movq %rdi, -40(%rbp)
+ movq %r8, -48(%rbp)
+ movq %r9, -56(%rbp)
+ movq %r10, -64(%rbp)
+ movq %r11, -72(%rbp)
+ movdqa %xmm0, -128(%rbp)
+ movdqa %xmm1, -144(%rbp)
+ movdqa %xmm2, -160(%rbp)
+ movdqa %xmm3, -176(%rbp)
+ movdqa %xmm4, -192(%rbp)
+ movdqa %xmm5, -208(%rbp)
+ movdqa %xmm6, -224(%rbp)
+ movdqa %xmm7, -240(%rbp)
+ call __orc_rt_elfnix_tls_get_addr_impl
+ movq -16(%rbp), %rcx
+ movq -24(%rbp), %rdx
+ movq -32(%rbp), %rsi
+ movq -40(%rbp), %rdi
+ movq -48(%rbp), %r8
+ movq -56(%rbp), %r9
+ movq -64(%rbp), %r10
+ movq -72(%rbp), %r11
+ movdqa -128(%rbp), %xmm0
+ movdqa -144(%rbp), %xmm1
+ movdqa -160(%rbp), %xmm2
+ movdqa -176(%rbp), %xmm3
+ movdqa -192(%rbp), %xmm4
+ movdqa -208(%rbp), %xmm5
+ movdqa -224(%rbp), %xmm6
+ movdqa -240(%rbp), %xmm7
+ addq $REGISTER_SAVE_SPACE_SIZE, %rsp
+ popq %rbp
+ ret
+
+#endif // defined(__x86_64__)
bool isChecked() const { return ErrPtr & 0x1; }
- void setChecked(bool Checked) {
- ErrPtr = (reinterpret_cast<uintptr_t>(ErrPtr) & ~uintptr_t(1)) | Checked;
- }
+ void setChecked(bool Checked) { ErrPtr = (ErrPtr & ~uintptr_t(1)) | Checked; }
template <typename ErrT = ErrorInfoBase> std::unique_ptr<ErrT> takePayload() {
static_assert(std::is_base_of<ErrorInfoBase, ErrT>::value,
namespace __orc_rt {
-/// Represents the difference between two addresses in the executor process.
-class ExecutorAddrDiff {
-public:
- ExecutorAddrDiff() = default;
- explicit ExecutorAddrDiff(uint64_t Value) : Value(Value) {}
-
- uint64_t getValue() const { return Value; }
-
-private:
- int64_t Value = 0;
-};
+using ExecutorAddrDiff = uint64_t;
/// Represents an address in the executor process.
-class ExecutorAddress {
+class ExecutorAddr {
public:
- ExecutorAddress() = default;
- explicit ExecutorAddress(uint64_t Addr) : Addr(Addr) {}
+ /// A wrap/unwrap function that leaves pointers unmodified.
+ template <typename T> using rawPtr = __orc_rt::identity<T *>;
+
+ /// Default wrap function to use on this host.
+ template <typename T> using defaultWrap = rawPtr<T>;
+
+ /// Default unwrap function to use on this host.
+ template <typename T> using defaultUnwrap = rawPtr<T>;
+
+ /// Merges a tag into the raw address value:
+ /// P' = P | (TagValue << TagOffset).
+ class Tag {
+ public:
+ constexpr Tag(uintptr_t TagValue, uintptr_t TagOffset)
+ : TagMask(TagValue << TagOffset) {}
+
+ template <typename T> constexpr T *operator()(T *P) {
+ return reinterpret_cast<T *>(reinterpret_cast<uintptr_t>(P) | TagMask);
+ }
+
+ private:
+ uintptr_t TagMask;
+ };
+
+ /// Strips a tag of the given length from the given offset within the pointer:
+ /// P' = P & ~(((1 << TagLen) -1) << TagOffset)
+ class Untag {
+ public:
+ constexpr Untag(uintptr_t TagLen, uintptr_t TagOffset)
+ : UntagMask(~(((uintptr_t(1) << TagLen) - 1) << TagOffset)) {}
+
+ template <typename T> constexpr T *operator()(T *P) {
+ return reinterpret_cast<T *>(reinterpret_cast<uintptr_t>(P) & UntagMask);
+ }
+
+ private:
+ uintptr_t UntagMask;
+ };
+
+ ExecutorAddr() = default;
+ explicit ExecutorAddr(uint64_t Addr) : Addr(Addr) {}
+
+ /// Create an ExecutorAddr from the given pointer.
+ template <typename T, typename UnwrapFn = defaultUnwrap<T>>
+ static ExecutorAddr fromPtr(T *Ptr, UnwrapFn &&Unwrap = UnwrapFn()) {
+ return ExecutorAddr(
+ static_cast<uint64_t>(reinterpret_cast<uintptr_t>(Unwrap(Ptr))));
+ }
- /// Create an ExecutorAddress from the given pointer.
- /// Warning: This should only be used when JITing in-process.
- template <typename T> static ExecutorAddress fromPtr(T *Value) {
- return ExecutorAddress(
- static_cast<uint64_t>(reinterpret_cast<uintptr_t>(Value)));
+ /// Cast this ExecutorAddr to a pointer of the given type.
+ template <typename T, typename WrapFn = defaultWrap<std::remove_pointer_t<T>>>
+ std::enable_if_t<std::is_pointer<T>::value, T>
+ toPtr(WrapFn &&Wrap = WrapFn()) const {
+ uintptr_t IntPtr = static_cast<uintptr_t>(Addr);
+ assert(IntPtr == Addr && "ExecutorAddr value out of range for uintptr_t");
+ return Wrap(reinterpret_cast<T>(IntPtr));
}
- /// Cast this ExecutorAddress to a pointer of the given type.
- /// Warning: This should only be esude when JITing in-process.
- template <typename T> T toPtr() const {
- static_assert(std::is_pointer<T>::value, "T must be a pointer type");
+ /// Cast this ExecutorAddr to a pointer of the given function type.
+ template <typename T, typename WrapFn = defaultWrap<T>>
+ std::enable_if_t<std::is_function<T>::value, T *>
+ toPtr(WrapFn &&Wrap = WrapFn()) const {
uintptr_t IntPtr = static_cast<uintptr_t>(Addr);
- assert(IntPtr == Addr &&
- "JITTargetAddress value out of range for uintptr_t");
- return reinterpret_cast<T>(IntPtr);
+ assert(IntPtr == Addr && "ExecutorAddr value out of range for uintptr_t");
+ return Wrap(reinterpret_cast<T *>(IntPtr));
}
uint64_t getValue() const { return Addr; }
explicit operator bool() const { return Addr != 0; }
- friend bool operator==(const ExecutorAddress &LHS,
- const ExecutorAddress &RHS) {
+ friend bool operator==(const ExecutorAddr &LHS, const ExecutorAddr &RHS) {
return LHS.Addr == RHS.Addr;
}
- friend bool operator!=(const ExecutorAddress &LHS,
- const ExecutorAddress &RHS) {
+ friend bool operator!=(const ExecutorAddr &LHS, const ExecutorAddr &RHS) {
return LHS.Addr != RHS.Addr;
}
- friend bool operator<(const ExecutorAddress &LHS,
- const ExecutorAddress &RHS) {
+ friend bool operator<(const ExecutorAddr &LHS, const ExecutorAddr &RHS) {
return LHS.Addr < RHS.Addr;
}
- friend bool operator<=(const ExecutorAddress &LHS,
- const ExecutorAddress &RHS) {
+ friend bool operator<=(const ExecutorAddr &LHS, const ExecutorAddr &RHS) {
return LHS.Addr <= RHS.Addr;
}
- friend bool operator>(const ExecutorAddress &LHS,
- const ExecutorAddress &RHS) {
+ friend bool operator>(const ExecutorAddr &LHS, const ExecutorAddr &RHS) {
return LHS.Addr > RHS.Addr;
}
- friend bool operator>=(const ExecutorAddress &LHS,
- const ExecutorAddress &RHS) {
+ friend bool operator>=(const ExecutorAddr &LHS, const ExecutorAddr &RHS) {
return LHS.Addr >= RHS.Addr;
}
- ExecutorAddress &operator++() {
+ ExecutorAddr &operator++() {
++Addr;
return *this;
}
- ExecutorAddress &operator--() {
+ ExecutorAddr &operator--() {
--Addr;
return *this;
}
- ExecutorAddress operator++(int) { return ExecutorAddress(Addr++); }
- ExecutorAddress operator--(int) { return ExecutorAddress(Addr++); }
+ ExecutorAddr operator++(int) { return ExecutorAddr(Addr++); }
+ ExecutorAddr operator--(int) { return ExecutorAddr(Addr++); }
- ExecutorAddress &operator+=(const ExecutorAddrDiff Delta) {
- Addr += Delta.getValue();
+ ExecutorAddr &operator+=(const ExecutorAddrDiff Delta) {
+ Addr += Delta;
return *this;
}
- ExecutorAddress &operator-=(const ExecutorAddrDiff Delta) {
- Addr -= Delta.getValue();
+ ExecutorAddr &operator-=(const ExecutorAddrDiff Delta) {
+ Addr -= Delta;
return *this;
}
};
/// Subtracting two addresses yields an offset.
-inline ExecutorAddrDiff operator-(const ExecutorAddress &LHS,
- const ExecutorAddress &RHS) {
+inline ExecutorAddrDiff operator-(const ExecutorAddr &LHS,
+ const ExecutorAddr &RHS) {
return ExecutorAddrDiff(LHS.getValue() - RHS.getValue());
}
/// Adding an offset and an address yields an address.
-inline ExecutorAddress operator+(const ExecutorAddress &LHS,
- const ExecutorAddrDiff &RHS) {
- return ExecutorAddress(LHS.getValue() + RHS.getValue());
+inline ExecutorAddr operator+(const ExecutorAddr &LHS,
+ const ExecutorAddrDiff &RHS) {
+ return ExecutorAddr(LHS.getValue() + RHS);
}
/// Adding an address and an offset yields an address.
-inline ExecutorAddress operator+(const ExecutorAddrDiff &LHS,
- const ExecutorAddress &RHS) {
- return ExecutorAddress(LHS.getValue() + RHS.getValue());
+inline ExecutorAddr operator+(const ExecutorAddrDiff &LHS,
+ const ExecutorAddr &RHS) {
+ return ExecutorAddr(LHS + RHS.getValue());
}
/// Represents an address range in the exceutor process.
-struct ExecutorAddressRange {
- ExecutorAddressRange() = default;
- ExecutorAddressRange(ExecutorAddress StartAddress, ExecutorAddress EndAddress)
- : StartAddress(StartAddress), EndAddress(EndAddress) {}
-
- bool empty() const { return StartAddress == EndAddress; }
- ExecutorAddrDiff size() const { return EndAddress - StartAddress; }
+struct ExecutorAddrRange {
+ ExecutorAddrRange() = default;
+ ExecutorAddrRange(ExecutorAddr Start, ExecutorAddr End)
+ : Start(Start), End(End) {}
+ ExecutorAddrRange(ExecutorAddr Start, ExecutorAddrDiff Size)
+ : Start(Start), End(Start + Size) {}
+
+ bool empty() const { return Start == End; }
+ ExecutorAddrDiff size() const { return End - Start; }
+
+ friend bool operator==(const ExecutorAddrRange &LHS,
+ const ExecutorAddrRange &RHS) {
+ return LHS.Start == RHS.Start && LHS.End == RHS.End;
+ }
+ friend bool operator!=(const ExecutorAddrRange &LHS,
+ const ExecutorAddrRange &RHS) {
+ return !(LHS == RHS);
+ }
+ bool contains(ExecutorAddr Addr) const { return Start <= Addr && Addr < End; }
+ bool overlaps(const ExecutorAddrRange &Other) {
+ return !(Other.End <= Start || End <= Other.Start);
+ }
template <typename T> span<T> toSpan() const {
- assert(size().getValue() % sizeof(T) == 0 &&
+ assert(size() % sizeof(T) == 0 &&
"AddressRange is not a multiple of sizeof(T)");
- return span<T>(StartAddress.toPtr<T *>(), size().getValue() / sizeof(T));
+ return span<T>(Start.toPtr<T *>(), size() / sizeof(T));
}
- ExecutorAddress StartAddress;
- ExecutorAddress EndAddress;
+ ExecutorAddr Start;
+ ExecutorAddr End;
};
-/// SPS serializatior for ExecutorAddress.
-template <> class SPSSerializationTraits<SPSExecutorAddress, ExecutorAddress> {
+/// SPS serializatior for ExecutorAddr.
+template <> class SPSSerializationTraits<SPSExecutorAddr, ExecutorAddr> {
public:
- static size_t size(const ExecutorAddress &EA) {
+ static size_t size(const ExecutorAddr &EA) {
return SPSArgList<uint64_t>::size(EA.getValue());
}
- static bool serialize(SPSOutputBuffer &BOB, const ExecutorAddress &EA) {
+ static bool serialize(SPSOutputBuffer &BOB, const ExecutorAddr &EA) {
return SPSArgList<uint64_t>::serialize(BOB, EA.getValue());
}
- static bool deserialize(SPSInputBuffer &BIB, ExecutorAddress &EA) {
+ static bool deserialize(SPSInputBuffer &BIB, ExecutorAddr &EA) {
uint64_t Tmp;
if (!SPSArgList<uint64_t>::deserialize(BIB, Tmp))
return false;
- EA = ExecutorAddress(Tmp);
+ EA = ExecutorAddr(Tmp);
return true;
}
};
-using SPSExecutorAddressRange =
- SPSTuple<SPSExecutorAddress, SPSExecutorAddress>;
+using SPSExecutorAddrRange = SPSTuple<SPSExecutorAddr, SPSExecutorAddr>;
/// Serialization traits for address ranges.
template <>
-class SPSSerializationTraits<SPSExecutorAddressRange, ExecutorAddressRange> {
+class SPSSerializationTraits<SPSExecutorAddrRange, ExecutorAddrRange> {
public:
- static size_t size(const ExecutorAddressRange &Value) {
- return SPSArgList<SPSExecutorAddress, SPSExecutorAddress>::size(
- Value.StartAddress, Value.EndAddress);
+ static size_t size(const ExecutorAddrRange &Value) {
+ return SPSArgList<SPSExecutorAddr, SPSExecutorAddr>::size(Value.Start,
+ Value.End);
}
- static bool serialize(SPSOutputBuffer &BOB,
- const ExecutorAddressRange &Value) {
- return SPSArgList<SPSExecutorAddress, SPSExecutorAddress>::serialize(
- BOB, Value.StartAddress, Value.EndAddress);
+ static bool serialize(SPSOutputBuffer &BOB, const ExecutorAddrRange &Value) {
+ return SPSArgList<SPSExecutorAddr, SPSExecutorAddr>::serialize(
+ BOB, Value.Start, Value.End);
}
- static bool deserialize(SPSInputBuffer &BIB, ExecutorAddressRange &Value) {
- return SPSArgList<SPSExecutorAddress, SPSExecutorAddress>::deserialize(
- BIB, Value.StartAddress, Value.EndAddress);
+ static bool deserialize(SPSInputBuffer &BIB, ExecutorAddrRange &Value) {
+ return SPSArgList<SPSExecutorAddr, SPSExecutorAddr>::deserialize(
+ BIB, Value.Start, Value.End);
}
};
-using SPSExecutorAddressRangeSequence = SPSSequence<SPSExecutorAddressRange>;
+using SPSExecutorAddrRangeSequence = SPSSequence<SPSExecutorAddrRange>;
} // End namespace __orc_rt
+namespace std {
+
+// Make ExecutorAddr hashable.
+template <> struct hash<__orc_rt::ExecutorAddr> {
+ size_t operator()(const __orc_rt::ExecutorAddr &A) const {
+ return hash<uint64_t>()(A.getValue());
+ }
+};
+
+} // namespace std
+
#endif // ORC_RT_EXECUTOR_ADDRESS_H
--- /dev/null
+//===--------- interval_map.h - A sorted interval map -----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implements a coalescing interval map.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef ORC_RT_INTERVAL_MAP_H
+#define ORC_RT_INTERVAL_MAP_H
+
+#include "adt.h"
+#include <cassert>
+#include <map>
+
+namespace __orc_rt {
+
+enum class IntervalCoalescing { Enabled, Disabled };
+
+/// Maps intervals to keys with optional coalescing.
+///
+/// NOTE: The interface is kept mostly compatible with LLVM's IntervalMap
+/// collection to make it easy to swap over in the future if we choose
+/// to.
+template <typename KeyT, typename ValT> class IntervalMapBase {
+private:
+ using KeyPairT = std::pair<KeyT, KeyT>;
+
+ struct Compare {
+ using is_transparent = std::true_type;
+ bool operator()(const KeyPairT &LHS, const KeyPairT &RHS) const {
+ return LHS < RHS;
+ }
+ bool operator()(const KeyPairT &LHS, const KeyT &RHS) const {
+ return LHS.first < RHS;
+ }
+ bool operator()(const KeyT &LHS, const KeyPairT &RHS) const {
+ return LHS < RHS.first;
+ }
+ };
+
+ using ImplMap = std::map<KeyPairT, ValT, Compare>;
+
+public:
+ using iterator = typename ImplMap::iterator;
+ using const_iterator = typename ImplMap::const_iterator;
+ using size_type = typename ImplMap::size_type;
+
+ bool empty() const { return Impl.empty(); }
+
+ void clear() { Impl.clear(); }
+
+ iterator begin() { return Impl.begin(); }
+ iterator end() { return Impl.end(); }
+
+ const_iterator begin() const { return Impl.begin(); }
+ const_iterator end() const { return Impl.end(); }
+
+ iterator find(KeyT K) {
+ // Early out if the key is clearly outside the range.
+ if (empty() || K < begin()->first.first ||
+ K >= std::prev(end())->first.second)
+ return end();
+
+ auto I = Impl.upper_bound(K);
+ assert(I != begin() && "Should have hit early out above");
+ I = std::prev(I);
+ if (K < I->first.second)
+ return I;
+ return end();
+ }
+
+ const_iterator find(KeyT K) const {
+ return const_cast<IntervalMapBase<KeyT, ValT> *>(this)->find(K);
+ }
+
+ ValT lookup(KeyT K, ValT NotFound = ValT()) const {
+ auto I = find(K);
+ if (I == end())
+ return NotFound;
+ return I->second;
+ }
+
+ // Erase [KS, KE), which must be entirely containing within one existing
+ // range in the map. Removal is allowed to split the range.
+ void erase(KeyT KS, KeyT KE) {
+ if (empty())
+ return;
+
+ auto J = Impl.upper_bound(KS);
+
+ // Check previous range. Bail out if range to remove is entirely after
+ // it.
+ auto I = std::prev(J);
+ if (KS >= I->first.second)
+ return;
+
+ // Assert that range is wholly contained.
+ assert(KE <= I->first.second);
+
+ auto Tmp = std::move(*I);
+ Impl.erase(I);
+
+ // Split-right -- introduce right-split range.
+ if (KE < Tmp.first.second) {
+ Impl.insert(
+ J, std::make_pair(std::make_pair(KE, Tmp.first.second), Tmp.second));
+ J = std::prev(J);
+ }
+
+ // Split-left -- introduce left-split range.
+ if (KS > Tmp.first.first)
+ Impl.insert(
+ J, std::make_pair(std::make_pair(Tmp.first.first, KS), Tmp.second));
+ }
+
+protected:
+ ImplMap Impl;
+};
+
+template <typename KeyT, typename ValT, IntervalCoalescing Coalescing>
+class IntervalMap;
+
+template <typename KeyT, typename ValT>
+class IntervalMap<KeyT, ValT, IntervalCoalescing::Enabled>
+ : public IntervalMapBase<KeyT, ValT> {
+public:
+ // Coalescing insert. Requires that ValTs be equality-comparable.
+ void insert(KeyT KS, KeyT KE, ValT V) {
+ auto J = this->Impl.upper_bound(KS);
+
+ // Coalesce-right if possible. Either way, J points at our insertion
+ // point.
+ if (J != this->end() && KE == J->first.first && J->second == V) {
+ KE = J->first.second;
+ auto Tmp = J++;
+ this->Impl.erase(Tmp);
+ }
+
+ // Coalesce-left if possible.
+ if (J != this->begin()) {
+ auto I = std::prev(J);
+ if (I->first.second == KS && I->second == V) {
+ KS = I->first.first;
+ this->Impl.erase(I);
+ }
+ }
+ this->Impl.insert(J, std::make_pair(std::make_pair(KS, KE), std::move(V)));
+ }
+};
+
+template <typename KeyT, typename ValT>
+class IntervalMap<KeyT, ValT, IntervalCoalescing::Disabled>
+ : public IntervalMapBase<KeyT, ValT> {
+public:
+ // Non-coalescing insert. Does not require ValT to be equality-comparable.
+ void insert(KeyT KS, KeyT KE, ValT V) {
+ this->Impl.insert(std::make_pair(std::make_pair(KS, KE), std::move(V)));
+ }
+};
+
+} // End namespace __orc_rt
+
+#endif // ORC_RT_INTERVAL_MAP_H
--- /dev/null
+//===--------- interval_set.h - A sorted interval set -----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implements a coalescing interval set.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef ORC_RT_INTERVAL_SET_H
+#define ORC_RT_INTERVAL_SET_H
+
+#include "interval_map.h"
+
+namespace __orc_rt {
+
+/// Implements a coalescing interval set.
+///
+/// Adjacent intervals are coalesced.
+///
+/// NOTE: The interface is kept mostly compatible with LLVM's IntervalMap
+/// collection to make it easy to swap over in the future if we choose
+/// to.
+template <typename KeyT, IntervalCoalescing Coalescing>
+class IntervalSet {
+private:
+ using ImplMap = IntervalMap<KeyT, std::monostate, Coalescing>;
+public:
+
+ using value_type = std::pair<KeyT, KeyT>;
+
+ class const_iterator {
+ friend class IntervalSet;
+ public:
+ using difference_type = typename ImplMap::iterator::difference_type;
+ using value_type = IntervalSet::value_type;
+ using pointer = const value_type *;
+ using reference = const value_type &;
+ using iterator_category = std::input_iterator_tag;
+
+ const_iterator() = default;
+ const value_type &operator*() const { return I->first; }
+ const value_type *operator->() const { return &I->first; }
+ const_iterator &operator++() { ++I; return *this; }
+ const_iterator operator++(int) { auto Tmp = I; ++I; return Tmp; }
+ friend bool operator==(const const_iterator &LHS,
+ const const_iterator &RHS) {
+ return LHS.I == RHS.I;
+ }
+ friend bool operator!=(const const_iterator &LHS,
+ const const_iterator &RHS) {
+ return LHS.I != RHS.I;
+ }
+ private:
+ const_iterator(typename ImplMap::const_iterator I) : I(std::move(I)) {}
+ typename ImplMap::const_iterator I;
+ };
+
+ bool empty() const { return Map.empty(); }
+
+ void clear() { Map.clear(); }
+
+ const_iterator begin() const { return const_iterator(Map.begin()); }
+ const_iterator end() const { return const_iterator(Map.end()); }
+
+ const_iterator find(KeyT K) const {
+ return const_iterator(Map.find(K));
+ }
+
+ void insert(KeyT KS, KeyT KE) {
+ Map.insert(std::move(KS), std::move(KE), std::monostate());
+ }
+
+ void erase(KeyT KS, KeyT KE) {
+ Map.erase(KS, KE);
+ }
+
+private:
+ ImplMap Map;
+};
+
+} // End namespace __orc_rt
+
+#endif // ORC_RT_INTERVAL_SET_H
#include "macho_platform.h"
#include "common.h"
+#include "debug.h"
#include "error.h"
+#include "interval_map.h"
#include "wrapper_function_utils.h"
+#include <algorithm>
+#include <ios>
#include <map>
#include <mutex>
#include <sstream>
+#include <string_view>
#include <unordered_map>
+#include <unordered_set>
#include <vector>
+#define DEBUG_TYPE "macho_platform"
+
using namespace __orc_rt;
using namespace __orc_rt::macho;
// Declare function tags for functions in the JIT process.
-ORC_RT_JIT_DISPATCH_TAG(__orc_rt_macho_get_initializers_tag)
-ORC_RT_JIT_DISPATCH_TAG(__orc_rt_macho_get_deinitializers_tag)
+ORC_RT_JIT_DISPATCH_TAG(__orc_rt_macho_push_initializers_tag)
ORC_RT_JIT_DISPATCH_TAG(__orc_rt_macho_symbol_lookup_tag)
-// eh-frame registration functions.
-// We expect these to be available for all processes.
-extern "C" void __register_frame(const void *);
-extern "C" void __deregister_frame(const void *);
-
// Objective-C types.
struct objc_class;
struct objc_image_info;
// Swift types.
class ProtocolRecord;
class ProtocolConformanceRecord;
+class TypeMetadataRecord;
extern "C" void
swift_registerProtocols(const ProtocolRecord *begin,
const ProtocolConformanceRecord *begin,
const ProtocolConformanceRecord *end) ORC_RT_WEAK_IMPORT;
-namespace {
-
-template <typename HandleFDEFn>
-void walkEHFrameSection(span<const char> EHFrameSection,
- HandleFDEFn HandleFDE) {
- const char *CurCFIRecord = EHFrameSection.data();
- uint64_t Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord);
-
- while (CurCFIRecord != EHFrameSection.end() && Size != 0) {
- const char *OffsetField = CurCFIRecord + (Size == 0xffffffff ? 12 : 4);
- if (Size == 0xffffffff)
- Size = *reinterpret_cast<const uint64_t *>(CurCFIRecord + 4) + 12;
- else
- Size += 4;
- uint32_t Offset = *reinterpret_cast<const uint32_t *>(OffsetField);
-
- if (Offset != 0)
- HandleFDE(CurCFIRecord);
-
- CurCFIRecord += Size;
- Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord);
- }
-}
-
-Error validatePointerSectionExtent(const char *SectionName,
- const ExecutorAddressRange &SE) {
- if (SE.size().getValue() % sizeof(uintptr_t)) {
- std::ostringstream ErrMsg;
- ErrMsg << std::hex << "Size of " << SectionName << " 0x"
- << SE.StartAddress.getValue() << " -- 0x" << SE.EndAddress.getValue()
- << " is not a pointer multiple";
- return make_error<StringError>(ErrMsg.str());
- }
- return Error::success();
-}
-
-Error registerObjCSelectors(
- const std::vector<ExecutorAddressRange> &ObjCSelRefsSections,
- const MachOJITDylibInitializers &MOJDIs) {
-
- if (ORC_RT_UNLIKELY(!sel_registerName))
- return make_error<StringError>("sel_registerName is not available");
+extern "C" void swift_registerTypeMetadataRecords(
+ const TypeMetadataRecord *begin,
+ const TypeMetadataRecord *end) ORC_RT_WEAK_IMPORT;
+
+// Libunwind prototypes.
+struct unw_dynamic_unwind_sections {
+ uintptr_t dso_base;
+ uintptr_t dwarf_section;
+ size_t dwarf_section_length;
+ uintptr_t compact_unwind_section;
+ size_t compact_unwind_section_length;
+};
- for (const auto &ObjCSelRefs : ObjCSelRefsSections) {
+typedef int (*unw_find_dynamic_unwind_sections)(
+ uintptr_t addr, struct unw_dynamic_unwind_sections *info);
- if (auto Err = validatePointerSectionExtent("__objc_selrefs", ObjCSelRefs))
- return Err;
-
- fprintf(stderr, "Processing selrefs section at 0x%llx\n",
- ObjCSelRefs.StartAddress.getValue());
- for (uintptr_t SelEntry : ObjCSelRefs.toSpan<uintptr_t>()) {
- const char *SelName = reinterpret_cast<const char *>(SelEntry);
- fprintf(stderr, "Registering selector \"%s\"\n", SelName);
- auto Sel = sel_registerName(SelName);
- *reinterpret_cast<SEL *>(SelEntry) = Sel;
- }
- }
+extern "C" int __unw_add_find_dynamic_unwind_sections(
+ unw_find_dynamic_unwind_sections find_dynamic_unwind_sections)
+ ORC_RT_WEAK_IMPORT;
- return Error::success();
-}
+extern "C" int __unw_remove_find_dynamic_unwind_sections(
+ unw_find_dynamic_unwind_sections find_dynamic_unwind_sections)
+ ORC_RT_WEAK_IMPORT;
-Error registerObjCClasses(
- const std::vector<ExecutorAddressRange> &ObjCClassListSections,
- const MachOJITDylibInitializers &MOJDIs) {
+namespace {
- if (ObjCClassListSections.empty())
- return Error::success();
+struct MachOJITDylibDepInfo {
+ bool Sealed = false;
+ std::vector<ExecutorAddr> DepHeaders;
+};
- if (ORC_RT_UNLIKELY(!objc_msgSend))
- return make_error<StringError>("objc_msgSend is not available");
- if (ORC_RT_UNLIKELY(!objc_readClassPair))
- return make_error<StringError>("objc_readClassPair is not available");
+using MachOJITDylibDepInfoMap =
+ std::unordered_map<ExecutorAddr, MachOJITDylibDepInfo>;
- struct ObjCClassCompiled {
- void *Metaclass;
- void *Parent;
- void *Cache1;
- void *Cache2;
- void *Data;
- };
+} // anonymous namespace
- auto *ImageInfo =
- MOJDIs.ObjCImageInfoAddress.toPtr<const objc_image_info *>();
- auto ClassSelector = sel_registerName("class");
+namespace __orc_rt {
- for (const auto &ObjCClassList : ObjCClassListSections) {
+using SPSMachOObjectPlatformSectionsMap =
+ SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRange>>;
- if (auto Err =
- validatePointerSectionExtent("__objc_classlist", ObjCClassList))
- return Err;
+using SPSMachOJITDylibDepInfo = SPSTuple<bool, SPSSequence<SPSExecutorAddr>>;
- for (uintptr_t ClassPtr : ObjCClassList.toSpan<uintptr_t>()) {
- auto *Cls = reinterpret_cast<Class>(ClassPtr);
- auto *ClassCompiled = reinterpret_cast<ObjCClassCompiled *>(ClassPtr);
- objc_msgSend(reinterpret_cast<id>(ClassCompiled->Parent), ClassSelector);
- auto Registered = objc_readClassPair(Cls, ImageInfo);
+using SPSMachOJITDylibDepInfoMap =
+ SPSSequence<SPSTuple<SPSExecutorAddr, SPSMachOJITDylibDepInfo>>;
- // FIXME: Improve diagnostic by reporting the failed class's name.
- if (Registered != Cls)
- return make_error<StringError>("Unable to register Objective-C class");
- }
+template <>
+class SPSSerializationTraits<SPSMachOJITDylibDepInfo, MachOJITDylibDepInfo> {
+public:
+ static size_t size(const MachOJITDylibDepInfo &JDI) {
+ return SPSMachOJITDylibDepInfo::AsArgList::size(JDI.Sealed, JDI.DepHeaders);
}
- return Error::success();
-}
-Error registerSwift5Protocols(
- const std::vector<ExecutorAddressRange> &Swift5ProtocolSections,
- const MachOJITDylibInitializers &MOJDIs) {
-
- if (ORC_RT_UNLIKELY(!Swift5ProtocolSections.empty() &&
- !swift_registerProtocols))
- return make_error<StringError>("swift_registerProtocols is not available");
-
- for (const auto &Swift5Protocols : Swift5ProtocolSections)
- swift_registerProtocols(
- Swift5Protocols.StartAddress.toPtr<const ProtocolRecord *>(),
- Swift5Protocols.EndAddress.toPtr<const ProtocolRecord *>());
-
- return Error::success();
-}
-
-Error registerSwift5ProtocolConformances(
- const std::vector<ExecutorAddressRange> &Swift5ProtocolConformanceSections,
- const MachOJITDylibInitializers &MOJDIs) {
+ static bool serialize(SPSOutputBuffer &OB, const MachOJITDylibDepInfo &JDI) {
+ return SPSMachOJITDylibDepInfo::AsArgList::serialize(OB, JDI.Sealed,
+ JDI.DepHeaders);
+ }
- if (ORC_RT_UNLIKELY(!Swift5ProtocolConformanceSections.empty() &&
- !swift_registerProtocolConformances))
- return make_error<StringError>(
- "swift_registerProtocolConformances is not available");
+ static bool deserialize(SPSInputBuffer &IB, MachOJITDylibDepInfo &JDI) {
+ return SPSMachOJITDylibDepInfo::AsArgList::deserialize(IB, JDI.Sealed,
+ JDI.DepHeaders);
+ }
+};
- for (const auto &ProtoConfSec : Swift5ProtocolConformanceSections)
- swift_registerProtocolConformances(
- ProtoConfSec.StartAddress.toPtr<const ProtocolConformanceRecord *>(),
- ProtoConfSec.EndAddress.toPtr<const ProtocolConformanceRecord *>());
+struct UnwindSectionInfo {
+ std::vector<ExecutorAddrRange> CodeRanges;
+ ExecutorAddrRange DwarfSection;
+ ExecutorAddrRange CompactUnwindSection;
+};
- return Error::success();
-}
+using SPSUnwindSectionInfo =
+ SPSTuple<SPSSequence<SPSExecutorAddrRange>, SPSExecutorAddrRange,
+ SPSExecutorAddrRange>;
-Error runModInits(const std::vector<ExecutorAddressRange> &ModInitsSections,
- const MachOJITDylibInitializers &MOJDIs) {
+template <>
+class SPSSerializationTraits<SPSUnwindSectionInfo, UnwindSectionInfo> {
+public:
+ static size_t size(const UnwindSectionInfo &USI) {
+ return SPSUnwindSectionInfo::AsArgList::size(
+ USI.CodeRanges, USI.DwarfSection, USI.CompactUnwindSection);
+ }
- for (const auto &ModInits : ModInitsSections) {
- if (auto Err = validatePointerSectionExtent("__mod_inits", ModInits))
- return Err;
+ static bool serialize(SPSOutputBuffer &OB, const UnwindSectionInfo &USI) {
+ return SPSUnwindSectionInfo::AsArgList::serialize(
+ OB, USI.CodeRanges, USI.DwarfSection, USI.CompactUnwindSection);
+ }
- using InitFunc = void (*)();
- for (auto *Init : ModInits.toSpan<InitFunc>())
- (*Init)();
+ static bool deserialize(SPSInputBuffer &IB, UnwindSectionInfo &USI) {
+ return SPSUnwindSectionInfo::AsArgList::deserialize(
+ IB, USI.CodeRanges, USI.DwarfSection, USI.CompactUnwindSection);
}
+};
- return Error::success();
-}
+} // namespace __orc_rt
+namespace {
struct TLVDescriptor {
void *(*Thunk)(TLVDescriptor *) = nullptr;
unsigned long Key = 0;
using AtExitsVector = std::vector<AtExitEntry>;
- struct PerJITDylibState {
+ /// Used to manage sections of fixed-sized metadata records (e.g. pointer
+ /// sections, selector refs, etc.)
+ template <typename RecordElement> class RecordSectionsTracker {
+ public:
+ /// Add a section to the "new" list.
+ void add(span<RecordElement> Sec) { New.push_back(std::move(Sec)); }
+
+ /// Returns true if there are new sections to process.
+ bool hasNewSections() const { return !New.empty(); }
+
+ /// Returns the number of new sections to process.
+ size_t numNewSections() const { return New.size(); }
+
+ /// Process all new sections.
+ template <typename ProcessSectionFunc>
+ std::enable_if_t<std::is_void_v<
+ std::invoke_result_t<ProcessSectionFunc, span<RecordElement>>>>
+ processNewSections(ProcessSectionFunc &&ProcessSection) {
+ for (auto &Sec : New)
+ ProcessSection(Sec);
+ moveNewToProcessed();
+ }
+
+ /// Proces all new sections with a fallible handler.
+ ///
+ /// Successfully handled sections will be moved to the Processed
+ /// list.
+ template <typename ProcessSectionFunc>
+ std::enable_if_t<
+ std::is_same_v<Error, std::invoke_result_t<ProcessSectionFunc,
+ span<RecordElement>>>,
+ Error>
+ processNewSections(ProcessSectionFunc &&ProcessSection) {
+ for (size_t I = 0; I != New.size(); ++I) {
+ if (auto Err = ProcessSection(New[I])) {
+ for (size_t J = 0; J != I; ++J)
+ Processed.push_back(New[J]);
+ New.erase(New.begin(), New.begin() + I);
+ return Err;
+ }
+ }
+ moveNewToProcessed();
+ return Error::success();
+ }
+
+ /// Move all sections back to New for reprocessing.
+ void reset() {
+ moveNewToProcessed();
+ New = std::move(Processed);
+ }
+
+ /// Remove the section with the given range.
+ bool removeIfPresent(ExecutorAddrRange R) {
+ if (removeIfPresent(New, R))
+ return true;
+ return removeIfPresent(Processed, R);
+ }
+
+ private:
+ void moveNewToProcessed() {
+ if (Processed.empty())
+ Processed = std::move(New);
+ else {
+ Processed.reserve(Processed.size() + New.size());
+ std::copy(New.begin(), New.end(), std::back_inserter(Processed));
+ New.clear();
+ }
+ }
+
+ bool removeIfPresent(std::vector<span<RecordElement>> &V,
+ ExecutorAddrRange R) {
+ auto RI = std::find_if(
+ V.rbegin(), V.rend(),
+ [RS = R.toSpan<RecordElement>()](const span<RecordElement> &E) {
+ return E.data() == RS.data();
+ });
+ if (RI != V.rend()) {
+ V.erase(std::next(RI).base());
+ return true;
+ }
+ return false;
+ }
+
+ std::vector<span<RecordElement>> Processed;
+ std::vector<span<RecordElement>> New;
+ };
+
+ struct UnwindSections {
+ UnwindSections(const UnwindSectionInfo &USI)
+ : DwarfSection(USI.DwarfSection.toSpan<char>()),
+ CompactUnwindSection(USI.CompactUnwindSection.toSpan<char>()) {}
+
+ span<char> DwarfSection;
+ span<char> CompactUnwindSection;
+ };
+
+ using UnwindSectionsMap =
+ IntervalMap<char *, UnwindSections, IntervalCoalescing::Disabled>;
+
+ struct JITDylibState {
+ std::string Name;
void *Header = nullptr;
- size_t RefCount = 0;
- bool AllowReinitialization = false;
+ bool Sealed = false;
+ size_t LinkedAgainstRefCount = 0;
+ size_t DlRefCount = 0;
+ std::vector<JITDylibState *> Deps;
AtExitsVector AtExits;
+ const objc_image_info *ObjCImageInfo = nullptr;
+ std::unordered_map<void *, std::vector<char>> DataSectionContent;
+ std::unordered_map<void *, size_t> ZeroInitRanges;
+ UnwindSectionsMap UnwindSections;
+ RecordSectionsTracker<void (*)()> ModInitsSections;
+ RecordSectionsTracker<void *> ObjCClassListSections;
+ RecordSectionsTracker<void *> ObjCSelRefsSections;
+ RecordSectionsTracker<char> Swift5ProtocolsSections;
+ RecordSectionsTracker<char> Swift5ProtocolConformancesSections;
+ RecordSectionsTracker<char> Swift5TypesSections;
+
+ bool referenced() const {
+ return LinkedAgainstRefCount != 0 || DlRefCount != 0;
+ }
};
public:
- static void initialize();
+ static Error create();
static MachOPlatformRuntimeState &get();
- static void destroy();
+ static Error destroy();
MachOPlatformRuntimeState() = default;
MachOPlatformRuntimeState(MachOPlatformRuntimeState &&) = delete;
MachOPlatformRuntimeState &operator=(MachOPlatformRuntimeState &&) = delete;
- Error registerObjectSections(MachOPerObjectSectionsToRegister POSR);
- Error deregisterObjectSections(MachOPerObjectSectionsToRegister POSR);
+ Error initialize();
+ Error shutdown();
+
+ Error registerJITDylib(std::string Name, void *Header);
+ Error deregisterJITDylib(void *Header);
+ Error registerThreadDataSection(span<const char> ThreadDataSection);
+ Error deregisterThreadDataSection(span<const char> ThreadDataSection);
+ Error registerObjectPlatformSections(
+ ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> UnwindSections,
+ std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs);
+ Error deregisterObjectPlatformSections(
+ ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> UnwindSections,
+ std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs);
const char *dlerror();
- void *dlopen(string_view Name, int Mode);
+ void *dlopen(std::string_view Name, int Mode);
int dlclose(void *DSOHandle);
- void *dlsym(void *DSOHandle, string_view Symbol);
+ void *dlsym(void *DSOHandle, std::string_view Symbol);
int registerAtExit(void (*F)(void *), void *Arg, void *DSOHandle);
+ void runAtExits(std::unique_lock<std::mutex> &JDStatesLock,
+ JITDylibState &JDS);
void runAtExits(void *DSOHandle);
/// Returns the base address of the section containing ThreadData.
getThreadDataSectionFor(const char *ThreadData);
private:
- PerJITDylibState *getJITDylibStateByHeaderAddr(void *DSOHandle);
- PerJITDylibState *getJITDylibStateByName(string_view Path);
- PerJITDylibState &getOrCreateJITDylibState(MachOJITDylibInitializers &MOJDIs);
+ JITDylibState *getJITDylibStateByHeader(void *DSOHandle);
+ JITDylibState *getJITDylibStateByName(std::string_view Path);
- Error registerThreadDataSection(span<const char> ThreadDataSec);
+ Expected<ExecutorAddr> lookupSymbolInJITDylib(void *DSOHandle,
+ std::string_view Symbol);
- Expected<ExecutorAddress> lookupSymbolInJITDylib(void *DSOHandle,
- string_view Symbol);
+ bool lookupUnwindSections(void *Addr, unw_dynamic_unwind_sections &Info);
- Expected<MachOJITDylibInitializerSequence>
- getJITDylibInitializersByName(string_view Path);
- Expected<void *> dlopenInitialize(string_view Path, int Mode);
- Error initializeJITDylib(MachOJITDylibInitializers &MOJDIs);
+ static int findDynamicUnwindSections(uintptr_t addr,
+ unw_dynamic_unwind_sections *info);
+ static Error registerEHFrames(span<const char> EHFrameSection);
+ static Error deregisterEHFrames(span<const char> EHFrameSection);
+
+ static Error registerObjCSelectors(JITDylibState &JDS);
+ static Error registerObjCClasses(JITDylibState &JDS);
+ static Error registerSwift5Protocols(JITDylibState &JDS);
+ static Error registerSwift5ProtocolConformances(JITDylibState &JDS);
+ static Error registerSwift5Types(JITDylibState &JDS);
+ static Error runModInits(std::unique_lock<std::mutex> &JDStatesLock,
+ JITDylibState &JDS);
+
+ Expected<void *> dlopenImpl(std::string_view Path, int Mode);
+ Error dlopenFull(std::unique_lock<std::mutex> &JDStatesLock,
+ JITDylibState &JDS);
+ Error dlopenInitialize(std::unique_lock<std::mutex> &JDStatesLock,
+ JITDylibState &JDS, MachOJITDylibDepInfoMap &DepInfo);
+
+ Error dlcloseImpl(void *DSOHandle);
+ Error dlcloseDeinitialize(std::unique_lock<std::mutex> &JDStatesLock,
+ JITDylibState &JDS);
static MachOPlatformRuntimeState *MOPS;
- using InitSectionHandler =
- Error (*)(const std::vector<ExecutorAddressRange> &Sections,
- const MachOJITDylibInitializers &MOJDIs);
- const std::vector<std::pair<const char *, InitSectionHandler>> InitSections =
- {{"__DATA,__objc_selrefs", registerObjCSelectors},
- {"__DATA,__objc_classlist", registerObjCClasses},
- {"__TEXT,__swift5_protos", registerSwift5Protocols},
- {"__TEXT,__swift5_proto", registerSwift5ProtocolConformances},
- {"__DATA,__mod_init_func", runModInits}};
+ bool UseCallbackStyleUnwindInfo = false;
// FIXME: Move to thread-state.
std::string DLFcnError;
- std::recursive_mutex JDStatesMutex;
- std::unordered_map<void *, PerJITDylibState> JDStates;
- std::unordered_map<std::string, void *> JDNameToHeader;
+ // APIMutex guards against concurrent entry into key "dyld" API functions
+ // (e.g. dlopen, dlclose).
+ std::recursive_mutex DyldAPIMutex;
+ // JDStatesMutex guards the data structures that hold JITDylib state.
+ std::mutex JDStatesMutex;
+ std::unordered_map<void *, JITDylibState> JDStates;
+ std::unordered_map<std::string_view, void *> JDNameToHeader;
+
+ // ThreadDataSectionsMutex guards thread local data section state.
std::mutex ThreadDataSectionsMutex;
std::map<const char *, size_t> ThreadDataSections;
};
MachOPlatformRuntimeState *MachOPlatformRuntimeState::MOPS = nullptr;
-void MachOPlatformRuntimeState::initialize() {
+Error MachOPlatformRuntimeState::create() {
assert(!MOPS && "MachOPlatformRuntimeState should be null");
MOPS = new MachOPlatformRuntimeState();
+ return MOPS->initialize();
}
MachOPlatformRuntimeState &MachOPlatformRuntimeState::get() {
return *MOPS;
}
-void MachOPlatformRuntimeState::destroy() {
+Error MachOPlatformRuntimeState::destroy() {
assert(MOPS && "MachOPlatformRuntimeState not initialized");
+ auto Err = MOPS->shutdown();
delete MOPS;
+ return Err;
}
-Error MachOPlatformRuntimeState::registerObjectSections(
- MachOPerObjectSectionsToRegister POSR) {
- if (POSR.EHFrameSection.StartAddress)
- walkEHFrameSection(POSR.EHFrameSection.toSpan<const char>(),
- __register_frame);
+Error MachOPlatformRuntimeState::initialize() {
+ UseCallbackStyleUnwindInfo = __unw_add_find_dynamic_unwind_sections &&
+ __unw_remove_find_dynamic_unwind_sections;
+ if (UseCallbackStyleUnwindInfo) {
+ ORC_RT_DEBUG({
+ printdbg("__unw_add/remove_find_dynamic_unwind_sections available."
+ " Using callback-based frame info lookup.\n");
+ });
+ if (__unw_add_find_dynamic_unwind_sections(&findDynamicUnwindSections))
+ return make_error<StringError>(
+ "Could not register findDynamicUnwindSections");
+ } else {
+ ORC_RT_DEBUG({
+ printdbg("__unw_add/remove_find_dynamic_unwind_sections not available."
+ " Using classic frame info registration.\n");
+ });
+ }
+ return Error::success();
+}
- if (POSR.ThreadDataSection.StartAddress) {
- if (auto Err = registerThreadDataSection(
- POSR.ThreadDataSection.toSpan<const char>()))
- return Err;
+Error MachOPlatformRuntimeState::shutdown() {
+ if (__unw_add_find_dynamic_unwind_sections &&
+ __unw_remove_find_dynamic_unwind_sections) {
+ if (__unw_remove_find_dynamic_unwind_sections(&findDynamicUnwindSections)) {
+ ORC_RT_DEBUG(
+ { printdbg("__unw_remove_find_dynamic_unwind_sections failed.\n"); });
+ }
+ }
+ return Error::success();
+}
+
+Error MachOPlatformRuntimeState::registerJITDylib(std::string Name,
+ void *Header) {
+ ORC_RT_DEBUG({
+ printdbg("Registering JITDylib %s: Header = %p\n", Name.c_str(), Header);
+ });
+ std::lock_guard<std::mutex> Lock(JDStatesMutex);
+ if (JDStates.count(Header)) {
+ std::ostringstream ErrStream;
+ ErrStream << "Duplicate JITDylib registration for header " << Header
+ << " (name = " << Name << ")";
+ return make_error<StringError>(ErrStream.str());
+ }
+ if (JDNameToHeader.count(Name)) {
+ std::ostringstream ErrStream;
+ ErrStream << "Duplicate JITDylib registration for header " << Header
+ << " (header = " << Header << ")";
+ return make_error<StringError>(ErrStream.str());
}
+ auto &JDS = JDStates[Header];
+ JDS.Name = std::move(Name);
+ JDS.Header = Header;
+ JDNameToHeader[JDS.Name] = Header;
return Error::success();
}
-Error MachOPlatformRuntimeState::deregisterObjectSections(
- MachOPerObjectSectionsToRegister POSR) {
- if (POSR.EHFrameSection.StartAddress)
- walkEHFrameSection(POSR.EHFrameSection.toSpan<const char>(),
- __deregister_frame);
+Error MachOPlatformRuntimeState::deregisterJITDylib(void *Header) {
+ std::lock_guard<std::mutex> Lock(JDStatesMutex);
+ auto I = JDStates.find(Header);
+ if (I == JDStates.end()) {
+ std::ostringstream ErrStream;
+ ErrStream << "Attempted to deregister unrecognized header " << Header;
+ return make_error<StringError>(ErrStream.str());
+ }
+
+ // Remove std::string construction once we can use C++20.
+ auto J = JDNameToHeader.find(
+ std::string(I->second.Name.data(), I->second.Name.size()));
+ assert(J != JDNameToHeader.end() &&
+ "Missing JDNameToHeader entry for JITDylib");
+ ORC_RT_DEBUG({
+ printdbg("Deregistering JITDylib %s: Header = %p\n", I->second.Name.c_str(),
+ Header);
+ });
+
+ JDNameToHeader.erase(J);
+ JDStates.erase(I);
return Error::success();
}
-const char *MachOPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); }
+Error MachOPlatformRuntimeState::registerThreadDataSection(
+ span<const char> ThreadDataSection) {
+ std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
+ auto I = ThreadDataSections.upper_bound(ThreadDataSection.data());
+ if (I != ThreadDataSections.begin()) {
+ auto J = std::prev(I);
+ if (J->first + J->second > ThreadDataSection.data())
+ return make_error<StringError>("Overlapping __thread_data sections");
+ }
+ ThreadDataSections.insert(
+ I, std::make_pair(ThreadDataSection.data(), ThreadDataSection.size()));
+ return Error::success();
+}
+
+Error MachOPlatformRuntimeState::deregisterThreadDataSection(
+ span<const char> ThreadDataSection) {
+ std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
+ auto I = ThreadDataSections.find(ThreadDataSection.data());
+ if (I == ThreadDataSections.end())
+ return make_error<StringError>("Attempt to deregister unknown thread data "
+ "section");
+ ThreadDataSections.erase(I);
+ return Error::success();
+}
-void *MachOPlatformRuntimeState::dlopen(string_view Path, int Mode) {
- std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
+Error MachOPlatformRuntimeState::registerObjectPlatformSections(
+ ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> UnwindInfo,
+ std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs) {
+
+ // FIXME: Reject platform section registration after the JITDylib is
+ // sealed?
+
+ ORC_RT_DEBUG({
+ printdbg("MachOPlatform: Registering object sections for %p.\n",
+ HeaderAddr.toPtr<void *>());
+ });
+
+ std::lock_guard<std::mutex> Lock(JDStatesMutex);
+ auto *JDS = getJITDylibStateByHeader(HeaderAddr.toPtr<void *>());
+ if (!JDS) {
+ std::ostringstream ErrStream;
+ ErrStream << "Could not register object platform sections for "
+ "unrecognized header "
+ << HeaderAddr.toPtr<void *>();
+ return make_error<StringError>(ErrStream.str());
+ }
- // Use fast path if all JITDylibs are already loaded and don't require
- // re-running initializers.
- if (auto *JDS = getJITDylibStateByName(Path)) {
- if (!JDS->AllowReinitialization) {
- ++JDS->RefCount;
- return JDS->Header;
+ if (UnwindInfo && UseCallbackStyleUnwindInfo) {
+ ORC_RT_DEBUG({
+ printdbg(" Registering new-style unwind info for:\n"
+ " DWARF: %p -- %p\n"
+ " Compact-unwind: %p -- %p\n"
+ " for:\n",
+ UnwindInfo->DwarfSection.Start.toPtr<void *>(),
+ UnwindInfo->DwarfSection.End.toPtr<void *>(),
+ UnwindInfo->CompactUnwindSection.Start.toPtr<void *>(),
+ UnwindInfo->CompactUnwindSection.End.toPtr<void *>());
+ });
+ for (auto &CodeRange : UnwindInfo->CodeRanges) {
+ JDS->UnwindSections.insert(CodeRange.Start.toPtr<char *>(),
+ CodeRange.End.toPtr<char *>(), *UnwindInfo);
+ ORC_RT_DEBUG({
+ printdbg(" [ %p -- %p ]\n", CodeRange.Start.toPtr<void *>(),
+ CodeRange.End.toPtr<void *>());
+ });
}
}
- auto H = dlopenInitialize(Path, Mode);
- if (!H) {
+ for (auto &KV : Secs) {
+ // FIXME: Validate section ranges?
+ if (KV.first == "__TEXT,__eh_frame") {
+ if (!UseCallbackStyleUnwindInfo) {
+ // Use classic libunwind registration.
+ if (auto Err = registerEHFrames(KV.second.toSpan<const char>()))
+ return Err;
+ }
+ } else if (KV.first == "__DATA,__data") {
+ assert(!JDS->DataSectionContent.count(KV.second.Start.toPtr<char *>()) &&
+ "Address already registered.");
+ auto S = KV.second.toSpan<char>();
+ JDS->DataSectionContent[KV.second.Start.toPtr<char *>()] =
+ std::vector<char>(S.begin(), S.end());
+ } else if (KV.first == "__DATA,__common") {
+ // fprintf(stderr, "Adding zero-init range %llx -- %llx\n",
+ // KV.second.Start.getValue(), KV.second.size());
+ JDS->ZeroInitRanges[KV.second.Start.toPtr<char *>()] = KV.second.size();
+ } else if (KV.first == "__DATA,__thread_data") {
+ if (auto Err = registerThreadDataSection(KV.second.toSpan<const char>()))
+ return Err;
+ } else if (KV.first == "__DATA,__objc_selrefs")
+ JDS->ObjCSelRefsSections.add(KV.second.toSpan<void *>());
+ else if (KV.first == "__DATA,__objc_classlist")
+ JDS->ObjCClassListSections.add(KV.second.toSpan<void *>());
+ else if (KV.first == "__TEXT,__swift5_protos")
+ JDS->Swift5ProtocolsSections.add(KV.second.toSpan<char>());
+ else if (KV.first == "__TEXT,__swift5_proto")
+ JDS->Swift5ProtocolConformancesSections.add(KV.second.toSpan<char>());
+ else if (KV.first == "__TEXT,__swift5_types")
+ JDS->Swift5TypesSections.add(KV.second.toSpan<char>());
+ else if (KV.first == "__DATA,__mod_init_func")
+ JDS->ModInitsSections.add(KV.second.toSpan<void (*)()>());
+ else {
+ // Should this be a warning instead?
+ return make_error<StringError>(
+ "Encountered unexpected section " +
+ std::string(KV.first.data(), KV.first.size()) +
+ " while registering object platform sections");
+ }
+ }
+
+ return Error::success();
+}
+
+Error MachOPlatformRuntimeState::deregisterObjectPlatformSections(
+ ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> UnwindInfo,
+ std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs) {
+ // TODO: Make this more efficient? (maybe unnecessary if removal is rare?)
+ // TODO: Add a JITDylib prepare-for-teardown operation that clears all
+ // registered sections, causing this function to take the fast-path.
+ ORC_RT_DEBUG({
+ printdbg("MachOPlatform: Registering object sections for %p.\n",
+ HeaderAddr.toPtr<void *>());
+ });
+
+ std::lock_guard<std::mutex> Lock(JDStatesMutex);
+ auto *JDS = getJITDylibStateByHeader(HeaderAddr.toPtr<void *>());
+ if (!JDS) {
+ std::ostringstream ErrStream;
+ ErrStream << "Could not register object platform sections for unrecognized "
+ "header "
+ << HeaderAddr.toPtr<void *>();
+ return make_error<StringError>(ErrStream.str());
+ }
+
+ // FIXME: Implement faster-path by returning immediately if JDS is being
+ // torn down entirely?
+
+ // TODO: Make library permanent (i.e. not able to be dlclosed) if it contains
+ // any Swift or ObjC. Once this happens we can clear (and no longer record)
+ // data section content, as the library could never be re-initialized.
+
+ if (UnwindInfo && UseCallbackStyleUnwindInfo) {
+ ORC_RT_DEBUG({
+ printdbg(" Deregistering new-style unwind info for:\n"
+ " DWARF: %p -- %p\n"
+ " Compact-unwind: %p -- %p\n"
+ " for:\n",
+ UnwindInfo->DwarfSection.Start.toPtr<void *>(),
+ UnwindInfo->DwarfSection.End.toPtr<void *>(),
+ UnwindInfo->CompactUnwindSection.Start.toPtr<void *>(),
+ UnwindInfo->CompactUnwindSection.End.toPtr<void *>());
+ });
+ for (auto &CodeRange : UnwindInfo->CodeRanges) {
+ JDS->UnwindSections.erase(CodeRange.Start.toPtr<char *>(),
+ CodeRange.End.toPtr<char *>());
+ ORC_RT_DEBUG({
+ printdbg(" [ %p -- %p ]\n", CodeRange.Start.toPtr<void *>(),
+ CodeRange.End.toPtr<void *>());
+ });
+ }
+ }
+
+ for (auto &KV : Secs) {
+ // FIXME: Validate section ranges?
+ if (KV.first == "__TEXT,__eh_frame") {
+ if (!UseCallbackStyleUnwindInfo) {
+ // Use classic libunwind registration.
+ if (auto Err = deregisterEHFrames(KV.second.toSpan<const char>()))
+ return Err;
+ }
+ } else if (KV.first == "__DATA,__data") {
+ JDS->DataSectionContent.erase(KV.second.Start.toPtr<char *>());
+ } else if (KV.first == "__DATA,__common") {
+ JDS->ZeroInitRanges.erase(KV.second.Start.toPtr<char *>());
+ } else if (KV.first == "__DATA,__thread_data") {
+ if (auto Err =
+ deregisterThreadDataSection(KV.second.toSpan<const char>()))
+ return Err;
+ } else if (KV.first == "__DATA,__objc_selrefs")
+ JDS->ObjCSelRefsSections.removeIfPresent(KV.second);
+ else if (KV.first == "__DATA,__objc_classlist")
+ JDS->ObjCClassListSections.removeIfPresent(KV.second);
+ else if (KV.first == "__TEXT,__swift5_protos")
+ JDS->Swift5ProtocolsSections.removeIfPresent(KV.second);
+ else if (KV.first == "__TEXT,__swift5_proto")
+ JDS->Swift5ProtocolConformancesSections.removeIfPresent(KV.second);
+ else if (KV.first == "__TEXT,__swift5_types")
+ JDS->Swift5TypesSections.removeIfPresent(KV.second);
+ else if (KV.first == "__DATA,__mod_init_func")
+ JDS->ModInitsSections.removeIfPresent(KV.second);
+ else {
+ // Should this be a warning instead?
+ return make_error<StringError>(
+ "Encountered unexpected section " +
+ std::string(KV.first.data(), KV.first.size()) +
+ " while deregistering object platform sections");
+ }
+ }
+ return Error::success();
+}
+
+const char *MachOPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); }
+
+void *MachOPlatformRuntimeState::dlopen(std::string_view Path, int Mode) {
+ ORC_RT_DEBUG({
+ std::string S(Path.data(), Path.size());
+ printdbg("MachOPlatform::dlopen(\"%s\")\n", S.c_str());
+ });
+ std::lock_guard<std::recursive_mutex> Lock(DyldAPIMutex);
+ if (auto H = dlopenImpl(Path, Mode))
+ return *H;
+ else {
+ // FIXME: Make dlerror thread safe.
DLFcnError = toString(H.takeError());
return nullptr;
}
-
- return *H;
}
int MachOPlatformRuntimeState::dlclose(void *DSOHandle) {
- runAtExits(DSOHandle);
+ ORC_RT_DEBUG({
+ auto *JDS = getJITDylibStateByHeader(DSOHandle);
+ std::string DylibName;
+ if (JDS) {
+ std::string S;
+ printdbg("MachOPlatform::dlclose(%p) (%s)\n", DSOHandle, S.c_str());
+ } else
+ printdbg("MachOPlatform::dlclose(%p) (%s)\n", DSOHandle,
+ "invalid handle");
+ });
+ std::lock_guard<std::recursive_mutex> Lock(DyldAPIMutex);
+ if (auto Err = dlcloseImpl(DSOHandle)) {
+ // FIXME: Make dlerror thread safe.
+ DLFcnError = toString(std::move(Err));
+ return -1;
+ }
return 0;
}
-void *MachOPlatformRuntimeState::dlsym(void *DSOHandle, string_view Symbol) {
+void *MachOPlatformRuntimeState::dlsym(void *DSOHandle,
+ std::string_view Symbol) {
auto Addr = lookupSymbolInJITDylib(DSOHandle, Symbol);
if (!Addr) {
DLFcnError = toString(Addr.takeError());
int MachOPlatformRuntimeState::registerAtExit(void (*F)(void *), void *Arg,
void *DSOHandle) {
// FIXME: Handle out-of-memory errors, returning -1 if OOM.
- std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
- auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
- assert(JDS && "JITDylib state not initialized");
+ std::lock_guard<std::mutex> Lock(JDStatesMutex);
+ auto *JDS = getJITDylibStateByHeader(DSOHandle);
+ if (!JDS) {
+ ORC_RT_DEBUG({
+ printdbg("MachOPlatformRuntimeState::registerAtExit called with "
+ "unrecognized dso handle %p\n",
+ DSOHandle);
+ });
+ return -1;
+ }
JDS->AtExits.push_back({F, Arg});
return 0;
}
-void MachOPlatformRuntimeState::runAtExits(void *DSOHandle) {
- // FIXME: Should atexits be allowed to run concurrently with access to
- // JDState?
- AtExitsVector V;
- {
- std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
- auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
- assert(JDS && "JITDlybi state not initialized");
- std::swap(V, JDS->AtExits);
- }
-
- while (!V.empty()) {
- auto &AE = V.back();
+void MachOPlatformRuntimeState::runAtExits(
+ std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) {
+ auto AtExits = std::move(JDS.AtExits);
+
+ // Unlock while running atexits, as they may trigger operations that modify
+ // JDStates.
+ JDStatesLock.unlock();
+ while (!AtExits.empty()) {
+ auto &AE = AtExits.back();
AE.Func(AE.Arg);
- V.pop_back();
+ AtExits.pop_back();
}
+ JDStatesLock.lock();
+}
+
+void MachOPlatformRuntimeState::runAtExits(void *DSOHandle) {
+ std::unique_lock<std::mutex> Lock(JDStatesMutex);
+ auto *JDS = getJITDylibStateByHeader(DSOHandle);
+ ORC_RT_DEBUG({
+ printdbg("MachOPlatformRuntimeState::runAtExits called on unrecognized "
+ "dso_handle %p\n",
+ DSOHandle);
+ });
+ if (JDS)
+ runAtExits(Lock, *JDS);
}
Expected<std::pair<const char *, size_t>>
return *I;
}
-MachOPlatformRuntimeState::PerJITDylibState *
-MachOPlatformRuntimeState::getJITDylibStateByHeaderAddr(void *DSOHandle) {
+MachOPlatformRuntimeState::JITDylibState *
+MachOPlatformRuntimeState::getJITDylibStateByHeader(void *DSOHandle) {
auto I = JDStates.find(DSOHandle);
- if (I == JDStates.end())
- return nullptr;
+ if (I == JDStates.end()) {
+ I = JDStates.insert(std::make_pair(DSOHandle, JITDylibState())).first;
+ I->second.Header = DSOHandle;
+ }
return &I->second;
}
-MachOPlatformRuntimeState::PerJITDylibState *
-MachOPlatformRuntimeState::getJITDylibStateByName(string_view Name) {
- // FIXME: Avoid creating string copy here.
+MachOPlatformRuntimeState::JITDylibState *
+MachOPlatformRuntimeState::getJITDylibStateByName(std::string_view Name) {
+ // FIXME: Avoid creating string once we have C++20.
auto I = JDNameToHeader.find(std::string(Name.data(), Name.size()));
- if (I == JDNameToHeader.end())
- return nullptr;
- void *H = I->second;
- auto J = JDStates.find(H);
- assert(J != JDStates.end() &&
- "JITDylib has name map entry but no header map entry");
- return &J->second;
+ if (I != JDNameToHeader.end())
+ return getJITDylibStateByHeader(I->second);
+ return nullptr;
}
-MachOPlatformRuntimeState::PerJITDylibState &
-MachOPlatformRuntimeState::getOrCreateJITDylibState(
- MachOJITDylibInitializers &MOJDIs) {
- void *Header = MOJDIs.MachOHeaderAddress.toPtr<void *>();
+Expected<ExecutorAddr>
+MachOPlatformRuntimeState::lookupSymbolInJITDylib(void *DSOHandle,
+ std::string_view Sym) {
+ Expected<ExecutorAddr> Result((ExecutorAddr()));
+ if (auto Err = WrapperFunction<SPSExpected<SPSExecutorAddr>(
+ SPSExecutorAddr, SPSString)>::call(&__orc_rt_macho_symbol_lookup_tag,
+ Result,
+ ExecutorAddr::fromPtr(DSOHandle),
+ Sym))
+ return std::move(Err);
+ return Result;
+}
- auto &JDS = JDStates[Header];
+// eh-frame registration functions.
+// We expect these to be available for all processes.
+extern "C" void __register_frame(const void *);
+extern "C" void __deregister_frame(const void *);
- // If this entry hasn't been created yet.
- if (!JDS.Header) {
- assert(!JDNameToHeader.count(MOJDIs.Name) &&
- "JITDylib has header map entry but no name map entry");
- JDNameToHeader[MOJDIs.Name] = Header;
- JDS.Header = Header;
- }
+template <typename HandleFDEFn>
+void walkEHFrameSection(span<const char> EHFrameSection,
+ HandleFDEFn HandleFDE) {
+ const char *CurCFIRecord = EHFrameSection.data();
+ uint64_t Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord);
- return JDS;
+ while (CurCFIRecord != EHFrameSection.end() && Size != 0) {
+ const char *OffsetField = CurCFIRecord + (Size == 0xffffffff ? 12 : 4);
+ if (Size == 0xffffffff)
+ Size = *reinterpret_cast<const uint64_t *>(CurCFIRecord + 4) + 12;
+ else
+ Size += 4;
+ uint32_t Offset = *reinterpret_cast<const uint32_t *>(OffsetField);
+
+ if (Offset != 0)
+ HandleFDE(CurCFIRecord);
+
+ CurCFIRecord += Size;
+ Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord);
+ }
}
-Error MachOPlatformRuntimeState::registerThreadDataSection(
- span<const char> ThreadDataSection) {
- std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
- auto I = ThreadDataSections.upper_bound(ThreadDataSection.data());
- if (I != ThreadDataSections.begin()) {
- auto J = std::prev(I);
- if (J->first + J->second > ThreadDataSection.data())
- return make_error<StringError>("Overlapping __thread_data sections");
+bool MachOPlatformRuntimeState::lookupUnwindSections(
+ void *Addr, unw_dynamic_unwind_sections &Info) {
+ ORC_RT_DEBUG(
+ { printdbg("Tried to lookup unwind-info via new lookup call.\n"); });
+ std::lock_guard<std::mutex> Lock(JDStatesMutex);
+ for (auto &KV : JDStates) {
+ auto &JD = KV.second;
+ auto I = JD.UnwindSections.find(reinterpret_cast<char *>(Addr));
+ if (I != JD.UnwindSections.end()) {
+ Info.dso_base = reinterpret_cast<uintptr_t>(JD.Header);
+ Info.dwarf_section =
+ reinterpret_cast<uintptr_t>(I->second.DwarfSection.data());
+ Info.dwarf_section_length = I->second.DwarfSection.size();
+ Info.compact_unwind_section =
+ reinterpret_cast<uintptr_t>(I->second.CompactUnwindSection.data());
+ Info.compact_unwind_section_length =
+ I->second.CompactUnwindSection.size();
+ return true;
+ }
}
- ThreadDataSections.insert(
- I, std::make_pair(ThreadDataSection.data(), ThreadDataSection.size()));
+ return false;
+}
+
+int MachOPlatformRuntimeState::findDynamicUnwindSections(
+ uintptr_t addr, unw_dynamic_unwind_sections *info) {
+ if (!info)
+ return 0;
+ return MachOPlatformRuntimeState::get().lookupUnwindSections((void *)addr,
+ *info);
+}
+
+Error MachOPlatformRuntimeState::registerEHFrames(
+ span<const char> EHFrameSection) {
+ walkEHFrameSection(EHFrameSection, __register_frame);
return Error::success();
}
-Expected<ExecutorAddress>
-MachOPlatformRuntimeState::lookupSymbolInJITDylib(void *DSOHandle,
- string_view Sym) {
- Expected<ExecutorAddress> Result((ExecutorAddress()));
- if (auto Err = WrapperFunction<SPSExpected<SPSExecutorAddress>(
- SPSExecutorAddress,
- SPSString)>::call(&__orc_rt_macho_symbol_lookup_tag, Result,
- ExecutorAddress::fromPtr(DSOHandle), Sym))
- return std::move(Err);
- return Result;
+Error MachOPlatformRuntimeState::deregisterEHFrames(
+ span<const char> EHFrameSection) {
+ walkEHFrameSection(EHFrameSection, __deregister_frame);
+ return Error::success();
}
-Expected<MachOJITDylibInitializerSequence>
-MachOPlatformRuntimeState::getJITDylibInitializersByName(string_view Path) {
- Expected<MachOJITDylibInitializerSequence> Result(
- (MachOJITDylibInitializerSequence()));
- std::string PathStr(Path.data(), Path.size());
- if (auto Err =
- WrapperFunction<SPSExpected<SPSMachOJITDylibInitializerSequence>(
- SPSString)>::call(&__orc_rt_macho_get_initializers_tag, Result,
- Path))
- return std::move(Err);
- return Result;
+Error MachOPlatformRuntimeState::registerObjCSelectors(JITDylibState &JDS) {
+ if (!JDS.ObjCSelRefsSections.hasNewSections())
+ return Error::success();
+
+ if (ORC_RT_UNLIKELY(!sel_registerName))
+ return make_error<StringError>("sel_registerName is not available");
+
+ JDS.ObjCSelRefsSections.processNewSections([](span<void *> SelRefs) {
+ for (void *&SelEntry : SelRefs) {
+ const char *SelName = reinterpret_cast<const char *>(SelEntry);
+ auto Sel = sel_registerName(SelName);
+ *reinterpret_cast<SEL *>(&SelEntry) = Sel;
+ }
+ });
+
+ return Error::success();
+}
+
+Error MachOPlatformRuntimeState::registerObjCClasses(JITDylibState &JDS) {
+ if (!JDS.ObjCClassListSections.hasNewSections())
+ return Error::success();
+
+ if (ORC_RT_UNLIKELY(!objc_msgSend))
+ return make_error<StringError>("objc_msgSend is not available");
+ if (ORC_RT_UNLIKELY(!objc_readClassPair))
+ return make_error<StringError>("objc_readClassPair is not available");
+
+ struct ObjCClassCompiled {
+ void *Metaclass;
+ void *Parent;
+ void *Cache1;
+ void *Cache2;
+ void *Data;
+ };
+
+ auto ClassSelector = sel_registerName("class");
+
+ return JDS.ObjCClassListSections.processNewSections(
+ [&](span<void *> ClassPtrs) -> Error {
+ for (void *ClassPtr : ClassPtrs) {
+ auto *Cls = reinterpret_cast<Class>(ClassPtr);
+ auto *ClassCompiled = reinterpret_cast<ObjCClassCompiled *>(ClassPtr);
+ objc_msgSend(reinterpret_cast<id>(ClassCompiled->Parent),
+ ClassSelector);
+ auto Registered = objc_readClassPair(Cls, JDS.ObjCImageInfo);
+ // FIXME: Improve diagnostic by reporting the failed class's name.
+ if (Registered != Cls)
+ return make_error<StringError>(
+ "Unable to register Objective-C class");
+ }
+ return Error::success();
+ });
}
-Expected<void *> MachOPlatformRuntimeState::dlopenInitialize(string_view Path,
- int Mode) {
- // Either our JITDylib wasn't loaded, or it or one of its dependencies allows
- // reinitialization. We need to call in to the JIT to see if there's any new
- // work pending.
- auto InitSeq = getJITDylibInitializersByName(Path);
- if (!InitSeq)
- return InitSeq.takeError();
+Error MachOPlatformRuntimeState::registerSwift5Protocols(JITDylibState &JDS) {
- // Init sequences should be non-empty.
- if (InitSeq->empty())
+ if (!JDS.Swift5ProtocolsSections.hasNewSections())
+ return Error::success();
+
+ if (ORC_RT_UNLIKELY(!swift_registerProtocols))
+ return make_error<StringError>("swift_registerProtocols is not available");
+
+ JDS.Swift5ProtocolsSections.processNewSections([](span<char> ProtoSec) {
+ swift_registerProtocols(
+ reinterpret_cast<const ProtocolRecord *>(ProtoSec.data()),
+ reinterpret_cast<const ProtocolRecord *>(ProtoSec.data() +
+ ProtoSec.size()));
+ });
+
+ return Error::success();
+}
+
+Error MachOPlatformRuntimeState::registerSwift5ProtocolConformances(
+ JITDylibState &JDS) {
+
+ if (!JDS.Swift5ProtocolConformancesSections.hasNewSections())
+ return Error::success();
+
+ if (ORC_RT_UNLIKELY(!swift_registerProtocolConformances))
return make_error<StringError>(
- "__orc_rt_macho_get_initializers returned an "
- "empty init sequence");
+ "swift_registerProtocolConformances is not available");
+
+ JDS.Swift5ProtocolConformancesSections.processNewSections(
+ [](span<char> ProtoConfSec) {
+ swift_registerProtocolConformances(
+ reinterpret_cast<const ProtocolConformanceRecord *>(
+ ProtoConfSec.data()),
+ reinterpret_cast<const ProtocolConformanceRecord *>(
+ ProtoConfSec.data() + ProtoConfSec.size()));
+ });
+
+ return Error::success();
+}
+
+Error MachOPlatformRuntimeState::registerSwift5Types(JITDylibState &JDS) {
+
+ if (!JDS.Swift5TypesSections.hasNewSections())
+ return Error::success();
+
+ if (ORC_RT_UNLIKELY(!swift_registerTypeMetadataRecords))
+ return make_error<StringError>(
+ "swift_registerTypeMetadataRecords is not available");
+
+ JDS.Swift5TypesSections.processNewSections([&](span<char> TypesSec) {
+ swift_registerTypeMetadataRecords(
+ reinterpret_cast<const TypeMetadataRecord *>(TypesSec.data()),
+ reinterpret_cast<const TypeMetadataRecord *>(TypesSec.data() +
+ TypesSec.size()));
+ });
- // Otherwise register and run initializers for each JITDylib.
- for (auto &MOJDIs : *InitSeq)
- if (auto Err = initializeJITDylib(MOJDIs))
+ return Error::success();
+}
+
+Error MachOPlatformRuntimeState::runModInits(
+ std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) {
+ std::vector<span<void (*)()>> InitSections;
+ InitSections.reserve(JDS.ModInitsSections.numNewSections());
+
+ // Copy initializer sections: If the JITDylib is unsealed then the
+ // initializers could reach back into the JIT and cause more initializers to
+ // be added.
+ // FIXME: Skip unlock and run in-place on sealed JITDylibs?
+ JDS.ModInitsSections.processNewSections(
+ [&](span<void (*)()> Inits) { InitSections.push_back(Inits); });
+
+ JDStatesLock.unlock();
+ for (auto InitSec : InitSections)
+ for (auto *Init : InitSec)
+ Init();
+ JDStatesLock.lock();
+
+ return Error::success();
+}
+
+Expected<void *> MachOPlatformRuntimeState::dlopenImpl(std::string_view Path,
+ int Mode) {
+ std::unique_lock<std::mutex> Lock(JDStatesMutex);
+
+ // Try to find JITDylib state by name.
+ auto *JDS = getJITDylibStateByName(Path);
+
+ if (!JDS)
+ return make_error<StringError>("No registered JTIDylib for path " +
+ std::string(Path.data(), Path.size()));
+
+ // If this JITDylib is unsealed, or this is the first dlopen then run
+ // full dlopen path (update deps, push and run initializers, update ref
+ // counts on all JITDylibs in the dep tree).
+ if (!JDS->referenced() || !JDS->Sealed) {
+ if (auto Err = dlopenFull(Lock, *JDS))
return std::move(Err);
+ }
+
+ // Bump the ref-count on this dylib.
+ ++JDS->DlRefCount;
- // Return the header for the last item in the list.
- auto *JDS = getJITDylibStateByHeaderAddr(
- InitSeq->back().MachOHeaderAddress.toPtr<void *>());
- assert(JDS && "Missing state entry for JD");
+ // Return the header address.
return JDS->Header;
}
-Error MachOPlatformRuntimeState::initializeJITDylib(
- MachOJITDylibInitializers &MOJDIs) {
+Error MachOPlatformRuntimeState::dlopenFull(
+ std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) {
+ // Call back to the JIT to push the initializers.
+ Expected<MachOJITDylibDepInfoMap> DepInfo((MachOJITDylibDepInfoMap()));
+ // Unlock so that we can accept the initializer update.
+ JDStatesLock.unlock();
+ if (auto Err = WrapperFunction<SPSExpected<SPSMachOJITDylibDepInfoMap>(
+ SPSExecutorAddr)>::call(&__orc_rt_macho_push_initializers_tag,
+ DepInfo, ExecutorAddr::fromPtr(JDS.Header)))
+ return Err;
+ JDStatesLock.lock();
+
+ if (!DepInfo)
+ return DepInfo.takeError();
+
+ if (auto Err = dlopenInitialize(JDStatesLock, JDS, *DepInfo))
+ return Err;
+
+ if (!DepInfo->empty()) {
+ ORC_RT_DEBUG({
+ printdbg("Unrecognized dep-info key headers in dlopen of %s\n",
+ JDS.Name.c_str());
+ });
+ std::ostringstream ErrStream;
+ ErrStream << "Encountered unrecognized dep-info key headers "
+ "while processing dlopen of "
+ << JDS.Name;
+ return make_error<StringError>(ErrStream.str());
+ }
- auto &JDS = getOrCreateJITDylibState(MOJDIs);
- ++JDS.RefCount;
+ return Error::success();
+}
- for (auto &KV : InitSections) {
- const auto &Name = KV.first;
- const auto &Handler = KV.second;
- auto I = MOJDIs.InitSections.find(Name);
- if (I != MOJDIs.InitSections.end()) {
- if (auto Err = Handler(I->second, MOJDIs))
- return Err;
+Error MachOPlatformRuntimeState::dlopenInitialize(
+ std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS,
+ MachOJITDylibDepInfoMap &DepInfo) {
+ ORC_RT_DEBUG({
+ printdbg("MachOPlatformRuntimeState::dlopenInitialize(\"%s\")\n",
+ JDS.Name.c_str());
+ });
+
+ // If the header is not present in the dep map then assume that we
+ // already processed it earlier in the dlopenInitialize traversal and
+ // return.
+ // TODO: Keep a visited set instead so that we can error out on missing
+ // entries?
+ auto I = DepInfo.find(ExecutorAddr::fromPtr(JDS.Header));
+ if (I == DepInfo.end())
+ return Error::success();
+
+ auto DI = std::move(I->second);
+ DepInfo.erase(I);
+
+ // We don't need to re-initialize sealed JITDylibs that have already been
+ // initialized. Just check that their dep-map entry is empty as expected.
+ if (JDS.Sealed) {
+ if (!DI.DepHeaders.empty()) {
+ std::ostringstream ErrStream;
+ ErrStream << "Sealed JITDylib " << JDS.Header
+ << " already has registered dependencies";
+ return make_error<StringError>(ErrStream.str());
}
+ if (JDS.referenced())
+ return Error::success();
+ } else
+ JDS.Sealed = DI.Sealed;
+
+ // This is an unsealed or newly sealed JITDylib. Run initializers.
+ std::vector<JITDylibState *> OldDeps;
+ std::swap(JDS.Deps, OldDeps);
+ JDS.Deps.reserve(DI.DepHeaders.size());
+ for (auto DepHeaderAddr : DI.DepHeaders) {
+ auto *DepJDS = getJITDylibStateByHeader(DepHeaderAddr.toPtr<void *>());
+ if (!DepJDS) {
+ std::ostringstream ErrStream;
+ ErrStream << "Encountered unrecognized dep header "
+ << DepHeaderAddr.toPtr<void *>() << " while initializing "
+ << JDS.Name;
+ return make_error<StringError>(ErrStream.str());
+ }
+ ++DepJDS->LinkedAgainstRefCount;
+ if (auto Err = dlopenInitialize(JDStatesLock, *DepJDS, DepInfo))
+ return Err;
+ }
+
+ // Initialize this JITDylib.
+ if (auto Err = registerObjCSelectors(JDS))
+ return Err;
+ if (auto Err = registerObjCClasses(JDS))
+ return Err;
+ if (auto Err = registerSwift5Protocols(JDS))
+ return Err;
+ if (auto Err = registerSwift5ProtocolConformances(JDS))
+ return Err;
+ if (auto Err = registerSwift5Types(JDS))
+ return Err;
+ if (auto Err = runModInits(JDStatesLock, JDS))
+ return Err;
+
+ // Decrement old deps.
+ // FIXME: We should probably continue and just report deinitialize errors
+ // here.
+ for (auto *DepJDS : OldDeps) {
+ --DepJDS->LinkedAgainstRefCount;
+ if (!DepJDS->referenced())
+ if (auto Err = dlcloseDeinitialize(JDStatesLock, *DepJDS))
+ return Err;
+ }
+
+ return Error::success();
+}
+
+Error MachOPlatformRuntimeState::dlcloseImpl(void *DSOHandle) {
+ std::unique_lock<std::mutex> Lock(JDStatesMutex);
+
+ // Try to find JITDylib state by header.
+ auto *JDS = getJITDylibStateByHeader(DSOHandle);
+
+ if (!JDS) {
+ std::ostringstream ErrStream;
+ ErrStream << "No registered JITDylib for " << DSOHandle;
+ return make_error<StringError>(ErrStream.str());
+ }
+
+ // Bump the ref-count.
+ --JDS->DlRefCount;
+
+ if (!JDS->referenced())
+ return dlcloseDeinitialize(Lock, *JDS);
+
+ return Error::success();
+}
+
+Error MachOPlatformRuntimeState::dlcloseDeinitialize(
+ std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) {
+
+ ORC_RT_DEBUG({
+ printdbg("MachOPlatformRuntimeState::dlcloseDeinitialize(\"%s\")\n",
+ JDS.Name.c_str());
+ });
+
+ runAtExits(JDStatesLock, JDS);
+
+ // Reset mod-inits
+ JDS.ModInitsSections.reset();
+
+ // Reset data section contents.
+ for (auto &KV : JDS.DataSectionContent)
+ memcpy(KV.first, KV.second.data(), KV.second.size());
+ for (auto &KV : JDS.ZeroInitRanges)
+ memset(KV.first, 0, KV.second);
+
+ // Deinitialize any dependencies.
+ for (auto *DepJDS : JDS.Deps) {
+ --DepJDS->LinkedAgainstRefCount;
+ if (!DepJDS->referenced())
+ if (auto Err = dlcloseDeinitialize(JDStatesLock, *DepJDS))
+ return Err;
}
return Error::success();
delete static_cast<MachOPlatformRuntimeTLVManager *>(MachOTLVMgr);
}
+Error runWrapperFunctionCalls(std::vector<WrapperFunctionCall> WFCs) {
+ for (auto &WFC : WFCs)
+ if (auto Err = WFC.runWithSPSRet<void>())
+ return Err;
+ return Error::success();
+}
+
} // end anonymous namespace
//------------------------------------------------------------------------------
ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
__orc_rt_macho_platform_bootstrap(char *ArgData, size_t ArgSize) {
- MachOPlatformRuntimeState::initialize();
- return WrapperFunctionResult().release();
+ return WrapperFunction<SPSError()>::handle(
+ ArgData, ArgSize,
+ []() { return MachOPlatformRuntimeState::create(); })
+ .release();
}
ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
__orc_rt_macho_platform_shutdown(char *ArgData, size_t ArgSize) {
- MachOPlatformRuntimeState::destroy();
- return WrapperFunctionResult().release();
+ return WrapperFunction<SPSError()>::handle(
+ ArgData, ArgSize,
+ []() { return MachOPlatformRuntimeState::destroy(); })
+ .release();
}
-/// Wrapper function for registering metadata on a per-object basis.
ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
-__orc_rt_macho_register_object_sections(char *ArgData, size_t ArgSize) {
- return WrapperFunction<SPSError(SPSMachOPerObjectSectionsToRegister)>::handle(
+__orc_rt_macho_register_jitdylib(char *ArgData, size_t ArgSize) {
+ return WrapperFunction<SPSError(SPSString, SPSExecutorAddr)>::handle(
ArgData, ArgSize,
- [](MachOPerObjectSectionsToRegister &POSR) {
- return MachOPlatformRuntimeState::get().registerObjectSections(
- std::move(POSR));
+ [](std::string &Name, ExecutorAddr HeaderAddr) {
+ return MachOPlatformRuntimeState::get().registerJITDylib(
+ std::move(Name), HeaderAddr.toPtr<void *>());
})
.release();
}
-/// Wrapper for releasing per-object metadat.
ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
-__orc_rt_macho_deregister_object_sections(char *ArgData, size_t ArgSize) {
- return WrapperFunction<SPSError(SPSMachOPerObjectSectionsToRegister)>::handle(
+__orc_rt_macho_deregister_jitdylib(char *ArgData, size_t ArgSize) {
+ return WrapperFunction<SPSError(SPSExecutorAddr)>::handle(
ArgData, ArgSize,
- [](MachOPerObjectSectionsToRegister &POSR) {
- return MachOPlatformRuntimeState::get().deregisterObjectSections(
- std::move(POSR));
+ [](ExecutorAddr HeaderAddr) {
+ return MachOPlatformRuntimeState::get().deregisterJITDylib(
+ HeaderAddr.toPtr<void *>());
})
.release();
}
+ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
+__orc_rt_macho_register_object_platform_sections(char *ArgData,
+ size_t ArgSize) {
+ return WrapperFunction<SPSError(SPSExecutorAddr,
+ SPSOptional<SPSUnwindSectionInfo>,
+ SPSMachOObjectPlatformSectionsMap)>::
+ handle(ArgData, ArgSize,
+ [](ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> USI,
+ std::vector<std::pair<std::string_view, ExecutorAddrRange>>
+ &Secs) {
+ return MachOPlatformRuntimeState::get()
+ .registerObjectPlatformSections(HeaderAddr, std::move(USI),
+ std::move(Secs));
+ })
+ .release();
+}
+
+ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
+__orc_rt_macho_deregister_object_platform_sections(char *ArgData,
+ size_t ArgSize) {
+ return WrapperFunction<SPSError(SPSExecutorAddr,
+ SPSOptional<SPSUnwindSectionInfo>,
+ SPSMachOObjectPlatformSectionsMap)>::
+ handle(ArgData, ArgSize,
+ [](ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> USI,
+ std::vector<std::pair<std::string_view, ExecutorAddrRange>>
+ &Secs) {
+ return MachOPlatformRuntimeState::get()
+ .deregisterObjectPlatformSections(HeaderAddr, std::move(USI),
+ std::move(Secs));
+ })
+ .release();
+}
+
+ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
+__orc_rt_macho_run_wrapper_function_calls(char *ArgData, size_t ArgSize) {
+ return WrapperFunction<SPSError(SPSSequence<SPSWrapperFunctionCall>)>::handle(
+ ArgData, ArgSize, runWrapperFunctionCalls)
+ .release();
+}
+
//------------------------------------------------------------------------------
// TLV support
//------------------------------------------------------------------------------
namespace __orc_rt {
namespace macho {
-struct MachOPerObjectSectionsToRegister {
- ExecutorAddressRange EHFrameSection;
- ExecutorAddressRange ThreadDataSection;
-};
-
-struct MachOJITDylibInitializers {
- using SectionList = std::vector<ExecutorAddressRange>;
-
- MachOJITDylibInitializers() = default;
- MachOJITDylibInitializers(std::string Name,
- ExecutorAddress MachOHeaderAddress)
- : Name(std::move(Name)),
- MachOHeaderAddress(std::move(MachOHeaderAddress)) {}
-
- std::string Name;
- ExecutorAddress MachOHeaderAddress;
- ExecutorAddress ObjCImageInfoAddress;
-
- std::unordered_map<std::string, SectionList> InitSections;
-};
-
-class MachOJITDylibDeinitializers {};
-
-using MachOJITDylibInitializerSequence = std::vector<MachOJITDylibInitializers>;
-
-using MachOJITDylibDeinitializerSequence =
- std::vector<MachOJITDylibDeinitializers>;
-
enum dlopen_mode : int {
ORC_RT_RTLD_LAZY = 0x1,
ORC_RT_RTLD_NOW = 0x2,
};
} // end namespace macho
-
-using SPSMachOPerObjectSectionsToRegister =
- SPSTuple<SPSExecutorAddressRange, SPSExecutorAddressRange>;
-
-template <>
-class SPSSerializationTraits<SPSMachOPerObjectSectionsToRegister,
- macho::MachOPerObjectSectionsToRegister> {
-
-public:
- static size_t size(const macho::MachOPerObjectSectionsToRegister &MOPOSR) {
- return SPSMachOPerObjectSectionsToRegister::AsArgList::size(
- MOPOSR.EHFrameSection, MOPOSR.ThreadDataSection);
- }
-
- static bool serialize(SPSOutputBuffer &OB,
- const macho::MachOPerObjectSectionsToRegister &MOPOSR) {
- return SPSMachOPerObjectSectionsToRegister::AsArgList::serialize(
- OB, MOPOSR.EHFrameSection, MOPOSR.ThreadDataSection);
- }
-
- static bool deserialize(SPSInputBuffer &IB,
- macho::MachOPerObjectSectionsToRegister &MOPOSR) {
- return SPSMachOPerObjectSectionsToRegister::AsArgList::deserialize(
- IB, MOPOSR.EHFrameSection, MOPOSR.ThreadDataSection);
- }
-};
-
-using SPSNamedExecutorAddressRangeSequenceMap =
- SPSSequence<SPSTuple<SPSString, SPSExecutorAddressRangeSequence>>;
-
-using SPSMachOJITDylibInitializers =
- SPSTuple<SPSString, SPSExecutorAddress, SPSExecutorAddress,
- SPSNamedExecutorAddressRangeSequenceMap>;
-
-using SPSMachOJITDylibInitializerSequence =
- SPSSequence<SPSMachOJITDylibInitializers>;
-
-/// Serialization traits for MachOJITDylibInitializers.
-template <>
-class SPSSerializationTraits<SPSMachOJITDylibInitializers,
- macho::MachOJITDylibInitializers> {
-public:
- static size_t size(const macho::MachOJITDylibInitializers &MOJDIs) {
- return SPSMachOJITDylibInitializers::AsArgList::size(
- MOJDIs.Name, MOJDIs.MachOHeaderAddress, MOJDIs.ObjCImageInfoAddress,
- MOJDIs.InitSections);
- }
-
- static bool serialize(SPSOutputBuffer &OB,
- const macho::MachOJITDylibInitializers &MOJDIs) {
- return SPSMachOJITDylibInitializers::AsArgList::serialize(
- OB, MOJDIs.Name, MOJDIs.MachOHeaderAddress, MOJDIs.ObjCImageInfoAddress,
- MOJDIs.InitSections);
- }
-
- static bool deserialize(SPSInputBuffer &IB,
- macho::MachOJITDylibInitializers &MOJDIs) {
- return SPSMachOJITDylibInitializers::AsArgList::deserialize(
- IB, MOJDIs.Name, MOJDIs.MachOHeaderAddress, MOJDIs.ObjCImageInfoAddress,
- MOJDIs.InitSections);
- }
-};
-
} // end namespace __orc_rt
#endif // ORC_RT_MACHO_PLATFORM_H
--- /dev/null
+//===-- macho_tlv.arm64.s ---------------------------------------*- ASM -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime support library.
+//
+//===----------------------------------------------------------------------===//
+
+// The content of this file is arm64-only
+#if defined(__arm64__) || defined(__aarch64__)
+
+#define REGISTER_SAVE_SPACE_SIZE 32 * 24
+
+ .text
+
+ // returns address of TLV in x0, all other registers preserved
+ .globl ___orc_rt_macho_tlv_get_addr
+___orc_rt_macho_tlv_get_addr:
+ sub sp, sp, #REGISTER_SAVE_SPACE_SIZE
+ stp x29, x30, [sp, #16 * 1]
+ stp x27, x28, [sp, #16 * 2]
+ stp x25, x26, [sp, #16 * 3]
+ stp x23, x24, [sp, #16 * 4]
+ stp x21, x22, [sp, #16 * 5]
+ stp x19, x20, [sp, #16 * 6]
+ stp x17, x18, [sp, #16 * 7]
+ stp x15, x16, [sp, #16 * 8]
+ stp x13, x14, [sp, #16 * 9]
+ stp x11, x12, [sp, #16 * 10]
+ stp x9, x10, [sp, #16 * 11]
+ stp x7, x8, [sp, #16 * 12]
+ stp x5, x6, [sp, #16 * 13]
+ stp x3, x4, [sp, #16 * 14]
+ stp x1, x2, [sp, #16 * 15]
+ stp q30, q31, [sp, #32 * 8]
+ stp q28, q29, [sp, #32 * 9]
+ stp q26, q27, [sp, #32 * 10]
+ stp q24, q25, [sp, #32 * 11]
+ stp q22, q23, [sp, #32 * 12]
+ stp q20, q21, [sp, #32 * 13]
+ stp q18, q19, [sp, #32 * 14]
+ stp q16, q17, [sp, #32 * 15]
+ stp q14, q15, [sp, #32 * 16]
+ stp q12, q13, [sp, #32 * 17]
+ stp q10, q11, [sp, #32 * 18]
+ stp q8, q9, [sp, #32 * 19]
+ stp q6, q7, [sp, #32 * 20]
+ stp q4, q5, [sp, #32 * 21]
+ stp q2, q3, [sp, #32 * 22]
+ stp q0, q1, [sp, #32 * 23]
+
+ bl ___orc_rt_macho_tlv_get_addr_impl
+
+ ldp q0, q1, [sp, #32 * 23]
+ ldp q2, q3, [sp, #32 * 22]
+ ldp q4, q5, [sp, #32 * 21]
+ ldp q6, q7, [sp, #32 * 20]
+ ldp q8, q9, [sp, #32 * 19]
+ ldp q10, q11, [sp, #32 * 18]
+ ldp q12, q13, [sp, #32 * 17]
+ ldp q14, q15, [sp, #32 * 16]
+ ldp q16, q17, [sp, #32 * 15]
+ ldp q18, q19, [sp, #32 * 14]
+ ldp q20, q21, [sp, #32 * 13]
+ ldp q22, q23, [sp, #32 * 12]
+ ldp q24, q25, [sp, #32 * 11]
+ ldp q26, q27, [sp, #32 * 10]
+ ldp q28, q29, [sp, #32 * 9]
+ ldp q30, q31, [sp, #32 * 8]
+ ldp x1, x2, [sp, #16 * 15]
+ ldp x3, x4, [sp, #16 * 14]
+ ldp x5, x6, [sp, #16 * 13]
+ ldp x7, x8, [sp, #16 * 12]
+ ldp x9, x10, [sp, #16 * 11]
+ ldp x11, x12, [sp, #16 * 10]
+ ldp x13, x14, [sp, #16 * 9]
+ ldp x15, x16, [sp, #16 * 8]
+ ldp x17, x18, [sp, #16 * 7]
+ ldp x19, x20, [sp, #16 * 6]
+ ldp x21, x22, [sp, #16 * 5]
+ ldp x23, x24, [sp, #16 * 4]
+ ldp x25, x26, [sp, #16 * 3]
+ ldp x27, x28, [sp, #16 * 2]
+ ldp x29, x30, [sp, #16 * 1]
+ add sp, sp, #REGISTER_SAVE_SPACE_SIZE
+ ret
+
+#endif // defined(__arm64__) || defined(__aarch64__)
//
//===----------------------------------------------------------------------===//
+// The content of this file is x86_64-only
+#if defined(__x86_64__)
+
#define REGISTER_SAVE_SPACE_SIZE 512
.text
addq $REGISTER_SAVE_SPACE_SIZE, %rsp
popq %rbp
ret
+
+#endif // defined(__x86_64__)
handle(ArgData, ArgSize,
[](const std::string &JITDylibName,
const std::string &EntrySymbolName,
- const std::vector<string_view> &Args) {
+ const std::vector<std::string_view> &Args) {
std::vector<std::unique_ptr<char[]>> ArgVStorage;
ArgVStorage.reserve(Args.size());
for (auto &Arg : Args) {
#include "error.h"
#include "stl_extras.h"
+#include <optional>
#include <string>
+#include <string_view>
#include <tuple>
#include <type_traits>
#include <unordered_map>
class SPSEmpty {};
/// Represents an address in the executor.
-class SPSExecutorAddress {};
+class SPSExecutorAddr {};
/// SPS tag type for tuples.
///
typedef SPSArgList<SPSTagTs...> AsArgList;
};
+/// SPS tag type for optionals.
+///
+/// SPSOptionals should be serialized as a bool with true indicating that an
+/// SPSTagT value is present, and false indicating that there is no value.
+/// If the boolean is true then the serialized SPSTagT will follow immediately
+/// after it.
+template <typename SPSTagT> class SPSOptional {};
+
/// SPS tag type for sequences.
///
/// SPSSequences should be serialized as a uint64_t sequence length,
}
};
+/// Trivial serialization / deserialization for span<char>
+template <> class SPSSerializationTraits<SPSSequence<char>, span<const char>> {
+public:
+ static size_t size(const span<const char> &S) {
+ return SPSArgList<uint64_t>::size(static_cast<uint64_t>(S.size())) +
+ S.size();
+ }
+ static bool serialize(SPSOutputBuffer &OB, const span<const char> &S) {
+ if (!SPSArgList<uint64_t>::serialize(OB, static_cast<uint64_t>(S.size())))
+ return false;
+ return OB.write(S.data(), S.size());
+ }
+ static bool deserialize(SPSInputBuffer &IB, span<const char> &S) {
+ uint64_t Size;
+ if (!SPSArgList<uint64_t>::deserialize(IB, Size))
+ return false;
+ S = span<const char>(IB.data(), Size);
+ return IB.skip(Size);
+ }
+};
+
/// SPSTuple serialization for std::pair.
template <typename SPSTagT1, typename SPSTagT2, typename T1, typename T2>
class SPSSerializationTraits<SPSTuple<SPSTagT1, SPSTagT2>, std::pair<T1, T2>> {
}
};
+/// SPSOptional serialization for std::optional.
+template <typename SPSTagT, typename T>
+class SPSSerializationTraits<SPSOptional<SPSTagT>, std::optional<T>> {
+public:
+ static size_t size(const std::optional<T> &Value) {
+ size_t Size = SPSArgList<bool>::size(!!Value);
+ if (Value)
+ Size += SPSArgList<SPSTagT>::size(*Value);
+ return Size;
+ }
+
+ static bool serialize(SPSOutputBuffer &OB, const std::optional<T> &Value) {
+ if (!SPSArgList<bool>::serialize(OB, !!Value))
+ return false;
+ if (Value)
+ return SPSArgList<SPSTagT>::serialize(OB, *Value);
+ return true;
+ }
+
+ static bool deserialize(SPSInputBuffer &IB, std::optional<T> &Value) {
+ bool HasValue;
+ if (!SPSArgList<bool>::deserialize(IB, HasValue))
+ return false;
+ if (HasValue) {
+ Value = T();
+ return SPSArgList<SPSTagT>::deserialize(IB, *Value);
+ } else
+ Value = std::optional<T>();
+ return true;
+ }
+};
+
/// Serialization for string_views.
///
/// Serialization is as for regular strings. Deserialization points directly
/// into the blob.
-template <> class SPSSerializationTraits<SPSString, __orc_rt::string_view> {
+template <> class SPSSerializationTraits<SPSString, std::string_view> {
public:
- static size_t size(const __orc_rt::string_view &S) {
+ static size_t size(const std::string_view &S) {
return SPSArgList<uint64_t>::size(static_cast<uint64_t>(S.size())) +
S.size();
}
- static bool serialize(SPSOutputBuffer &OB, const __orc_rt::string_view &S) {
+ static bool serialize(SPSOutputBuffer &OB, const std::string_view &S) {
if (!SPSArgList<uint64_t>::serialize(OB, static_cast<uint64_t>(S.size())))
return false;
return OB.write(S.data(), S.size());
}
- static bool deserialize(SPSInputBuffer &IB, __orc_rt::string_view &S) {
+ static bool deserialize(SPSInputBuffer &IB, std::string_view &S) {
const char *Data = nullptr;
uint64_t Size;
if (!SPSArgList<uint64_t>::deserialize(IB, Size))
return false;
+ if (Size > std::numeric_limits<size_t>::max())
+ return false;
Data = IB.data();
if (!IB.skip(Size))
return false;
- S = {Data, Size};
+ S = {Data, static_cast<size_t>(Size)};
return true;
}
};
namespace __orc_rt {
-namespace detail {
-
-template <typename F, typename Tuple, std::size_t... I>
-decltype(auto) apply_tuple_impl(F &&f, Tuple &&t, std::index_sequence<I...>) {
- return std::forward<F>(f)(std::get<I>(std::forward<Tuple>(t))...);
-}
-
-} // end namespace detail
-
-/// Given an input tuple (a1, a2, ..., an), pass the arguments of the
-/// tuple variadically to f as if by calling f(a1, a2, ..., an) and
-/// return the result.
-///
-/// FIXME: Switch to std::apply once we can use c++17.
-template <typename F, typename Tuple>
-decltype(auto) apply_tuple(F &&f, Tuple &&t) {
- using Indices = std::make_index_sequence<
- std::tuple_size<typename std::decay<Tuple>::type>::value>;
-
- return detail::apply_tuple_impl(std::forward<F>(f), std::forward<Tuple>(t),
- Indices{});
-}
+/// Substitute for std::identity.
+/// Switch to std::identity once we can use c++20.
+template <class Ty> struct identity {
+ using is_transparent = void;
+ using argument_type = Ty;
+
+ Ty &operator()(Ty &self) const { return self; }
+ const Ty &operator()(const Ty &self) const { return self; }
+};
} // namespace __orc_rt
--- /dev/null
+//===------- string_pool.h - Thread-safe pool for strings -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Contains a thread-safe string pool. Strings are ref-counted, but not
+// automatically deallocated. Unused entries can be cleared by calling
+// StringPool::clearDeadEntries.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef ORC_RT_STRING_POOL_H
+#define ORC_RT_STRING_POOL_H
+
+#include <atomic>
+#include <cassert>
+#include <functional>
+#include <mutex>
+#include <string>
+#include <unordered_map>
+
+namespace __orc_rt {
+
+class PooledStringPtr;
+
+/// String pool for strings names used by the ORC runtime.
+class StringPool {
+ friend class PooledStringPtr;
+
+public:
+ /// Destroy a StringPool.
+ ~StringPool();
+
+ /// Create a string pointer from the given string.
+ PooledStringPtr intern(std::string S);
+
+ /// Remove from the pool any entries that are no longer referenced.
+ void clearDeadEntries();
+
+ /// Returns true if the pool is empty.
+ bool empty() const;
+
+private:
+ using RefCountType = std::atomic<size_t>;
+ using PoolMap = std::unordered_map<std::string, RefCountType>;
+ using PoolMapEntry = PoolMap::value_type;
+ mutable std::mutex PoolMutex;
+ PoolMap Pool;
+};
+
+/// Pointer to a pooled string.
+class PooledStringPtr {
+ friend class StringPool;
+ friend struct std::hash<PooledStringPtr>;
+
+public:
+ PooledStringPtr() = default;
+ PooledStringPtr(std::nullptr_t) {}
+ PooledStringPtr(const PooledStringPtr &Other) : S(Other.S) {
+ if (S)
+ ++S->second;
+ }
+
+ PooledStringPtr &operator=(const PooledStringPtr &Other) {
+ if (S) {
+ assert(S->second && "Releasing PooledStringPtr with zero ref count");
+ --S->second;
+ }
+ S = Other.S;
+ if (S)
+ ++S->second;
+ return *this;
+ }
+
+ PooledStringPtr(PooledStringPtr &&Other) : S(nullptr) {
+ std::swap(S, Other.S);
+ }
+
+ PooledStringPtr &operator=(PooledStringPtr &&Other) {
+ if (S) {
+ assert(S->second && "Releasing PooledStringPtr with zero ref count");
+ --S->second;
+ }
+ S = nullptr;
+ std::swap(S, Other.S);
+ return *this;
+ }
+
+ ~PooledStringPtr() {
+ if (S) {
+ assert(S->second && "Releasing PooledStringPtr with zero ref count");
+ --S->second;
+ }
+ }
+
+ explicit operator bool() const { return S; }
+
+ const std::string &operator*() const { return S->first; }
+
+ friend bool operator==(const PooledStringPtr &LHS,
+ const PooledStringPtr &RHS) {
+ return LHS.S == RHS.S;
+ }
+
+ friend bool operator!=(const PooledStringPtr &LHS,
+ const PooledStringPtr &RHS) {
+ return !(LHS == RHS);
+ }
+
+ friend bool operator<(const PooledStringPtr &LHS,
+ const PooledStringPtr &RHS) {
+ return LHS.S < RHS.S;
+ }
+
+private:
+ using PoolEntry = StringPool::PoolMapEntry;
+ using PoolEntryPtr = PoolEntry *;
+
+ PooledStringPtr(StringPool::PoolMapEntry *S) : S(S) {
+ if (S)
+ ++S->second;
+ }
+
+ PoolEntryPtr S = nullptr;
+};
+
+inline StringPool::~StringPool() {
+#ifndef NDEBUG
+ clearDeadEntries();
+ assert(Pool.empty() && "Dangling references at pool destruction time");
+#endif // NDEBUG
+}
+
+inline PooledStringPtr StringPool::intern(std::string S) {
+ std::lock_guard<std::mutex> Lock(PoolMutex);
+ PoolMap::iterator I;
+ bool Added;
+ std::tie(I, Added) = Pool.try_emplace(std::move(S), 0);
+ return PooledStringPtr(&*I);
+}
+
+inline void StringPool::clearDeadEntries() {
+ std::lock_guard<std::mutex> Lock(PoolMutex);
+ for (auto I = Pool.begin(), E = Pool.end(); I != E;) {
+ auto Tmp = I++;
+ if (Tmp->second == 0)
+ Pool.erase(Tmp);
+ }
+}
+
+inline bool StringPool::empty() const {
+ std::lock_guard<std::mutex> Lock(PoolMutex);
+ return Pool.empty();
+}
+
+} // end namespace __orc_rt
+
+namespace std {
+
+// Make PooledStringPtrs hashable.
+template <> struct hash<__orc_rt::PooledStringPtr> {
+ size_t operator()(const __orc_rt::PooledStringPtr &A) const {
+ return hash<__orc_rt::PooledStringPtr::PoolEntryPtr>()(A.S);
+ }
+};
+
+} // namespace std
+
+#endif // ORC_RT_REF_COUNTED_STRING_POOL_H
--- /dev/null
+include(CompilerRTCompile)
+
+include_directories(..)
+
+# Unit tests target.
+add_custom_target(OrcRTUnitTests)
+set_target_properties(OrcRTUnitTests PROPERTIES FOLDER "OrcRT unittests")
+
+# Testing tools target.
+add_custom_target(OrcRTTools)
+set_target_properties(OrcRTTools PROPERTIES FOLDER "OrcRT tools")
+
+set(ORC_UNITTEST_CFLAGS
+# FIXME: This should be set for all unit tests.
+ -std=c++17
+ ${ORC_CFLAGS}
+ ${COMPILER_RT_UNITTEST_CFLAGS}
+ -I${COMPILER_RT_SOURCE_DIR}/lib/orc
+ )
+
+function(add_orc_lib library)
+ add_library(${library} STATIC ${ARGN})
+ set_target_properties(${library} PROPERTIES
+ ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+ FOLDER "Compiler-RT Runtime tests")
+endfunction()
+
+function(get_orc_lib_for_arch arch lib)
+ if(APPLE)
+ set(tgt_name "RTOrc.test.osx")
+ else()
+ set(tgt_name "RTOrc.test.${arch}")
+ endif()
+ set(${lib} "${tgt_name}" PARENT_SCOPE)
+endfunction()
+
+set(ORC_TEST_ARCH ${ORC_SUPPORTED_ARCH})
+set(ORC_UNITTEST_LINK_FLAGS
+ ${COMPILER_RT_UNITTEST_LINK_FLAGS}
+ ${CMAKE_THREAD_LIBS_INIT}
+ ${COMPILER_RT_UNWINDER_LINK_LIBS}
+ ${COMPILER_RT_CXX_LINK_LIBS})
+
+if(APPLE)
+ darwin_filter_host_archs(ORC_SUPPORTED_ARCH ORC_TEST_ARCH)
+ list(APPEND ORC_UNITTEST_CFLAGS ${DARWIN_osx_CFLAGS})
+ list(APPEND ORC_UNITTEST_LINK_FLAGS ${DARWIN_osx_LINK_FLAGS})
+else()
+ append_list_if(COMPILER_RT_HAS_LIBM -lm ORC_UNITTEST_LINK_FLAGS)
+ append_list_if(COMPILER_RT_HAS_LIBRT -lrt ORC_UNITTEST_LINK_FLAGS)
+ append_list_if(COMPILER_RT_HAS_LIBDL -ldl ORC_UNITTEST_LINK_FLAGS)
+ append_list_if(COMPILER_RT_HAS_LIBPTHREAD -pthread ORC_UNITTEST_LINK_FLAGS)
+ append_list_if(COMPILER_RT_HAS_LIBEXECINFO -lexecinfo ORC_UNITTEST_LINK_FLAGS)
+endif()
+
+set(ORC_DEPS orc)
+# ORC uses C++ standard library headers.
+if (TARGET cxx-headers OR HAVE_LIBCXX)
+ list(APPEND ORC_DEPS cxx-headers)
+endif()
+
+macro(add_orc_unittest testname)
+ cmake_parse_arguments(TEST "" "" "SOURCES;HEADERS" ${ARGN})
+ if(UNIX)
+ foreach(arch ${ORC_TEST_ARCH})
+ set(TEST_OBJECTS)
+ get_orc_lib_for_arch(${arch} ORC_RUNTIME_LIBS)
+ generate_compiler_rt_tests(TEST_OBJECTS
+ OrcRTUnitTests "${testname}-${arch}-Test" "${arch}"
+ SOURCES ${TEST_SOURCES} ${COMPILER_RT_GTEST_SOURCE}
+ RUNTIME "${ORC_RUNTIME_LIBS}"
+ COMPILE_DEPS ${TEST_HEADERS} ${ORC_HEADERS}
+ DEPS llvm_gtest ${ORC_DEPS}
+ CFLAGS ${ORC_UNITTEST_CFLAGS} ${COMPILER_RT_GTEST_CFLAGS}
+ LINK_FLAGS ${ORC_UNITTEST_LINK_FLAGS})
+ endforeach()
+ endif()
+endmacro()
+
+macro(add_orc_tool toolname)
+ cmake_parse_arguments(TOOL "" "" "SOURCES;HEADERS" ${ARGN})
+ if(UNIX)
+ foreach(arch ${ORC_TEST_ARCH})
+ set(TOOL_OBJECTS)
+ get_orc_lib_for_arch(${arch} ORC_RUNTIME_LIBS)
+ generate_compiler_rt_tests(TOOL_OBJECTS
+ OrcRTTools "${toolname}-${arch}" "${arch}"
+ SOURCES ${TOOL_SOURCES}
+ RUNTIME "${ORC_RUNTIME_LIBS}"
+ COMPILE_DEPS ${TOOL_HEADERS} ${ORC_HEADERS}
+ DEPS ${ORC_DEPS}
+ CFLAGS ${ORC_UNITTEST_CFLAGS}
+ LINK_FLAGS ${ORC_UNITTEST_LINK_FLAGS})
+ endforeach()
+ endif()
+endmacro()
+
+if (APPLE)
+ add_orc_lib("RTOrc.test.osx"
+ $<TARGET_OBJECTS:RTOrc.osx>)
+else()
+ foreach(arch ${ORC_SUPPORTED_ARCH})
+ add_orc_lib("RTOrc.test.${arch}"
+ $<TARGET_OBJECTS:RTOrc.${arch}>)
+ endforeach()
+endif()
+
+if(COMPILER_RT_INCLUDE_TESTS)
+ add_subdirectory(unit)
+endif()
+add_subdirectory(tools)
--- /dev/null
+add_orc_tool(orc-rt-executor SOURCES orc-rt-executor.cpp)
--- /dev/null
+//===- orc-rt-executor.cpp ------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Contains the orc-rt-executor test tool. This is a "blank executable" that
+// links the ORC runtime and can accept code from a JIT controller like lii or
+// llvm-jitlink.
+//
+//===----------------------------------------------------------------------===//
+
+#include <cstring>
+#include <iostream>
+#include <optional>
+#include <string_view>
+
+void printHelp(std::string_view ProgName, std::ostream &OS) {
+ OS << "usage: " << ProgName << " [help] [<mode>] <program arguments>...\n"
+ << " <mode> -- specify how to listen for JIT'd program\n"
+ << " filedesc=<in>,<out> -- read from <in> filedesc, write to out\n"
+ << " tcp=<host>:<port> -- listen on the given host/port\n"
+ << " help -- print help and exit\n"
+ << "\n"
+ << " Notes:\n"
+ << " Program arguments will be made available to the JIT controller.\n"
+ << " When running a JIT'd program containing a main function the\n"
+ << " controller may choose to pass these on to main, however\n"
+ << " orc-rt-executor does not enforce this.\n";
+}
+
+int main(int argc, char *argv[]) {
+ if (argc < 2) {
+ printHelp("orc-rt-executor", std::cerr);
+ std::cerr << "error: insufficient arguments.\n";
+ exit(1);
+ }
+
+ if (!strcmp(argv[1], "help")) {
+ printHelp(argv[0], std::cerr);
+ exit(0);
+ }
+
+ std::cerr << "error: One day I will be a real program, but I am not yet.\n";
+
+ return 0;
+}
--- /dev/null
+set(UNITTEST_SOURCES
+ adt_test.cpp
+ c_api_test.cpp
+ endian_test.cpp
+ error_test.cpp
+ executor_address_test.cpp
+ extensible_rtti_test.cpp
+ interval_map_test.cpp
+ interval_set_test.cpp
+ orc_unit_test_main.cpp
+ wrapper_function_utils_test.cpp
+ simple_packed_serialization_test.cpp
+ string_pool_test.cpp
+ )
+
+if (COMPILER_RT_CAN_EXECUTE_TESTS)
+ add_orc_unittest(OrcUnitTest SOURCES ${UNITTEST_SOURCES})
+endif()
--- /dev/null
+//===-- adt_test.cpp ------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime.
+//
+//===----------------------------------------------------------------------===//
+
+#include "adt.h"
+#include "gtest/gtest.h"
+
+#include <sstream>
+#include <string>
+
+using namespace __orc_rt;
+
+TEST(ADTTest, SpanDefaultConstruction) {
+ span<int> S;
+ EXPECT_TRUE(S.empty()) << "Default constructed span not empty";
+ EXPECT_EQ(S.size(), 0U) << "Default constructed span size not zero";
+ EXPECT_EQ(S.begin(), S.end()) << "Default constructed span begin != end";
+}
+
+TEST(ADTTest, SpanConstructFromFixedArray) {
+ int A[] = {1, 2, 3, 4, 5};
+ span<int> S(A);
+ EXPECT_FALSE(S.empty()) << "Span should be non-empty";
+ EXPECT_EQ(S.size(), 5U) << "Span has unexpected size";
+ EXPECT_EQ(std::distance(S.begin(), S.end()), 5U)
+ << "Unexpected iterator range size";
+ EXPECT_EQ(S.data(), &A[0]) << "Span data has unexpected value";
+ for (unsigned I = 0; I != S.size(); ++I)
+ EXPECT_EQ(S[I], A[I]) << "Unexpected span element value";
+}
+
+TEST(ADTTest, SpanConstructFromIteratorAndSize) {
+ int A[] = {1, 2, 3, 4, 5};
+ span<int> S(&A[0], 5);
+ EXPECT_FALSE(S.empty()) << "Span should be non-empty";
+ EXPECT_EQ(S.size(), 5U) << "Span has unexpected size";
+ EXPECT_EQ(std::distance(S.begin(), S.end()), 5U)
+ << "Unexpected iterator range size";
+ EXPECT_EQ(S.data(), &A[0]) << "Span data has unexpected value";
+ for (unsigned I = 0; I != S.size(); ++I)
+ EXPECT_EQ(S[I], A[I]) << "Unexpected span element value";
+}
--- /dev/null
+//===-- c_api_test.cpp ----------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime.
+//
+//===----------------------------------------------------------------------===//
+
+#include "orc_rt/c_api.h"
+#include "gtest/gtest.h"
+
+TEST(CAPITest, CWrapperFunctionResultInit) {
+ __orc_rt_CWrapperFunctionResult R;
+ __orc_rt_CWrapperFunctionResultInit(&R);
+
+ EXPECT_EQ(R.Size, 0U);
+ EXPECT_EQ(R.Data.ValuePtr, nullptr);
+
+ // Check that this value isn't treated as an out-of-band error.
+ EXPECT_EQ(__orc_rt_CWrapperFunctionResultGetOutOfBandError(&R), nullptr);
+
+ // Check that we can dispose of the value.
+ __orc_rt_DisposeCWrapperFunctionResult(&R);
+}
+
+TEST(CAPITest, CWrapperFunctionResultAllocSmall) {
+ constexpr size_t SmallAllocSize = sizeof(const char *);
+
+ auto R = __orc_rt_CWrapperFunctionResultAllocate(SmallAllocSize);
+ char *DataPtr = __orc_rt_CWrapperFunctionResultData(&R);
+
+ for (size_t I = 0; I != SmallAllocSize; ++I)
+ DataPtr[I] = 0x55 + I;
+
+ // Check that the inline storage in R.Data.Value contains the expected
+ // sequence.
+ EXPECT_EQ(R.Size, SmallAllocSize);
+ for (size_t I = 0; I != SmallAllocSize; ++I)
+ EXPECT_EQ(R.Data.Value[I], (char)(0x55 + I))
+ << "Unexpected value at index " << I;
+
+ // Check that this value isn't treated as an out-of-band error.
+ EXPECT_EQ(__orc_rt_CWrapperFunctionResultGetOutOfBandError(&R), nullptr);
+
+ // Check that __orc_rt_CWrapperFunctionResult(Data|Result|Size) and
+ // __orc_rt_CWrapperFunctionResultGetOutOfBandError behave as expected.
+ EXPECT_EQ(__orc_rt_CWrapperFunctionResultData(&R), R.Data.Value);
+ EXPECT_EQ(__orc_rt_CWrapperFunctionResultSize(&R), SmallAllocSize);
+ EXPECT_FALSE(__orc_rt_CWrapperFunctionResultEmpty(&R));
+ EXPECT_EQ(__orc_rt_CWrapperFunctionResultGetOutOfBandError(&R), nullptr);
+
+ // Check that we can dispose of the value.
+ __orc_rt_DisposeCWrapperFunctionResult(&R);
+}
+
+TEST(CAPITest, CWrapperFunctionResultAllocLarge) {
+ constexpr size_t LargeAllocSize = sizeof(const char *) + 1;
+
+ auto R = __orc_rt_CWrapperFunctionResultAllocate(LargeAllocSize);
+ char *DataPtr = __orc_rt_CWrapperFunctionResultData(&R);
+
+ for (size_t I = 0; I != LargeAllocSize; ++I)
+ DataPtr[I] = 0x55 + I;
+
+ // Check that the inline storage in R.Data.Value contains the expected
+ // sequence.
+ EXPECT_EQ(R.Size, LargeAllocSize);
+ EXPECT_EQ(R.Data.ValuePtr, DataPtr);
+ for (size_t I = 0; I != LargeAllocSize; ++I)
+ EXPECT_EQ(R.Data.ValuePtr[I], (char)(0x55 + I))
+ << "Unexpected value at index " << I;
+
+ // Check that this value isn't treated as an out-of-band error.
+ EXPECT_EQ(__orc_rt_CWrapperFunctionResultGetOutOfBandError(&R), nullptr);
+
+ // Check that __orc_rt_CWrapperFunctionResult(Data|Result|Size) and
+ // __orc_rt_CWrapperFunctionResultGetOutOfBandError behave as expected.
+ EXPECT_EQ(__orc_rt_CWrapperFunctionResultData(&R), R.Data.ValuePtr);
+ EXPECT_EQ(__orc_rt_CWrapperFunctionResultSize(&R), LargeAllocSize);
+ EXPECT_FALSE(__orc_rt_CWrapperFunctionResultEmpty(&R));
+ EXPECT_EQ(__orc_rt_CWrapperFunctionResultGetOutOfBandError(&R), nullptr);
+
+ // Check that we can dispose of the value.
+ __orc_rt_DisposeCWrapperFunctionResult(&R);
+}
+
+TEST(CAPITest, CWrapperFunctionResultFromRangeSmall) {
+ constexpr size_t SmallAllocSize = sizeof(const char *);
+
+ char Source[SmallAllocSize];
+ for (size_t I = 0; I != SmallAllocSize; ++I)
+ Source[I] = 0x55 + I;
+
+ __orc_rt_CWrapperFunctionResult R =
+ __orc_rt_CreateCWrapperFunctionResultFromRange(Source, SmallAllocSize);
+
+ // Check that the inline storage in R.Data.Value contains the expected
+ // sequence.
+ EXPECT_EQ(R.Size, SmallAllocSize);
+ for (size_t I = 0; I != SmallAllocSize; ++I)
+ EXPECT_EQ(R.Data.Value[I], (char)(0x55 + I))
+ << "Unexpected value at index " << I;
+
+ // Check that we can dispose of the value.
+ __orc_rt_DisposeCWrapperFunctionResult(&R);
+}
+
+TEST(CAPITest, CWrapperFunctionResultFromRangeLarge) {
+ constexpr size_t LargeAllocSize = sizeof(const char *) + 1;
+
+ char Source[LargeAllocSize];
+ for (size_t I = 0; I != LargeAllocSize; ++I)
+ Source[I] = 0x55 + I;
+
+ __orc_rt_CWrapperFunctionResult R =
+ __orc_rt_CreateCWrapperFunctionResultFromRange(Source, LargeAllocSize);
+
+ // Check that the inline storage in R.Data.Value contains the expected
+ // sequence.
+ EXPECT_EQ(R.Size, LargeAllocSize);
+ for (size_t I = 0; I != LargeAllocSize; ++I)
+ EXPECT_EQ(R.Data.ValuePtr[I], (char)(0x55 + I))
+ << "Unexpected value at index " << I;
+
+ // Check that we can dispose of the value.
+ __orc_rt_DisposeCWrapperFunctionResult(&R);
+}
+
+TEST(CAPITest, CWrapperFunctionResultFromStringSmall) {
+ constexpr size_t SmallAllocSize = sizeof(const char *);
+
+ char Source[SmallAllocSize];
+ for (size_t I = 0; I != SmallAllocSize - 1; ++I)
+ Source[I] = 'a' + I;
+ Source[SmallAllocSize - 1] = '\0';
+
+ __orc_rt_CWrapperFunctionResult R =
+ __orc_rt_CreateCWrapperFunctionResultFromString(Source);
+
+ // Check that the inline storage in R.Data.Value contains the expected
+ // sequence.
+ EXPECT_EQ(R.Size, SmallAllocSize);
+ for (size_t I = 0; I != SmallAllocSize - 1; ++I)
+ EXPECT_EQ(R.Data.Value[I], (char)('a' + I))
+ << "Unexpected value at index " << I;
+ EXPECT_EQ(R.Data.Value[SmallAllocSize - 1], '\0')
+ << "Unexpected value at index " << (SmallAllocSize - 1);
+
+ // Check that we can dispose of the value.
+ __orc_rt_DisposeCWrapperFunctionResult(&R);
+}
+
+TEST(CAPITest, CWrapperFunctionResultFromStringLarge) {
+ constexpr size_t LargeAllocSize = sizeof(const char *) + 1;
+
+ char Source[LargeAllocSize];
+ for (size_t I = 0; I != LargeAllocSize - 1; ++I)
+ Source[I] = 'a' + I;
+ Source[LargeAllocSize - 1] = '\0';
+
+ __orc_rt_CWrapperFunctionResult R =
+ __orc_rt_CreateCWrapperFunctionResultFromString(Source);
+
+ // Check that the inline storage in R.Data.Value contains the expected
+ // sequence.
+ EXPECT_EQ(R.Size, LargeAllocSize);
+ for (size_t I = 0; I != LargeAllocSize - 1; ++I)
+ EXPECT_EQ(R.Data.ValuePtr[I], (char)('a' + I))
+ << "Unexpected value at index " << I;
+ EXPECT_EQ(R.Data.ValuePtr[LargeAllocSize - 1], '\0')
+ << "Unexpected value at index " << (LargeAllocSize - 1);
+
+ // Check that we can dispose of the value.
+ __orc_rt_DisposeCWrapperFunctionResult(&R);
+}
+
+TEST(CAPITest, CWrapperFunctionResultFromOutOfBandError) {
+ constexpr const char *ErrMsg = "test error message";
+ __orc_rt_CWrapperFunctionResult R =
+ __orc_rt_CreateCWrapperFunctionResultFromOutOfBandError(ErrMsg);
+
+#ifndef NDEBUG
+ EXPECT_DEATH({ __orc_rt_CWrapperFunctionResultData(&R); },
+ "Cannot get data for out-of-band error value");
+ EXPECT_DEATH({ __orc_rt_CWrapperFunctionResultSize(&R); },
+ "Cannot get size for out-of-band error value");
+#endif
+
+ EXPECT_FALSE(__orc_rt_CWrapperFunctionResultEmpty(&R));
+ const char *OOBErrMsg = __orc_rt_CWrapperFunctionResultGetOutOfBandError(&R);
+ EXPECT_NE(OOBErrMsg, nullptr);
+ EXPECT_NE(OOBErrMsg, ErrMsg);
+ EXPECT_TRUE(strcmp(OOBErrMsg, ErrMsg) == 0);
+
+ __orc_rt_DisposeCWrapperFunctionResult(&R);
+}
--- /dev/null
+//===- endian_test.cpp ------------------------- swap byte order test -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime.
+//
+// Adapted from the llvm/unittests/Support/SwapByteOrderTest.cpp LLVM unit test.
+//
+//===----------------------------------------------------------------------===//
+
+#include "endianness.h"
+#include "gtest/gtest.h"
+
+using namespace __orc_rt;
+
+TEST(Endian, ByteSwap_32) {
+ EXPECT_EQ(0x44332211u, ByteSwap_32(0x11223344));
+ EXPECT_EQ(0xDDCCBBAAu, ByteSwap_32(0xAABBCCDD));
+}
+
+TEST(Endian, ByteSwap_64) {
+ EXPECT_EQ(0x8877665544332211ULL, ByteSwap_64(0x1122334455667788LL));
+ EXPECT_EQ(0x1100FFEEDDCCBBAAULL, ByteSwap_64(0xAABBCCDDEEFF0011LL));
+}
+
+// In these first two tests all of the original_uintx values are truncated
+// except for 64. We could avoid this, but there's really no point.
+TEST(Endian, getSwappedBytes_UnsignedRoundTrip) {
+ // The point of the bit twiddling of magic is to test with and without bits
+ // in every byte.
+ uint64_t value = 1;
+ for (std::size_t i = 0; i <= sizeof(value); ++i) {
+ uint8_t original_uint8 = static_cast<uint8_t>(value);
+ EXPECT_EQ(original_uint8, getSwappedBytes(getSwappedBytes(original_uint8)));
+
+ uint16_t original_uint16 = static_cast<uint16_t>(value);
+ EXPECT_EQ(original_uint16,
+ getSwappedBytes(getSwappedBytes(original_uint16)));
+
+ uint32_t original_uint32 = static_cast<uint32_t>(value);
+ EXPECT_EQ(original_uint32,
+ getSwappedBytes(getSwappedBytes(original_uint32)));
+
+ uint64_t original_uint64 = static_cast<uint64_t>(value);
+ EXPECT_EQ(original_uint64,
+ getSwappedBytes(getSwappedBytes(original_uint64)));
+
+ value = (value << 8) | 0x55; // binary 0101 0101.
+ }
+}
+
+TEST(Endian, getSwappedBytes_SignedRoundTrip) {
+ // The point of the bit twiddling of magic is to test with and without bits
+ // in every byte.
+ uint64_t value = 1;
+ for (std::size_t i = 0; i <= sizeof(value); ++i) {
+ int8_t original_int8 = static_cast<int8_t>(value);
+ EXPECT_EQ(original_int8, getSwappedBytes(getSwappedBytes(original_int8)));
+
+ int16_t original_int16 = static_cast<int16_t>(value);
+ EXPECT_EQ(original_int16, getSwappedBytes(getSwappedBytes(original_int16)));
+
+ int32_t original_int32 = static_cast<int32_t>(value);
+ EXPECT_EQ(original_int32, getSwappedBytes(getSwappedBytes(original_int32)));
+
+ int64_t original_int64 = static_cast<int64_t>(value);
+ EXPECT_EQ(original_int64, getSwappedBytes(getSwappedBytes(original_int64)));
+
+ // Test other sign.
+ value *= -1;
+
+ original_int8 = static_cast<int8_t>(value);
+ EXPECT_EQ(original_int8, getSwappedBytes(getSwappedBytes(original_int8)));
+
+ original_int16 = static_cast<int16_t>(value);
+ EXPECT_EQ(original_int16, getSwappedBytes(getSwappedBytes(original_int16)));
+
+ original_int32 = static_cast<int32_t>(value);
+ EXPECT_EQ(original_int32, getSwappedBytes(getSwappedBytes(original_int32)));
+
+ original_int64 = static_cast<int64_t>(value);
+ EXPECT_EQ(original_int64, getSwappedBytes(getSwappedBytes(original_int64)));
+
+ // Return to normal sign and twiddle.
+ value *= -1;
+ value = (value << 8) | 0x55; // binary 0101 0101.
+ }
+}
+
+TEST(Endian, getSwappedBytes_uint8_t) {
+ EXPECT_EQ(uint8_t(0x11), getSwappedBytes(uint8_t(0x11)));
+}
+
+TEST(Endian, getSwappedBytes_uint16_t) {
+ EXPECT_EQ(uint16_t(0x1122), getSwappedBytes(uint16_t(0x2211)));
+}
+
+TEST(Endian, getSwappedBytes_uint32_t) {
+ EXPECT_EQ(uint32_t(0x11223344), getSwappedBytes(uint32_t(0x44332211)));
+}
+
+TEST(Endian, getSwappedBytes_uint64_t) {
+ EXPECT_EQ(uint64_t(0x1122334455667788ULL),
+ getSwappedBytes(uint64_t(0x8877665544332211ULL)));
+}
+
+TEST(Endian, getSwappedBytes_int8_t) {
+ EXPECT_EQ(int8_t(0x11), getSwappedBytes(int8_t(0x11)));
+}
+
+TEST(Endian, getSwappedBytes_int16_t) {
+ EXPECT_EQ(int16_t(0x1122), getSwappedBytes(int16_t(0x2211)));
+}
+
+TEST(Endian, getSwappedBytes_int32_t) {
+ EXPECT_EQ(int32_t(0x11223344), getSwappedBytes(int32_t(0x44332211)));
+}
+
+TEST(Endian, getSwappedBytes_int64_t) {
+ EXPECT_EQ(int64_t(0x1122334455667788LL),
+ getSwappedBytes(int64_t(0x8877665544332211LL)));
+}
+
+TEST(Endian, swapByteOrder_uint8_t) {
+ uint8_t value = 0x11;
+ swapByteOrder(value);
+ EXPECT_EQ(uint8_t(0x11), value);
+}
+
+TEST(Endian, swapByteOrder_uint16_t) {
+ uint16_t value = 0x2211;
+ swapByteOrder(value);
+ EXPECT_EQ(uint16_t(0x1122), value);
+}
+
+TEST(Endian, swapByteOrder_uint32_t) {
+ uint32_t value = 0x44332211;
+ swapByteOrder(value);
+ EXPECT_EQ(uint32_t(0x11223344), value);
+}
+
+TEST(Endian, swapByteOrder_uint64_t) {
+ uint64_t value = 0x8877665544332211ULL;
+ swapByteOrder(value);
+ EXPECT_EQ(uint64_t(0x1122334455667788ULL), value);
+}
+
+TEST(Endian, swapByteOrder_int8_t) {
+ int8_t value = 0x11;
+ swapByteOrder(value);
+ EXPECT_EQ(int8_t(0x11), value);
+}
+
+TEST(Endian, swapByteOrder_int16_t) {
+ int16_t value = 0x2211;
+ swapByteOrder(value);
+ EXPECT_EQ(int16_t(0x1122), value);
+}
+
+TEST(Endian, swapByteOrder_int32_t) {
+ int32_t value = 0x44332211;
+ swapByteOrder(value);
+ EXPECT_EQ(int32_t(0x11223344), value);
+}
+
+TEST(Endian, swapByteOrder_int64_t) {
+ int64_t value = 0x8877665544332211LL;
+ swapByteOrder(value);
+ EXPECT_EQ(int64_t(0x1122334455667788LL), value);
+}
--- /dev/null
+//===-- error_test.cpp --sssssssss-----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime.
+//
+// Note:
+// This unit test was adapted from
+// llvm/unittests/Support/ExtensibleRTTITest.cpp
+//
+//===----------------------------------------------------------------------===//
+
+#include "error.h"
+#include "gtest/gtest.h"
+
+using namespace __orc_rt;
+
+namespace {
+
+class CustomError : public RTTIExtends<CustomError, ErrorInfoBase> {
+public:
+ CustomError(int V1) : V1(V1) {}
+ std::string toString() const override {
+ return "CustomError V1 = " + std::to_string(V1);
+ }
+ int getV1() const { return V1; }
+
+protected:
+ int V1;
+};
+
+class CustomSubError : public RTTIExtends<CustomSubError, CustomError> {
+public:
+ CustomSubError(int V1, std::string V2)
+ : RTTIExtends<CustomSubError, CustomError>(V1), V2(std::move(V2)) {}
+ std::string toString() const override {
+ return "CustomSubError V1 = " + std::to_string(V1) + ", " + V2;
+ }
+ const std::string &getV2() const { return V2; }
+
+protected:
+ std::string V2;
+};
+
+} // end anonymous namespace
+
+// Test that a checked success value doesn't cause any issues.
+TEST(Error, CheckedSuccess) {
+ Error E = Error::success();
+ EXPECT_FALSE(E) << "Unexpected error while testing Error 'Success'";
+}
+
+// Check that a consumed success value doesn't cause any issues.
+TEST(Error, ConsumeSuccess) { consumeError(Error::success()); }
+
+TEST(Error, ConsumeError) {
+ Error E = make_error<CustomError>(42);
+ if (E) {
+ consumeError(std::move(E));
+ } else
+ ADD_FAILURE() << "Error failure value should convert to true";
+}
+
+// Test that unchecked success values cause an abort.
+TEST(Error, UncheckedSuccess) {
+ EXPECT_DEATH({ Error E = Error::success(); },
+ "Error must be checked prior to destruction")
+ << "Unchecked Error Succes value did not cause abort()";
+}
+
+// Test that a checked but unhandled error causes an abort.
+TEST(Error, CheckedButUnhandledError) {
+ auto DropUnhandledError = []() {
+ Error E = make_error<CustomError>(42);
+ (void)!E;
+ };
+ EXPECT_DEATH(DropUnhandledError(),
+ "Error must be checked prior to destruction")
+ << "Unhandled Error failure value did not cause an abort()";
+}
+
+// Test that error_cast works as expected.
+TEST(Error, BasicErrorCast) {
+ {
+ // Check casting base error value to base error type.
+ auto E = make_error<CustomError>(42);
+ if (auto CSE = error_cast<CustomSubError>(E)) {
+ ADD_FAILURE() << "Derived cast incorrectly matched base error";
+ } else if (auto CE = error_cast<CustomError>(E)) {
+ EXPECT_EQ(CE->getV1(), 42) << "Unexpected wrapped value";
+ } else
+ ADD_FAILURE() << "Unexpected error value";
+ }
+
+ {
+ // Check casting derived error value to base error type.
+ auto E = make_error<CustomSubError>(42, "foo");
+ if (auto CE = error_cast<CustomError>(E)) {
+ EXPECT_EQ(CE->getV1(), 42) << "Unexpected wrapped value";
+ } else
+ ADD_FAILURE() << "Unexpected error value";
+ }
+
+ {
+ // Check casting derived error value to derived error type.
+ auto E = make_error<CustomSubError>(42, "foo");
+ if (auto CSE = error_cast<CustomSubError>(E)) {
+ EXPECT_EQ(CSE->getV1(), 42) << "Unexpected wrapped value";
+ EXPECT_EQ(CSE->getV2(), "foo") << "Unexpected wrapped value";
+ } else
+ ADD_FAILURE() << "Unexpected error value";
+ }
+}
+
+// ErrorAsOutParameter tester.
+static void errAsOutParamHelper(Error &Err) {
+ ErrorAsOutParameter ErrAsOutParam(&Err);
+ // Verify that checked flag is raised - assignment should not crash.
+ Err = Error::success();
+ // Raise the checked bit manually - caller should still have to test the
+ // error.
+ (void)!!Err;
+}
+
+// Test that ErrorAsOutParameter sets the checked flag on construction.
+TEST(Error, ErrorAsOutParameterChecked) {
+ Error E = Error::success();
+ errAsOutParamHelper(E);
+ (void)!!E;
+}
+
+// Test that ErrorAsOutParameter clears the checked flag on destruction.
+TEST(Error, ErrorAsOutParameterUnchecked) {
+ EXPECT_DEATH(
+ {
+ Error E = Error::success();
+ errAsOutParamHelper(E);
+ },
+ "Error must be checked prior to destruction")
+ << "ErrorAsOutParameter did not clear the checked flag on destruction.";
+}
+
+// Check 'Error::isA<T>' method handling.
+TEST(Error, IsAHandling) {
+ // Check 'isA' handling.
+ Error E = make_error<CustomError>(42);
+ Error F = make_error<CustomSubError>(42, "foo");
+ Error G = Error::success();
+
+ EXPECT_TRUE(E.isA<CustomError>());
+ EXPECT_FALSE(E.isA<CustomSubError>());
+ EXPECT_TRUE(F.isA<CustomError>());
+ EXPECT_TRUE(F.isA<CustomSubError>());
+ EXPECT_FALSE(G.isA<CustomError>());
+
+ consumeError(std::move(E));
+ consumeError(std::move(F));
+ consumeError(std::move(G));
+}
+
+TEST(Error, StringError) {
+ auto E = make_error<StringError>("foo");
+ if (auto SE = error_cast<StringError>(E)) {
+ EXPECT_EQ(SE->toString(), "foo") << "Unexpected StringError value";
+ } else
+ ADD_FAILURE() << "Expected StringError value";
+}
+
+// Test Checked Expected<T> in success mode.
+TEST(Error, CheckedExpectedInSuccessMode) {
+ Expected<int> A = 7;
+ EXPECT_TRUE(!!A) << "Expected with non-error value doesn't convert to 'true'";
+ // Access is safe in second test, since we checked the error in the first.
+ EXPECT_EQ(*A, 7) << "Incorrect Expected non-error value";
+}
+
+// Test Expected with reference type.
+TEST(Error, ExpectedWithReferenceType) {
+ int A = 7;
+ Expected<int &> B = A;
+ // 'Check' B.
+ (void)!!B;
+ int &C = *B;
+ EXPECT_EQ(&A, &C) << "Expected failed to propagate reference";
+}
+
+// Test Unchecked Expected<T> in success mode.
+// We expect this to blow up the same way Error would.
+// Test runs in debug mode only.
+TEST(Error, UncheckedExpectedInSuccessModeDestruction) {
+ EXPECT_DEATH({ Expected<int> A = 7; },
+ "Expected<T> must be checked before access or destruction.")
+ << "Unchecekd Expected<T> success value did not cause an abort().";
+}
+
+// Test Unchecked Expected<T> in success mode.
+// We expect this to blow up the same way Error would.
+// Test runs in debug mode only.
+TEST(Error, UncheckedExpectedInSuccessModeAccess) {
+ EXPECT_DEATH(
+ {
+ Expected<int> A = 7;
+ *A;
+ },
+ "Expected<T> must be checked before access or destruction.")
+ << "Unchecekd Expected<T> success value did not cause an abort().";
+}
+
+// Test Unchecked Expected<T> in success mode.
+// We expect this to blow up the same way Error would.
+// Test runs in debug mode only.
+TEST(Error, UncheckedExpectedInSuccessModeAssignment) {
+ EXPECT_DEATH(
+ {
+ Expected<int> A = 7;
+ A = 7;
+ },
+ "Expected<T> must be checked before access or destruction.")
+ << "Unchecekd Expected<T> success value did not cause an abort().";
+}
+
+// Test Expected<T> in failure mode.
+TEST(Error, ExpectedInFailureMode) {
+ Expected<int> A = make_error<CustomError>(42);
+ EXPECT_FALSE(!!A) << "Expected with error value doesn't convert to 'false'";
+ Error E = A.takeError();
+ EXPECT_TRUE(E.isA<CustomError>()) << "Incorrect Expected error value";
+ consumeError(std::move(E));
+}
+
+// Check that an Expected instance with an error value doesn't allow access to
+// operator*.
+// Test runs in debug mode only.
+TEST(Error, AccessExpectedInFailureMode) {
+ Expected<int> A = make_error<CustomError>(42);
+ EXPECT_DEATH(*A, "Expected<T> must be checked before access or destruction.")
+ << "Incorrect Expected error value";
+ consumeError(A.takeError());
+}
+
+// Check that an Expected instance with an error triggers an abort if
+// unhandled.
+// Test runs in debug mode only.
+TEST(Error, UnhandledExpectedInFailureMode) {
+ EXPECT_DEATH({ Expected<int> A = make_error<CustomError>(42); },
+ "Expected<T> must be checked before access or destruction.")
+ << "Unchecked Expected<T> failure value did not cause an abort()";
+}
+
+// Test covariance of Expected.
+TEST(Error, ExpectedCovariance) {
+ class B {};
+ class D : public B {};
+
+ Expected<B *> A1(Expected<D *>(nullptr));
+ // Check A1 by converting to bool before assigning to it.
+ (void)!!A1;
+ A1 = Expected<D *>(nullptr);
+ // Check A1 again before destruction.
+ (void)!!A1;
+
+ Expected<std::unique_ptr<B>> A2(Expected<std::unique_ptr<D>>(nullptr));
+ // Check A2 by converting to bool before assigning to it.
+ (void)!!A2;
+ A2 = Expected<std::unique_ptr<D>>(nullptr);
+ // Check A2 again before destruction.
+ (void)!!A2;
+}
+
+// Test that the ExitOnError utility works as expected.
+TEST(Error, CantFailSuccess) {
+ cantFail(Error::success());
+
+ int X = cantFail(Expected<int>(42));
+ EXPECT_EQ(X, 42) << "Expected value modified by cantFail";
+
+ int Dummy = 42;
+ int &Y = cantFail(Expected<int &>(Dummy));
+ EXPECT_EQ(&Dummy, &Y) << "Reference mangled by cantFail";
+}
+
+// Test that cantFail results in a crash if you pass it a failure value.
+TEST(Error, CantFailDeath) {
+ EXPECT_DEATH(cantFail(make_error<StringError>("foo")),
+ "cantFail called on failure value")
+ << "cantFail(Error) did not cause an abort for failure value";
+
+ EXPECT_DEATH(cantFail(Expected<int>(make_error<StringError>("foo"))),
+ "cantFail called on failure value")
+ << "cantFail(Expected<int>) did not cause an abort for failure value";
+}
--- /dev/null
+//===-- executor_address_test.cpp -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime.
+//
+// Note:
+// This unit test was adapted from
+// llvm/unittests/Support/ExecutorAddressTest.cpp
+//
+//===----------------------------------------------------------------------===//
+
+#include "executor_address.h"
+#include "gtest/gtest.h"
+
+using namespace __orc_rt;
+
+TEST(ExecutorAddrTest, DefaultAndNull) {
+ // Check that default constructed values and isNull behave as expected.
+
+ ExecutorAddr Default;
+ ExecutorAddr Null(0);
+ ExecutorAddr NonNull(1);
+
+ EXPECT_TRUE(Null.isNull());
+ EXPECT_EQ(Default, Null);
+
+ EXPECT_FALSE(NonNull.isNull());
+ EXPECT_NE(Default, NonNull);
+}
+
+TEST(ExecutorAddrTest, Ordering) {
+ // Check that ordering operations.
+ ExecutorAddr A1(1), A2(2);
+
+ EXPECT_LE(A1, A1);
+ EXPECT_LT(A1, A2);
+ EXPECT_GT(A2, A1);
+ EXPECT_GE(A2, A2);
+}
+
+TEST(ExecutorAddrTest, PtrConversion) {
+ // Test toPtr / fromPtr round-tripping.
+ int X = 0;
+ auto XAddr = ExecutorAddr::fromPtr(&X);
+ int *XPtr = XAddr.toPtr<int *>();
+
+ EXPECT_EQ(XPtr, &X);
+}
+
+static void F() {}
+
+TEST(ExecutorAddrTest, PtrConversionWithFunctionType) {
+ // Test that function types (as opposed to function pointer types) can be
+ // used with toPtr.
+ auto FAddr = ExecutorAddr::fromPtr(F);
+ void (*FPtr)() = FAddr.toPtr<void()>();
+
+ EXPECT_EQ(FPtr, &F);
+}
+
+TEST(ExecutorAddrTest, WrappingAndUnwrapping) {
+ constexpr uintptr_t RawAddr = 0x123456;
+ int *RawPtr = (int *)RawAddr;
+
+ constexpr uintptr_t TagOffset = 8 * (sizeof(uintptr_t) - 1);
+ uintptr_t TagVal = 0xA5;
+ uintptr_t TagBits = TagVal << TagOffset;
+ void *TaggedPtr = (void *)((uintptr_t)RawPtr | TagBits);
+
+ ExecutorAddr EA =
+ ExecutorAddr::fromPtr(TaggedPtr, ExecutorAddr::Untag(8, TagOffset));
+
+ EXPECT_EQ(EA.getValue(), RawAddr);
+
+ void *ReconstitutedTaggedPtr =
+ EA.toPtr<void *>(ExecutorAddr::Tag(TagVal, TagOffset));
+
+ EXPECT_EQ(TaggedPtr, ReconstitutedTaggedPtr);
+}
+
+TEST(ExecutorAddrTest, AddrRanges) {
+ ExecutorAddr A0(0), A1(1), A2(2), A3(3);
+ ExecutorAddrRange R0(A0, A1), R1(A1, A2), R2(A2, A3), R3(A0, A2), R4(A1, A3);
+ // 012
+ // R0: # -- Before R1
+ // R1: # --
+ // R2: # -- After R1
+ // R3: ## -- Overlaps R1 start
+ // R4: ## -- Overlaps R1 end
+
+ EXPECT_EQ(R1, ExecutorAddrRange(A1, A2));
+ EXPECT_EQ(R1, ExecutorAddrRange(A1, ExecutorAddrDiff(1)));
+ EXPECT_NE(R1, R2);
+
+ EXPECT_TRUE(R1.contains(A1));
+ EXPECT_FALSE(R1.contains(A0));
+ EXPECT_FALSE(R1.contains(A2));
+
+ EXPECT_FALSE(R1.overlaps(R0));
+ EXPECT_FALSE(R1.overlaps(R2));
+ EXPECT_TRUE(R1.overlaps(R3));
+ EXPECT_TRUE(R1.overlaps(R4));
+}
+
+TEST(ExecutorAddrTest, Hashable) {
+ uint64_t RawAddr = 0x1234567890ABCDEF;
+ ExecutorAddr Addr(RawAddr);
+
+ EXPECT_EQ(std::hash<uint64_t>()(RawAddr), std::hash<ExecutorAddr>()(Addr));
+}
--- /dev/null
+//===-- extensible_rtti_test.cpp ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime.
+//
+// Note:
+// This unit test was adapted from
+// llvm/unittests/Support/ExtensibleRTTITest.cpp
+//
+//===----------------------------------------------------------------------===//
+
+#include "extensible_rtti.h"
+#include "gtest/gtest.h"
+
+using namespace __orc_rt;
+
+namespace {
+
+class MyBase : public RTTIExtends<MyBase, RTTIRoot> {};
+
+class MyDerivedA : public RTTIExtends<MyDerivedA, MyBase> {};
+
+class MyDerivedB : public RTTIExtends<MyDerivedB, MyBase> {};
+
+} // end anonymous namespace
+
+TEST(ExtensibleRTTITest, BaseCheck) {
+ MyBase MB;
+ MyDerivedA MDA;
+ MyDerivedB MDB;
+
+ // Check MB properties.
+ EXPECT_TRUE(isa<RTTIRoot>(MB));
+ EXPECT_TRUE(isa<MyBase>(MB));
+ EXPECT_FALSE(isa<MyDerivedA>(MB));
+ EXPECT_FALSE(isa<MyDerivedB>(MB));
+
+ // Check MDA properties.
+ EXPECT_TRUE(isa<RTTIRoot>(MDA));
+ EXPECT_TRUE(isa<MyBase>(MDA));
+ EXPECT_TRUE(isa<MyDerivedA>(MDA));
+ EXPECT_FALSE(isa<MyDerivedB>(MDA));
+
+ // Check MDB properties.
+ EXPECT_TRUE(isa<RTTIRoot>(MDB));
+ EXPECT_TRUE(isa<MyBase>(MDB));
+ EXPECT_FALSE(isa<MyDerivedA>(MDB));
+ EXPECT_TRUE(isa<MyDerivedB>(MDB));
+}
--- /dev/null
+//===-- interval_map_test.cpp ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime.
+//
+//===----------------------------------------------------------------------===//
+
+#include "interval_map.h"
+#include "gtest/gtest.h"
+
+using namespace __orc_rt;
+
+TEST(IntervalMapTest, DefaultConstructed) {
+ // Check that a default-constructed IntervalMap behaves as expected.
+ IntervalMap<unsigned, unsigned, IntervalCoalescing::Enabled> M;
+
+ EXPECT_TRUE(M.empty());
+ EXPECT_TRUE(M.begin() == M.end());
+ EXPECT_TRUE(M.find(0) == M.end());
+}
+
+TEST(IntervalMapTest, InsertSingleElement) {
+ // Check that a map with a single element inserted behaves as expected.
+ IntervalMap<unsigned, unsigned, IntervalCoalescing::Enabled> M;
+
+ M.insert(7, 8, 42);
+
+ EXPECT_FALSE(M.empty());
+ EXPECT_EQ(std::next(M.begin()), M.end());
+ EXPECT_EQ(M.find(7), M.begin());
+ EXPECT_EQ(M.find(8), M.end());
+ EXPECT_EQ(M.lookup(7), 42U);
+ EXPECT_EQ(M.lookup(8), 0U); // 8 not present, so should return unsigned().
+}
+
+TEST(IntervalMapTest, InsertCoalesceWithPrevious) {
+ // Check that insertions coalesce with previous ranges that share the same
+ // value. Also check that they _don't_ coalesce if the values are different.
+
+ // Check that insertion coalesces with previous range when values are equal.
+ IntervalMap<unsigned, unsigned, IntervalCoalescing::Enabled> M1;
+
+ M1.insert(7, 8, 42);
+ M1.insert(8, 9, 42);
+
+ EXPECT_FALSE(M1.empty());
+ EXPECT_EQ(std::next(M1.begin()), M1.end()); // Should see just one range.
+ EXPECT_EQ(M1.find(7), M1.find(8)); // 7 and 8 should point to same range.
+ EXPECT_EQ(M1.lookup(7), 42U); // Value should be preserved.
+
+ // Check that insertion does not coalesce with previous range when values are
+ // not equal.
+ IntervalMap<unsigned, unsigned, IntervalCoalescing::Enabled> M2;
+
+ M2.insert(7, 8, 42);
+ M2.insert(8, 9, 7);
+
+ EXPECT_FALSE(M2.empty());
+ EXPECT_EQ(std::next(std::next(M2.begin())), M2.end()); // Expect two ranges.
+ EXPECT_NE(M2.find(7), M2.find(8)); // 7 and 8 should be different ranges.
+ EXPECT_EQ(M2.lookup(7), 42U); // Keys 7 and 8 should map to different values.
+ EXPECT_EQ(M2.lookup(8), 7U);
+}
+
+TEST(IntervalMapTest, InsertCoalesceWithFollowing) {
+ // Check that insertions coalesce with following ranges that share the same
+ // value. Also check that they _don't_ coalesce if the values are different.
+
+ // Check that insertion coalesces with following range when values are equal.
+ IntervalMap<unsigned, unsigned, IntervalCoalescing::Enabled> M1;
+
+ M1.insert(8, 9, 42);
+ M1.insert(7, 8, 42);
+
+ EXPECT_FALSE(M1.empty());
+ EXPECT_EQ(std::next(M1.begin()), M1.end()); // Should see just one range.
+ EXPECT_EQ(M1.find(7), M1.find(8)); // 7 and 8 should point to same range.
+ EXPECT_EQ(M1.lookup(7), 42U); // Value should be preserved.
+
+ // Check that insertion does not coalesce with previous range when values are
+ // not equal.
+ IntervalMap<unsigned, unsigned, IntervalCoalescing::Enabled> M2;
+
+ M2.insert(8, 9, 42);
+ M2.insert(7, 8, 7);
+
+ EXPECT_FALSE(M2.empty());
+ EXPECT_EQ(std::next(std::next(M2.begin())), M2.end()); // Expect two ranges.
+ EXPECT_EQ(M2.lookup(7), 7U); // Keys 7 and 8 should map to different values.
+ EXPECT_EQ(M2.lookup(8), 42U);
+}
+
+TEST(IntervalMapTest, InsertCoalesceBoth) {
+ // Check that insertions coalesce with ranges on both sides where posssible.
+ // Also check that they _don't_ coalesce if the values are different.
+
+ // Check that insertion coalesces with both previous and following ranges
+ // when values are equal.
+ IntervalMap<unsigned, unsigned, IntervalCoalescing::Enabled> M1;
+
+ M1.insert(7, 8, 42);
+ M1.insert(9, 10, 42);
+
+ // Check no coalescing yet.
+ EXPECT_NE(M1.find(7), M1.find(9));
+
+ // Insert a 3rd range to trigger coalescing on both sides.
+ M1.insert(8, 9, 42);
+
+ EXPECT_FALSE(M1.empty());
+ EXPECT_EQ(std::next(M1.begin()), M1.end()); // Should see just one range.
+ EXPECT_EQ(M1.find(7), M1.find(8)); // 7, 8, and 9 should point to same range.
+ EXPECT_EQ(M1.find(8), M1.find(9));
+ EXPECT_EQ(M1.lookup(7), 42U); // Value should be preserved.
+
+ // Check that insertion does not coalesce with previous range when values are
+ // not equal.
+ IntervalMap<unsigned, unsigned, IntervalCoalescing::Enabled> M2;
+
+ M2.insert(7, 8, 42);
+ M2.insert(8, 9, 7);
+ M2.insert(9, 10, 42);
+
+ EXPECT_FALSE(M2.empty());
+ // Expect three ranges.
+ EXPECT_EQ(std::next(std::next(std::next(M2.begin()))), M2.end());
+ EXPECT_NE(M2.find(7), M2.find(8)); // All keys should map to different ranges.
+ EXPECT_NE(M2.find(8), M2.find(9));
+ EXPECT_EQ(M2.lookup(7), 42U); // Key 7, 8, and 9 should map to different vals.
+ EXPECT_EQ(M2.lookup(8), 7U);
+ EXPECT_EQ(M2.lookup(9), 42U);
+}
+
+TEST(IntervalMapTest, EraseSingleElement) {
+ // Check that we can insert and then remove a single range.
+ IntervalMap<unsigned, unsigned, IntervalCoalescing::Enabled> M;
+
+ M.insert(7, 10, 42);
+ EXPECT_FALSE(M.empty());
+ M.erase(7, 10);
+ EXPECT_TRUE(M.empty());
+}
+
+TEST(IntervalMapTest, EraseSplittingLeft) {
+ // Check that removal of a trailing subrange succeeds, but leaves the
+ // residual range in-place.
+ IntervalMap<unsigned, unsigned, IntervalCoalescing::Enabled> M;
+
+ M.insert(7, 10, 42);
+ EXPECT_FALSE(M.empty());
+ M.erase(9, 10);
+ EXPECT_EQ(std::next(M.begin()), M.end());
+ EXPECT_EQ(M.begin()->first.first, 7U);
+ EXPECT_EQ(M.begin()->first.second, 9U);
+}
+
+TEST(IntervalMapTest, EraseSplittingRight) {
+ // Check that removal of a leading subrange succeeds, but leaves the
+ // residual range in-place.
+ IntervalMap<unsigned, unsigned, IntervalCoalescing::Enabled> M;
+
+ M.insert(7, 10, 42);
+ EXPECT_FALSE(M.empty());
+ M.erase(7, 8);
+ EXPECT_EQ(std::next(M.begin()), M.end());
+ EXPECT_EQ(M.begin()->first.first, 8U);
+ EXPECT_EQ(M.begin()->first.second, 10U);
+}
+
+TEST(IntervalMapTest, EraseSplittingBoth) {
+ // Check that removal of an interior subrange leaves both the leading and
+ // trailing residual subranges in-place.
+ IntervalMap<unsigned, unsigned, IntervalCoalescing::Enabled> M;
+
+ M.insert(7, 10, 42);
+ EXPECT_FALSE(M.empty());
+ M.erase(8, 9);
+ EXPECT_EQ(std::next(std::next(M.begin())), M.end());
+ EXPECT_EQ(M.begin()->first.first, 7U);
+ EXPECT_EQ(M.begin()->first.second, 8U);
+ EXPECT_EQ(std::next(M.begin())->first.first, 9U);
+ EXPECT_EQ(std::next(M.begin())->first.second, 10U);
+}
+
+TEST(IntervalMapTest, NonCoalescingMapPermitsNonComparableKeys) {
+ // Test that values that can't be equality-compared are still usable when
+ // coalescing is disabled and behave as expected.
+
+ struct S {}; // Struct with no equality comparison.
+
+ IntervalMap<unsigned, S, IntervalCoalescing::Disabled> M;
+
+ M.insert(7, 8, S());
+
+ EXPECT_FALSE(M.empty());
+ EXPECT_EQ(std::next(M.begin()), M.end());
+ EXPECT_EQ(M.find(7), M.begin());
+ EXPECT_EQ(M.find(8), M.end());
+}
--- /dev/null
+//===-- interval_set_test.cpp ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime.
+//
+//===----------------------------------------------------------------------===//
+
+#include "interval_set.h"
+#include "gtest/gtest.h"
+
+using namespace __orc_rt;
+
+TEST(IntervalSetTest, DefaultConstructed) {
+ // Check that a default-constructed IntervalSet behaves as expected.
+ IntervalSet<unsigned, IntervalCoalescing::Enabled> S;
+
+ EXPECT_TRUE(S.empty());
+ EXPECT_TRUE(S.begin() == S.end());
+ EXPECT_TRUE(S.find(0) == S.end());
+}
+
+TEST(IntervalSetTest, InsertSingleElement) {
+ // Check that a set with a single element inserted behaves as expected.
+ IntervalSet<unsigned, IntervalCoalescing::Enabled> S;
+
+ S.insert(7, 8);
+
+ EXPECT_FALSE(S.empty());
+ EXPECT_EQ(std::next(S.begin()), S.end());
+ EXPECT_EQ(S.find(7), S.begin());
+ EXPECT_EQ(S.find(8), S.end());
+}
+
+TEST(IntervalSetTest, InsertCoalesceWithPrevious) {
+ // Check that insertions coalesce with previous ranges.
+ IntervalSet<unsigned, IntervalCoalescing::Enabled> S;
+
+ S.insert(7, 8);
+ S.insert(8, 9);
+
+ EXPECT_FALSE(S.empty());
+ EXPECT_EQ(std::next(S.begin()), S.end()); // Should see just one range.
+ EXPECT_EQ(S.find(7), S.find(8)); // 7 and 8 should point to same range.
+}
+
+TEST(IntervalSetTest, InsertCoalesceWithFollowing) {
+ // Check that insertions coalesce with following ranges.
+ IntervalSet<unsigned, IntervalCoalescing::Enabled> S;
+
+ S.insert(8, 9);
+ S.insert(7, 8);
+
+ EXPECT_FALSE(S.empty());
+ EXPECT_EQ(std::next(S.begin()), S.end()); // Should see just one range.
+ EXPECT_EQ(S.find(7), S.find(8)); // 7 and 8 should point to same range.
+}
+
+TEST(IntervalSetTest, InsertCoalesceBoth) {
+ // Check that insertions coalesce with ranges on both sides.
+ IntervalSet<unsigned, IntervalCoalescing::Enabled> S;
+
+ S.insert(7, 8);
+ S.insert(9, 10);
+
+ // Check no coalescing yet.
+ EXPECT_NE(S.find(7), S.find(9));
+
+ // Insert a 3rd range to trigger coalescing on both sides.
+ S.insert(8, 9);
+
+ EXPECT_FALSE(S.empty());
+ EXPECT_EQ(std::next(S.begin()), S.end()); // Should see just one range.
+ EXPECT_EQ(S.find(7), S.find(8)); // 7, 8, and 9 should point to same range.
+ EXPECT_EQ(S.find(8), S.find(9));
+}
+
+TEST(IntervalSetTest, EraseSplittingLeft) {
+ // Check that removal of a trailing subrange succeeds, but leaves the
+ // residual range in-place.
+ IntervalSet<unsigned, IntervalCoalescing::Enabled> S;
+
+ S.insert(7, 10);
+ EXPECT_FALSE(S.empty());
+ S.erase(9, 10);
+ EXPECT_EQ(std::next(S.begin()), S.end());
+ EXPECT_EQ(S.begin()->first, 7U);
+ EXPECT_EQ(S.begin()->second, 9U);
+}
+
+TEST(IntervalSetTest, EraseSplittingRight) {
+ // Check that removal of a leading subrange succeeds, but leaves the
+ // residual range in-place.
+ IntervalSet<unsigned, IntervalCoalescing::Enabled> S;
+
+ S.insert(7, 10);
+ EXPECT_FALSE(S.empty());
+ S.erase(7, 8);
+ EXPECT_EQ(std::next(S.begin()), S.end());
+ EXPECT_EQ(S.begin()->first, 8U);
+ EXPECT_EQ(S.begin()->second, 10U);
+}
+
+TEST(IntervalSetTest, EraseSplittingBoth) {
+ // Check that removal of an interior subrange leaves both the leading and
+ // trailing residual subranges in-place.
+ IntervalSet<unsigned, IntervalCoalescing::Enabled> S;
+
+ S.insert(7, 10);
+ EXPECT_FALSE(S.empty());
+ S.erase(8, 9);
+ EXPECT_EQ(std::next(std::next(S.begin())), S.end());
+ EXPECT_EQ(S.begin()->first, 7U);
+ EXPECT_EQ(S.begin()->second, 8U);
+ EXPECT_EQ(std::next(S.begin())->first, 9U);
+ EXPECT_EQ(std::next(S.begin())->second, 10U);
+}
--- /dev/null
+//===-- orc_unit_test_main.cpp --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime.
+//
+//===----------------------------------------------------------------------===//
+#include "gtest/gtest.h"
+
+int main(int argc, char **argv) {
+ testing::GTEST_FLAG(death_test_style) = "threadsafe";
+ testing::InitGoogleTest(&argc, argv);
+ return RUN_ALL_TESTS();
+}
--- /dev/null
+//===-- simple_packed_serialization_test.cpp ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime.
+//
+//===----------------------------------------------------------------------===//
+
+#include "simple_packed_serialization.h"
+#include "gtest/gtest.h"
+
+using namespace __orc_rt;
+
+TEST(SimplePackedSerializationTest, SPSOutputBuffer) {
+ constexpr unsigned NumBytes = 8;
+ char Buffer[NumBytes];
+ char Zero = 0;
+ SPSOutputBuffer OB(Buffer, NumBytes);
+
+ // Expect that we can write NumBytes of content.
+ for (unsigned I = 0; I != NumBytes; ++I) {
+ char C = I;
+ EXPECT_TRUE(OB.write(&C, 1));
+ }
+
+ // Expect an error when we attempt to write an extra byte.
+ EXPECT_FALSE(OB.write(&Zero, 1));
+
+ // Check that the buffer contains the expected content.
+ for (unsigned I = 0; I != NumBytes; ++I)
+ EXPECT_EQ(Buffer[I], (char)I);
+}
+
+TEST(SimplePackedSerializationTest, SPSInputBuffer) {
+ char Buffer[] = {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07};
+ SPSInputBuffer IB(Buffer, sizeof(Buffer));
+
+ char C;
+ for (unsigned I = 0; I != sizeof(Buffer); ++I) {
+ EXPECT_TRUE(IB.read(&C, 1));
+ EXPECT_EQ(C, (char)I);
+ }
+
+ EXPECT_FALSE(IB.read(&C, 1));
+}
+
+template <typename SPSTagT, typename T>
+static void blobSerializationRoundTrip(const T &Value) {
+ using BST = SPSSerializationTraits<SPSTagT, T>;
+
+ size_t Size = BST::size(Value);
+ auto Buffer = std::make_unique<char[]>(Size);
+ SPSOutputBuffer OB(Buffer.get(), Size);
+
+ EXPECT_TRUE(BST::serialize(OB, Value));
+
+ SPSInputBuffer IB(Buffer.get(), Size);
+
+ T DSValue;
+ EXPECT_TRUE(BST::deserialize(IB, DSValue));
+
+ EXPECT_EQ(Value, DSValue)
+ << "Incorrect value after serialization/deserialization round-trip";
+}
+
+template <typename T> static void testFixedIntegralTypeSerialization() {
+ blobSerializationRoundTrip<T, T>(0);
+ blobSerializationRoundTrip<T, T>(static_cast<T>(1));
+ if (std::is_signed<T>::value) {
+ blobSerializationRoundTrip<T, T>(static_cast<T>(-1));
+ blobSerializationRoundTrip<T, T>(std::numeric_limits<T>::min());
+ }
+ blobSerializationRoundTrip<T, T>(std::numeric_limits<T>::max());
+}
+
+TEST(SimplePackedSerializationTest, BoolSerialization) {
+ blobSerializationRoundTrip<bool, bool>(true);
+ blobSerializationRoundTrip<bool, bool>(false);
+}
+
+TEST(SimplePackedSerializationTest, CharSerialization) {
+ blobSerializationRoundTrip<char, char>((char)0x00);
+ blobSerializationRoundTrip<char, char>((char)0xAA);
+ blobSerializationRoundTrip<char, char>((char)0xFF);
+}
+
+TEST(SimplePackedSerializationTest, Int8Serialization) {
+ testFixedIntegralTypeSerialization<int8_t>();
+}
+
+TEST(SimplePackedSerializationTest, UInt8Serialization) {
+ testFixedIntegralTypeSerialization<uint8_t>();
+}
+
+TEST(SimplePackedSerializationTest, Int16Serialization) {
+ testFixedIntegralTypeSerialization<int16_t>();
+}
+
+TEST(SimplePackedSerializationTest, UInt16Serialization) {
+ testFixedIntegralTypeSerialization<uint16_t>();
+}
+
+TEST(SimplePackedSerializationTest, Int32Serialization) {
+ testFixedIntegralTypeSerialization<int32_t>();
+}
+
+TEST(SimplePackedSerializationTest, UInt32Serialization) {
+ testFixedIntegralTypeSerialization<uint32_t>();
+}
+
+TEST(SimplePackedSerializationTest, Int64Serialization) {
+ testFixedIntegralTypeSerialization<int64_t>();
+}
+
+TEST(SimplePackedSerializationTest, UInt64Serialization) {
+ testFixedIntegralTypeSerialization<uint64_t>();
+}
+
+TEST(SimplePackedSerializationTest, SequenceSerialization) {
+ std::vector<int32_t> V({1, 2, -47, 139});
+ blobSerializationRoundTrip<SPSSequence<int32_t>, std::vector<int32_t>>(V);
+}
+
+TEST(SimplePackedSerializationTest, StringViewCharSequenceSerialization) {
+ const char *HW = "Hello, world!";
+ blobSerializationRoundTrip<SPSString, std::string_view>(std::string_view(HW));
+}
+
+TEST(SimplePackedSerializationTest, SpanSerialization) {
+ const char Data[] = {3, 2, 1, 0, 1, 2, 3}; // Span should handle nulls.
+ span<const char> OutS(Data, sizeof(Data));
+
+ size_t Size = SPSArgList<SPSSequence<char>>::size(OutS);
+ auto Buffer = std::make_unique<char[]>(Size);
+ SPSOutputBuffer OB(Buffer.get(), Size);
+
+ EXPECT_TRUE(SPSArgList<SPSSequence<char>>::serialize(OB, OutS));
+
+ SPSInputBuffer IB(Buffer.get(), Size);
+
+ span<const char> InS;
+
+ EXPECT_TRUE(SPSArgList<SPSSequence<char>>::deserialize(IB, InS));
+
+ // Check that the serialized and deserialized values match.
+ EXPECT_EQ(InS.size(), OutS.size());
+ EXPECT_EQ(memcmp(OutS.data(), InS.data(), InS.size()), 0);
+
+ // Check that the span points directly to the input buffer.
+ EXPECT_EQ(InS.data(), Buffer.get() + sizeof(uint64_t));
+}
+
+TEST(SimplePackedSerializationTest, StdPairSerialization) {
+ std::pair<int32_t, std::string> P(42, "foo");
+ blobSerializationRoundTrip<SPSTuple<int32_t, SPSString>,
+ std::pair<int32_t, std::string>>(P);
+}
+
+TEST(SimplePackedSerializationTest, StdOptionalNoValueSerialization) {
+ std::optional<int64_t> NoValue;
+ blobSerializationRoundTrip<SPSOptional<int64_t>>(NoValue);
+}
+
+TEST(SimplePackedSerializationTest, StdOptionalValueSerialization) {
+ std::optional<int64_t> Value(42);
+ blobSerializationRoundTrip<SPSOptional<int64_t>>(Value);
+}
+
+TEST(SimplePackedSerializationTest, ArgListSerialization) {
+ using BAL = SPSArgList<bool, int32_t, SPSString>;
+
+ bool Arg1 = true;
+ int32_t Arg2 = 42;
+ std::string Arg3 = "foo";
+
+ size_t Size = BAL::size(Arg1, Arg2, Arg3);
+ auto Buffer = std::make_unique<char[]>(Size);
+ SPSOutputBuffer OB(Buffer.get(), Size);
+
+ EXPECT_TRUE(BAL::serialize(OB, Arg1, Arg2, Arg3));
+
+ SPSInputBuffer IB(Buffer.get(), Size);
+
+ bool ArgOut1;
+ int32_t ArgOut2;
+ std::string ArgOut3;
+
+ EXPECT_TRUE(BAL::deserialize(IB, ArgOut1, ArgOut2, ArgOut3));
+
+ EXPECT_EQ(Arg1, ArgOut1);
+ EXPECT_EQ(Arg2, ArgOut2);
+ EXPECT_EQ(Arg3, ArgOut3);
+}
--- /dev/null
+//===---------- string_pool_test.cpp - Unit tests for StringPool ----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "string_pool.h"
+#include "gtest/gtest.h"
+
+using namespace __orc_rt;
+
+namespace {
+
+TEST(StringPool, UniquingAndComparisons) {
+ StringPool SP;
+ auto P1 = SP.intern("hello");
+
+ std::string S("hel");
+ S += "lo";
+ auto P2 = SP.intern(S);
+
+ auto P3 = SP.intern("goodbye");
+
+ EXPECT_EQ(P1, P2) << "Failed to unique entries";
+ EXPECT_NE(P1, P3) << "Unequal pooled strings comparing equal";
+
+ // We want to test that less-than comparison of PooledStringPtrs compiles,
+ // however we can't test the actual result as this is a pointer comparison and
+ // PooledStringPtr doesn't expose the underlying address of the string.
+ (void)(P1 < P3);
+}
+
+TEST(StringPool, Dereference) {
+ StringPool SP;
+ auto Foo = SP.intern("foo");
+ EXPECT_EQ(*Foo, "foo") << "Equality on dereferenced string failed";
+}
+
+TEST(StringPool, ClearDeadEntries) {
+ StringPool SP;
+ {
+ auto P1 = SP.intern("s1");
+ SP.clearDeadEntries();
+ EXPECT_FALSE(SP.empty()) << "\"s1\" entry in pool should still be retained";
+ }
+ SP.clearDeadEntries();
+ EXPECT_TRUE(SP.empty()) << "pool should be empty";
+}
+
+TEST(StringPool, NullPtr) {
+ // Make sure that we can default construct and then destroy a null
+ // PooledStringPtr.
+ PooledStringPtr Null;
+}
+
+TEST(StringPool, Hashable) {
+ StringPool SP;
+ PooledStringPtr P1 = SP.intern("s1");
+ PooledStringPtr Null;
+ EXPECT_NE(std::hash<PooledStringPtr>()(P1),
+ std::hash<PooledStringPtr>()(Null));
+}
+
+} // end anonymous namespace
--- /dev/null
+//===-- wrapper_function_utils_test.cpp -----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime.
+//
+//===----------------------------------------------------------------------===//
+
+#include "wrapper_function_utils.h"
+#include "gtest/gtest.h"
+
+using namespace __orc_rt;
+
+namespace {
+constexpr const char *TestString = "test string";
+} // end anonymous namespace
+
+TEST(WrapperFunctionUtilsTest, DefaultWrapperFunctionResult) {
+ WrapperFunctionResult R;
+ EXPECT_TRUE(R.empty());
+ EXPECT_EQ(R.size(), 0U);
+ EXPECT_EQ(R.getOutOfBandError(), nullptr);
+}
+
+TEST(WrapperFunctionUtilsTest, WrapperFunctionResultFromCStruct) {
+ __orc_rt_CWrapperFunctionResult CR =
+ __orc_rt_CreateCWrapperFunctionResultFromString(TestString);
+ WrapperFunctionResult R(CR);
+ EXPECT_EQ(R.size(), strlen(TestString) + 1);
+ EXPECT_TRUE(strcmp(R.data(), TestString) == 0);
+ EXPECT_FALSE(R.empty());
+ EXPECT_EQ(R.getOutOfBandError(), nullptr);
+}
+
+TEST(WrapperFunctionUtilsTest, WrapperFunctionResultFromRange) {
+ auto R = WrapperFunctionResult::copyFrom(TestString, strlen(TestString) + 1);
+ EXPECT_EQ(R.size(), strlen(TestString) + 1);
+ EXPECT_TRUE(strcmp(R.data(), TestString) == 0);
+ EXPECT_FALSE(R.empty());
+ EXPECT_EQ(R.getOutOfBandError(), nullptr);
+}
+
+TEST(WrapperFunctionUtilsTest, WrapperFunctionResultFromCString) {
+ auto R = WrapperFunctionResult::copyFrom(TestString);
+ EXPECT_EQ(R.size(), strlen(TestString) + 1);
+ EXPECT_TRUE(strcmp(R.data(), TestString) == 0);
+ EXPECT_FALSE(R.empty());
+ EXPECT_EQ(R.getOutOfBandError(), nullptr);
+}
+
+TEST(WrapperFunctionUtilsTest, WrapperFunctionResultFromStdString) {
+ auto R = WrapperFunctionResult::copyFrom(std::string(TestString));
+ EXPECT_EQ(R.size(), strlen(TestString) + 1);
+ EXPECT_TRUE(strcmp(R.data(), TestString) == 0);
+ EXPECT_FALSE(R.empty());
+ EXPECT_EQ(R.getOutOfBandError(), nullptr);
+}
+
+TEST(WrapperFunctionUtilsTest, WrapperFunctionResultFromOutOfBandError) {
+ auto R = WrapperFunctionResult::createOutOfBandError(TestString);
+ EXPECT_FALSE(R.empty());
+ EXPECT_TRUE(strcmp(R.getOutOfBandError(), TestString) == 0);
+}
+
+TEST(WrapperFunctionUtilsTest, WrapperFunctionCCallCreateEmpty) {
+ EXPECT_TRUE(!!WrapperFunctionCall::Create<SPSArgList<>>(ExecutorAddr()));
+}
+
+static void voidNoop() {}
+
+static __orc_rt_CWrapperFunctionResult voidNoopWrapper(const char *ArgData,
+ size_t ArgSize) {
+ return WrapperFunction<void()>::handle(ArgData, ArgSize, voidNoop).release();
+}
+
+static __orc_rt_CWrapperFunctionResult addWrapper(const char *ArgData,
+ size_t ArgSize) {
+ return WrapperFunction<int32_t(int32_t, int32_t)>::handle(
+ ArgData, ArgSize,
+ [](int32_t X, int32_t Y) -> int32_t { return X + Y; })
+ .release();
+}
+
+extern "C" __orc_rt_Opaque __orc_rt_jit_dispatch_ctx{};
+
+extern "C" __orc_rt_CWrapperFunctionResult
+__orc_rt_jit_dispatch(__orc_rt_Opaque *Ctx, const void *FnTag,
+ const char *ArgData, size_t ArgSize) {
+ using WrapperFunctionType =
+ __orc_rt_CWrapperFunctionResult (*)(const char *, size_t);
+
+ return reinterpret_cast<WrapperFunctionType>(const_cast<void *>(FnTag))(
+ ArgData, ArgSize);
+}
+
+TEST(WrapperFunctionUtilsTest, WrapperFunctionCallVoidNoopAndHandle) {
+ EXPECT_FALSE(!!WrapperFunction<void()>::call((void *)&voidNoopWrapper));
+}
+
+TEST(WrapperFunctionUtilsTest, WrapperFunctionCallAddWrapperAndHandle) {
+ int32_t Result;
+ EXPECT_FALSE(!!WrapperFunction<int32_t(int32_t, int32_t)>::call(
+ (void *)&addWrapper, Result, 1, 2));
+ EXPECT_EQ(Result, (int32_t)3);
+}
+
+class AddClass {
+public:
+ AddClass(int32_t X) : X(X) {}
+ int32_t addMethod(int32_t Y) { return X + Y; }
+
+private:
+ int32_t X;
+};
+
+static __orc_rt_CWrapperFunctionResult addMethodWrapper(const char *ArgData,
+ size_t ArgSize) {
+ return WrapperFunction<int32_t(SPSExecutorAddr, int32_t)>::handle(
+ ArgData, ArgSize, makeMethodWrapperHandler(&AddClass::addMethod))
+ .release();
+}
+
+TEST(WrapperFunctionUtilsTest, WrapperFunctionMethodCallAndHandleRet) {
+ int32_t Result;
+ AddClass AddObj(1);
+ EXPECT_FALSE(!!WrapperFunction<int32_t(SPSExecutorAddr, int32_t)>::call(
+ (void *)&addMethodWrapper, Result, ExecutorAddr::fromPtr(&AddObj), 2));
+ EXPECT_EQ(Result, (int32_t)3);
+}
+
+static __orc_rt_CWrapperFunctionResult sumArrayWrapper(const char *ArgData,
+ size_t ArgSize) {
+ return WrapperFunction<int8_t(SPSExecutorAddrRange)>::handle(
+ ArgData, ArgSize,
+ [](ExecutorAddrRange R) {
+ int8_t Sum = 0;
+ for (char C : R.toSpan<char>())
+ Sum += C;
+ return Sum;
+ })
+ .release();
+}
+
+TEST(WrapperFunctionUtilsTest, SerializedWrapperFunctionCallTest) {
+ {
+ // Check wrapper function calls.
+ char A[] = {1, 2, 3, 4};
+
+ auto WFC =
+ cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddrRange>>(
+ ExecutorAddr::fromPtr(sumArrayWrapper),
+ ExecutorAddrRange(ExecutorAddr::fromPtr(A),
+ ExecutorAddrDiff(sizeof(A)))));
+
+ WrapperFunctionResult WFR(WFC.run());
+ EXPECT_EQ(WFR.size(), 1U);
+ EXPECT_EQ(WFR.data()[0], 10);
+ }
+
+ {
+ // Check calls to void functions.
+ auto WFC =
+ cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddrRange>>(
+ ExecutorAddr::fromPtr(voidNoopWrapper), ExecutorAddrRange()));
+ auto Err = WFC.runWithSPSRet<void>();
+ EXPECT_FALSE(!!Err);
+ }
+
+ {
+ // Check calls with arguments and return values.
+ auto WFC =
+ cantFail(WrapperFunctionCall::Create<SPSArgList<int32_t, int32_t>>(
+ ExecutorAddr::fromPtr(addWrapper), 2, 4));
+
+ int32_t Result = 0;
+ auto Err = WFC.runWithSPSRet<int32_t>(Result);
+ EXPECT_FALSE(!!Err);
+ EXPECT_EQ(Result, 6);
+ }
+}
#ifndef ORC_RT_WRAPPER_FUNCTION_UTILS_H
#define ORC_RT_WRAPPER_FUNCTION_UTILS_H
-#include "c_api.h"
+#include "orc_rt/c_api.h"
#include "common.h"
#include "error.h"
+#include "executor_address.h"
#include "simple_packed_serialization.h"
#include <type_traits>
}
/// Get a pointer to the data contained in this instance.
- const char *data() const { return __orc_rt_CWrapperFunctionResultData(&R); }
+ char *data() { return __orc_rt_CWrapperFunctionResultData(&R); }
/// Returns the size of the data contained in this instance.
size_t size() const { return __orc_rt_CWrapperFunctionResultSize(&R); }
/// Create a WrapperFunctionResult with the given size and return a pointer
/// to the underlying memory.
- static char *allocate(WrapperFunctionResult &R, size_t Size) {
- __orc_rt_DisposeCWrapperFunctionResult(&R.R);
- __orc_rt_CWrapperFunctionResultInit(&R.R);
- return __orc_rt_CWrapperFunctionResultAllocate(&R.R, Size);
+ static WrapperFunctionResult allocate(size_t Size) {
+ WrapperFunctionResult R;
+ R.R = __orc_rt_CWrapperFunctionResultAllocate(Size);
+ return R;
}
/// Copy from the given char range.
return createOutOfBandError(Msg.c_str());
}
+ template <typename SPSArgListT, typename... ArgTs>
+ static WrapperFunctionResult fromSPSArgs(const ArgTs &...Args) {
+ auto Result = allocate(SPSArgListT::size(Args...));
+ SPSOutputBuffer OB(Result.data(), Result.size());
+ if (!SPSArgListT::serialize(OB, Args...))
+ return createOutOfBandError(
+ "Error serializing arguments to blob in call");
+ return Result;
+ }
+
/// If this value is an out-of-band error then this returns the error message,
/// otherwise returns nullptr.
const char *getOutOfBandError() const {
namespace detail {
-template <typename SPSArgListT, typename... ArgTs>
-Expected<WrapperFunctionResult>
-serializeViaSPSToWrapperFunctionResult(const ArgTs &...Args) {
- WrapperFunctionResult Result;
- char *DataPtr =
- WrapperFunctionResult::allocate(Result, SPSArgListT::size(Args...));
- SPSOutputBuffer OB(DataPtr, Result.size());
- if (!SPSArgListT::serialize(OB, Args...))
- return make_error<StringError>(
- "Error serializing arguments to blob in call");
- return std::move(Result);
-}
-
template <typename RetT> class WrapperFunctionHandlerCaller {
public:
template <typename HandlerT, typename ArgTupleT, std::size_t... I>
auto HandlerResult = WrapperFunctionHandlerCaller<RetT>::call(
std::forward<HandlerT>(H), Args, ArgIndices{});
- if (auto Result = ResultSerializer<decltype(HandlerResult)>::serialize(
- std::move(HandlerResult)))
- return std::move(*Result);
- else
- return WrapperFunctionResult::createOutOfBandError(
- toString(Result.takeError()));
+ return ResultSerializer<decltype(HandlerResult)>::serialize(
+ std::move(HandlerResult));
}
private:
SPSInputBuffer IB(ArgData, ArgSize);
return SPSArgList<SPSTagTs...>::deserialize(IB, std::get<I>(Args)...);
}
-
};
-// Map function references to function types.
+// Map function pointers to function types.
template <typename RetT, typename... ArgTs,
template <typename> class ResultSerializer, typename... SPSTagTs>
-class WrapperFunctionHandlerHelper<RetT (&)(ArgTs...), ResultSerializer,
+class WrapperFunctionHandlerHelper<RetT (*)(ArgTs...), ResultSerializer,
SPSTagTs...>
: public WrapperFunctionHandlerHelper<RetT(ArgTs...), ResultSerializer,
SPSTagTs...> {};
template <typename SPSRetTagT, typename RetT> class ResultSerializer {
public:
- static Expected<WrapperFunctionResult> serialize(RetT Result) {
- return serializeViaSPSToWrapperFunctionResult<SPSArgList<SPSRetTagT>>(
- Result);
+ static WrapperFunctionResult serialize(RetT Result) {
+ return WrapperFunctionResult::fromSPSArgs<SPSArgList<SPSRetTagT>>(Result);
}
};
template <typename SPSRetTagT> class ResultSerializer<SPSRetTagT, Error> {
public:
- static Expected<WrapperFunctionResult> serialize(Error Err) {
- return serializeViaSPSToWrapperFunctionResult<SPSArgList<SPSRetTagT>>(
+ static WrapperFunctionResult serialize(Error Err) {
+ return WrapperFunctionResult::fromSPSArgs<SPSArgList<SPSRetTagT>>(
toSPSSerializable(std::move(Err)));
}
};
template <typename SPSRetTagT, typename T>
class ResultSerializer<SPSRetTagT, Expected<T>> {
public:
- static Expected<WrapperFunctionResult> serialize(Expected<T> E) {
- return serializeViaSPSToWrapperFunctionResult<SPSArgList<SPSRetTagT>>(
+ static WrapperFunctionResult serialize(Expected<T> E) {
+ return WrapperFunctionResult::fromSPSArgs<SPSArgList<SPSRetTagT>>(
toSPSSerializable(std::move(E)));
}
};
return make_error<StringError>("__orc_rt_jit_dispatch not set");
auto ArgBuffer =
- detail::serializeViaSPSToWrapperFunctionResult<SPSArgList<SPSTagTs...>>(
- Args...);
- if (!ArgBuffer)
- return ArgBuffer.takeError();
-
- WrapperFunctionResult ResultBuffer =
- __orc_rt_jit_dispatch(&__orc_rt_jit_dispatch_ctx, FnTag,
- ArgBuffer->data(), ArgBuffer->size());
+ WrapperFunctionResult::fromSPSArgs<SPSArgList<SPSTagTs...>>(Args...);
+ if (const char *ErrMsg = ArgBuffer.getOutOfBandError())
+ return make_error<StringError>(ErrMsg);
+
+ WrapperFunctionResult ResultBuffer = __orc_rt_jit_dispatch(
+ &__orc_rt_jit_dispatch_ctx, FnTag, ArgBuffer.data(), ArgBuffer.size());
if (auto ErrMsg = ResultBuffer.getOutOfBandError())
return make_error<StringError>(ErrMsg);
static WrapperFunctionResult handle(const char *ArgData, size_t ArgSize,
HandlerT &&Handler) {
using WFHH =
- detail::WrapperFunctionHandlerHelper<HandlerT, ResultSerializer,
- SPSTagTs...>;
+ detail::WrapperFunctionHandlerHelper<std::remove_reference_t<HandlerT>,
+ ResultSerializer, SPSTagTs...>;
return WFHH::apply(std::forward<HandlerT>(Handler), ArgData, ArgSize);
}
using WrapperFunction<SPSEmpty(SPSTagTs...)>::handle;
};
+/// A function object that takes an ExecutorAddr as its first argument,
+/// casts that address to a ClassT*, then calls the given method on that
+/// pointer passing in the remaining function arguments. This utility
+/// removes some of the boilerplate from writing wrappers for method calls.
+///
+/// @code{.cpp}
+/// class MyClass {
+/// public:
+/// void myMethod(uint32_t, bool) { ... }
+/// };
+///
+/// // SPS Method signature -- note MyClass object address as first argument.
+/// using SPSMyMethodWrapperSignature =
+/// SPSTuple<SPSExecutorAddr, uint32_t, bool>;
+///
+/// WrapperFunctionResult
+/// myMethodCallWrapper(const char *ArgData, size_t ArgSize) {
+/// return WrapperFunction<SPSMyMethodWrapperSignature>::handle(
+/// ArgData, ArgSize, makeMethodWrapperHandler(&MyClass::myMethod));
+/// }
+/// @endcode
+///
+template <typename RetT, typename ClassT, typename... ArgTs>
+class MethodWrapperHandler {
+public:
+ using MethodT = RetT (ClassT::*)(ArgTs...);
+ MethodWrapperHandler(MethodT M) : M(M) {}
+ RetT operator()(ExecutorAddr ObjAddr, ArgTs &...Args) {
+ return (ObjAddr.toPtr<ClassT *>()->*M)(std::forward<ArgTs>(Args)...);
+ }
+
+private:
+ MethodT M;
+};
+
+/// Create a MethodWrapperHandler object from the given method pointer.
+template <typename RetT, typename ClassT, typename... ArgTs>
+MethodWrapperHandler<RetT, ClassT, ArgTs...>
+makeMethodWrapperHandler(RetT (ClassT::*Method)(ArgTs...)) {
+ return MethodWrapperHandler<RetT, ClassT, ArgTs...>(Method);
+}
+
+/// Represents a call to a wrapper function.
+class WrapperFunctionCall {
+public:
+ // FIXME: Switch to a SmallVector<char, 24> once ORC runtime has a
+ // smallvector.
+ using ArgDataBufferType = std::vector<char>;
+
+ /// Create a WrapperFunctionCall using the given SPS serializer to serialize
+ /// the arguments.
+ template <typename SPSSerializer, typename... ArgTs>
+ static Expected<WrapperFunctionCall> Create(ExecutorAddr FnAddr,
+ const ArgTs &...Args) {
+ ArgDataBufferType ArgData;
+ ArgData.resize(SPSSerializer::size(Args...));
+ SPSOutputBuffer OB(ArgData.empty() ? nullptr : ArgData.data(),
+ ArgData.size());
+ if (SPSSerializer::serialize(OB, Args...))
+ return WrapperFunctionCall(FnAddr, std::move(ArgData));
+ return make_error<StringError>("Cannot serialize arguments for "
+ "AllocActionCall");
+ }
+
+ WrapperFunctionCall() = default;
+
+ /// Create a WrapperFunctionCall from a target function and arg buffer.
+ WrapperFunctionCall(ExecutorAddr FnAddr, ArgDataBufferType ArgData)
+ : FnAddr(FnAddr), ArgData(std::move(ArgData)) {}
+
+ /// Returns the address to be called.
+ const ExecutorAddr &getCallee() const { return FnAddr; }
+
+ /// Returns the argument data.
+ const ArgDataBufferType &getArgData() const { return ArgData; }
+
+ /// WrapperFunctionCalls convert to true if the callee is non-null.
+ explicit operator bool() const { return !!FnAddr; }
+
+ /// Run call returning raw WrapperFunctionResult.
+ WrapperFunctionResult run() const {
+ using FnTy =
+ __orc_rt_CWrapperFunctionResult(const char *ArgData, size_t ArgSize);
+ return WrapperFunctionResult(
+ FnAddr.toPtr<FnTy *>()(ArgData.data(), ArgData.size()));
+ }
+
+ /// Run call and deserialize result using SPS.
+ template <typename SPSRetT, typename RetT>
+ std::enable_if_t<!std::is_same<SPSRetT, void>::value, Error>
+ runWithSPSRet(RetT &RetVal) const {
+ auto WFR = run();
+ if (const char *ErrMsg = WFR.getOutOfBandError())
+ return make_error<StringError>(ErrMsg);
+ SPSInputBuffer IB(WFR.data(), WFR.size());
+ if (!SPSSerializationTraits<SPSRetT, RetT>::deserialize(IB, RetVal))
+ return make_error<StringError>("Could not deserialize result from "
+ "serialized wrapper function call");
+ return Error::success();
+ }
+
+ /// Overload for SPS functions returning void.
+ template <typename SPSRetT>
+ std::enable_if_t<std::is_same<SPSRetT, void>::value, Error>
+ runWithSPSRet() const {
+ SPSEmpty E;
+ return runWithSPSRet<SPSEmpty>(E);
+ }
+
+ /// Run call and deserialize an SPSError result. SPSError returns and
+ /// deserialization failures are merged into the returned error.
+ Error runWithSPSRetErrorMerged() const {
+ detail::SPSSerializableError RetErr;
+ if (auto Err = runWithSPSRet<SPSError>(RetErr))
+ return Err;
+ return detail::fromSPSSerializable(std::move(RetErr));
+ }
+
+private:
+ ExecutorAddr FnAddr;
+ std::vector<char> ArgData;
+};
+
+using SPSWrapperFunctionCall = SPSTuple<SPSExecutorAddr, SPSSequence<char>>;
+
+template <>
+class SPSSerializationTraits<SPSWrapperFunctionCall, WrapperFunctionCall> {
+public:
+ static size_t size(const WrapperFunctionCall &WFC) {
+ return SPSArgList<SPSExecutorAddr, SPSSequence<char>>::size(
+ WFC.getCallee(), WFC.getArgData());
+ }
+
+ static bool serialize(SPSOutputBuffer &OB, const WrapperFunctionCall &WFC) {
+ return SPSArgList<SPSExecutorAddr, SPSSequence<char>>::serialize(
+ OB, WFC.getCallee(), WFC.getArgData());
+ }
+
+ static bool deserialize(SPSInputBuffer &IB, WrapperFunctionCall &WFC) {
+ ExecutorAddr FnAddr;
+ WrapperFunctionCall::ArgDataBufferType ArgData;
+ if (!SPSWrapperFunctionCall::AsArgList::deserialize(IB, FnAddr, ArgData))
+ return false;
+ WFC = WrapperFunctionCall(FnAddr, std::move(ArgData));
+ return true;
+ }
+};
+
} // end namespace __orc_rt
#endif // ORC_RT_WRAPPER_FUNCTION_UTILS_H
if(WIN32)
list(APPEND PROFILE_SOURCES
WindowsMMap.c
- )
+ )
endif()
include_directories(..)
# We don't use the C++ Standard Library here, so avoid including it by mistake.
append_list_if(COMPILER_RT_HAS_NOSTDINCXX_FLAG -nostdinc++ EXTRA_FLAGS)
# XRay uses C++ standard library headers.
-string(REGEX REPLACE "-stdlib=[a-zA-Z+]*" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
+string(REGEX REPLACE "-stdlib=[a-zA-Z+]*" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
# This appears to be a C-only warning banning the use of locals in aggregate
# initializers. All other compilers accept this, though.
# nonstandard extension used : 'identifier' : cannot be initialized using address of automatic variable
append_list_if(COMPILER_RT_HAS_WD4221_FLAG /wd4221 EXTRA_FLAGS)
+# Disable 'nonstandard extension used: translation unit is empty'.
+append_list_if(COMPILER_RT_HAS_WD4206_FLAG /wd4206 EXTRA_FLAGS)
+
if(APPLE)
add_compiler_rt_runtime(clang_rt.profile
STATIC
|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|* See https://llvm.org/LICENSE.txt for license information.
|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-|*
+|*
|*===----------------------------------------------------------------------===*|
-|*
+|*
|* This file implements the call back routines for the gcov profiling
|* instrumentation pass. Link against this library when running code through
|* the -insert-gcov-profiling LLVM pass.
/*
* The current file we're outputting.
- */
+ */
static FILE *output_file = NULL;
/*
#endif
static int fd = -1;
-typedef void (*fn_ptr)();
+typedef void (*fn_ptr)(void);
typedef void* dynamic_object_id;
// The address of this variable identifies a given dynamic object.
write_32bit_value(hi);
}
-static uint32_t read_32bit_value() {
+static uint32_t read_32bit_value(void) {
uint32_t val;
if (new_file)
return val;
}
-static uint64_t read_64bit_value() {
+static uint64_t read_64bit_value(void) {
// GCOV uses a lo-/hi-word format even on big-endian systems.
// See also GCOVBuffer::readInt64 in LLVM.
uint32_t lo = read_32bit_value();
return new_filename;
}
-static int map_file() {
+static int map_file(void) {
fseek(output_file, 0L, SEEK_END);
file_size = ftell(output_file);
return 0;
}
-static void unmap_file() {
+static void unmap_file(void) {
#if defined(_WIN32)
- if (!FlushViewOfFile(write_buffer, file_size)) {
- fprintf(stderr, "profiling: %s: cannot flush mapped view: %lu\n", filename,
- GetLastError());
- }
-
if (!UnmapViewOfFile(write_buffer)) {
fprintf(stderr, "profiling: %s: cannot unmap mapped view: %lu\n", filename,
GetLastError());
}
COMPILER_RT_VISIBILITY
-void llvm_gcda_summary_info() {
+void llvm_gcda_summary_info(void) {
uint32_t runs = 1;
static uint32_t run_counted = 0; // We only want to increase the run count once.
uint32_t val = 0;
}
COMPILER_RT_VISIBILITY
-void llvm_gcda_end_file() {
+void llvm_gcda_end_file(void) {
/* Write out EOF record. */
if (output_file) {
write_bytes("\0\0\0\0\0\0\0\0", 8);
: (INSTR_PROF_RAW_MAGIC_32);
}
-COMPILER_RT_VISIBILITY void __llvm_profile_set_dumped() {
+COMPILER_RT_VISIBILITY void __llvm_profile_set_dumped(void) {
lprofSetProfileDumped(1);
}
}
COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_version(void) {
- return __llvm_profile_raw_version;
+ return INSTR_PROF_RAW_VERSION_VAR;
}
COMPILER_RT_VISIBILITY void __llvm_profile_reset_counters(void) {
- uint64_t *I = __llvm_profile_begin_counters();
- uint64_t *E = __llvm_profile_end_counters();
+ char *I = __llvm_profile_begin_counters();
+ char *E = __llvm_profile_end_counters();
- memset(I, 0, sizeof(uint64_t) * (E - I));
+ char ResetValue =
+ (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) ? 0xFF : 0;
+ memset(I, ResetValue, E - I);
const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
CurrentVSiteCount += DI->NumValueSites[VKI];
for (i = 0; i < CurrentVSiteCount; ++i) {
- ValueProfNode *CurrentVNode = ValueCounters[i];
+ ValueProfNode *CurrVNode = ValueCounters[i];
- while (CurrentVNode) {
- CurrentVNode->Count = 0;
- CurrentVNode = CurrentVNode->Next;
+ while (CurrVNode) {
+ CurrVNode->Count = 0;
+ CurrVNode = CurrVNode->Next;
}
}
}
const __llvm_profile_data *__llvm_profile_end_data(void);
const char *__llvm_profile_begin_names(void);
const char *__llvm_profile_end_names(void);
-uint64_t *__llvm_profile_begin_counters(void);
-uint64_t *__llvm_profile_end_counters(void);
+char *__llvm_profile_begin_counters(void);
+char *__llvm_profile_end_counters(void);
ValueProfNode *__llvm_profile_begin_vnodes();
ValueProfNode *__llvm_profile_end_vnodes();
uint32_t *__llvm_profile_begin_orderfile();
int __llvm_orderfile_write_file(void);
/*!
* \brief this is a wrapper interface to \c __llvm_profile_write_file.
- * After this interface is invoked, a arleady dumped flag will be set
+ * After this interface is invoked, an already dumped flag will be set
* so that profile won't be dumped again during program exit.
* Invocation of interface __llvm_profile_reset_counters will clear
* the flag. This interface is designed to be used to collect profile
void __llvm_profile_set_filename(const char *Name);
/*!
- * \brief Set the FILE object for writing instrumentation data.
+ * \brief Set the FILE object for writing instrumentation data. Return 0 if set
+ * successfully or return 1 if failed.
*
* Sets the FILE object to be used for subsequent calls to
* \a __llvm_profile_write_file(). The profile file name set by environment
* instrumented image/DSO). This API only modifies the file object within the
* copy of the runtime available to the calling image.
*
- * Warning: This is a no-op if continuous mode (\ref
- * __llvm_profile_is_continuous_mode_enabled) is on. The reason for this is
- * that in continuous mode, profile counters are mmap()'d to the profile at
- * program initialization time. Support for transferring the mmap'd profile
- * counts to a new file has not been implemented.
+ * Warning: This is a no-op if EnableMerge is 0 in continuous mode (\ref
+ * __llvm_profile_is_continuous_mode_enabled), because disable merging requires
+ * copying the old profile file to new profile file and this function is usually
+ * used when the proess doesn't have permission to open file.
*/
-void __llvm_profile_set_file_object(FILE *File, int EnableMerge);
+int __llvm_profile_set_file_object(FILE *File, int EnableMerge);
/*! \brief Register to write instrumentation data to file at exit. */
int __llvm_profile_register_write_file_atexit(void);
uint64_t __llvm_profile_get_version(void);
/*! \brief Get the number of entries in the profile data section. */
+uint64_t __llvm_profile_get_num_data(const __llvm_profile_data *Begin,
+ const __llvm_profile_data *End);
+
+/*! \brief Get the size of the profile data section in bytes. */
uint64_t __llvm_profile_get_data_size(const __llvm_profile_data *Begin,
const __llvm_profile_data *End);
+/*! \brief Get the size in bytes of a single counter entry. */
+size_t __llvm_profile_counter_entry_size(void);
+
+/*! \brief Get the number of entries in the profile counters section. */
+uint64_t __llvm_profile_get_num_counters(const char *Begin, const char *End);
+
+/*! \brief Get the size of the profile counters section in bytes. */
+uint64_t __llvm_profile_get_counters_size(const char *Begin, const char *End);
+
/* ! \brief Given the sizes of the data and counter information, return the
* number of padding bytes before and after the counters, and after the names,
* in the raw profile.
*
- * Note: In this context, "size" means "number of entries", i.e. the first two
- * arguments must be the result of __llvm_profile_get_data_size() and of
- * (__llvm_profile_end_counters() - __llvm_profile_begin_counters()) resp.
- *
* Note: When mmap() mode is disabled, no padding bytes before/after counters
* are needed. However, in mmap() mode, the counter section in the raw profile
* must be page-aligned: this API computes the number of padding bytes
COMPILER_RT_VISIBILITY extern int INSTR_PROF_PROFILE_RUNTIME_VAR;
/*!
- * This variable is defined in InstrProfiling.c. Its main purpose is to
- * encode the raw profile version value and other format related information
- * such as whether the profile is from IR based instrumentation. The variable
- * is defined as weak so that compiler can emit an overriding definition
- * depending on user option. Since we don't support mixing FE and IR based
- * data in the same raw profile data file (in other words, shared libs and
- * main program are expected to be instrumented in the same way), there is
- * no need for this variable to be hidden.
+ * This variable is defined in InstrProfilingVersionVar.c as a hidden symbol
+ * (except on Apple platforms where this symbol is checked by TAPI). Its main
+ * purpose is to encode the raw profile version value and other format related
+ * information such as whether the profile is from IR based instrumentation. The
+ * variable is defined as weak so that compiler can emit an overriding
+ * definition depending on user option.
*/
extern uint64_t INSTR_PROF_RAW_VERSION_VAR; /* __llvm_profile_raw_version */
uint64_t __llvm_profile_get_size_for_buffer(void) {
const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
- const uint64_t *CountersBegin = __llvm_profile_begin_counters();
- const uint64_t *CountersEnd = __llvm_profile_end_counters();
+ const char *CountersBegin = __llvm_profile_begin_counters();
+ const char *CountersEnd = __llvm_profile_end_counters();
const char *NamesBegin = __llvm_profile_begin_names();
const char *NamesEnd = __llvm_profile_end_names();
}
COMPILER_RT_VISIBILITY
-uint64_t __llvm_profile_get_data_size(const __llvm_profile_data *Begin,
- const __llvm_profile_data *End) {
+uint64_t __llvm_profile_get_num_data(const __llvm_profile_data *Begin,
+ const __llvm_profile_data *End) {
intptr_t BeginI = (intptr_t)Begin, EndI = (intptr_t)End;
return ((EndI + sizeof(__llvm_profile_data) - 1) - BeginI) /
sizeof(__llvm_profile_data);
}
+COMPILER_RT_VISIBILITY
+uint64_t __llvm_profile_get_data_size(const __llvm_profile_data *Begin,
+ const __llvm_profile_data *End) {
+ return __llvm_profile_get_num_data(Begin, End) * sizeof(__llvm_profile_data);
+}
+
+COMPILER_RT_VISIBILITY size_t __llvm_profile_counter_entry_size(void) {
+ if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE)
+ return sizeof(uint8_t);
+ return sizeof(uint64_t);
+}
+
+COMPILER_RT_VISIBILITY
+uint64_t __llvm_profile_get_num_counters(const char *Begin, const char *End) {
+ intptr_t BeginI = (intptr_t)Begin, EndI = (intptr_t)End;
+ return ((EndI + __llvm_profile_counter_entry_size() - 1) - BeginI) /
+ __llvm_profile_counter_entry_size();
+}
+
+COMPILER_RT_VISIBILITY
+uint64_t __llvm_profile_get_counters_size(const char *Begin, const char *End) {
+ return __llvm_profile_get_num_counters(Begin, End) *
+ __llvm_profile_counter_entry_size();
+}
+
/// Calculate the number of padding bytes needed to add to \p Offset in order
/// for (\p Offset + Padding) to be page-aligned.
static uint64_t calculateBytesNeededToPageAlign(uint64_t Offset) {
// In continuous mode, the file offsets for headers and for the start of
// counter sections need to be page-aligned.
- uint64_t DataSizeInBytes = DataSize * sizeof(__llvm_profile_data);
- uint64_t CountersSizeInBytes = CountersSize * sizeof(uint64_t);
- *PaddingBytesBeforeCounters = calculateBytesNeededToPageAlign(
- sizeof(__llvm_profile_header) + DataSizeInBytes);
- *PaddingBytesAfterCounters =
- calculateBytesNeededToPageAlign(CountersSizeInBytes);
+ *PaddingBytesBeforeCounters =
+ calculateBytesNeededToPageAlign(sizeof(__llvm_profile_header) + DataSize);
+ *PaddingBytesAfterCounters = calculateBytesNeededToPageAlign(CountersSize);
*PaddingBytesAfterNames = calculateBytesNeededToPageAlign(NamesSize);
}
COMPILER_RT_VISIBILITY
uint64_t __llvm_profile_get_size_for_buffer_internal(
const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd,
- const uint64_t *CountersBegin, const uint64_t *CountersEnd,
- const char *NamesBegin, const char *NamesEnd) {
+ const char *CountersBegin, const char *CountersEnd, const char *NamesBegin,
+ const char *NamesEnd) {
/* Match logic in __llvm_profile_write_buffer(). */
const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char);
uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
- uint64_t CountersSize = CountersEnd - CountersBegin;
+ uint64_t CountersSize =
+ __llvm_profile_get_counters_size(CountersBegin, CountersEnd);
/* Determine how much padding is needed before/after the counters and after
* the names. */
&PaddingBytesAfterCounters, &PaddingBytesAfterNames);
return sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) +
- (DataSize * sizeof(__llvm_profile_data)) + PaddingBytesBeforeCounters +
- (CountersSize * sizeof(uint64_t)) + PaddingBytesAfterCounters +
- NamesSize + PaddingBytesAfterNames;
+ DataSize + PaddingBytesBeforeCounters + CountersSize +
+ PaddingBytesAfterCounters + NamesSize + PaddingBytesAfterNames;
}
COMPILER_RT_VISIBILITY
COMPILER_RT_VISIBILITY int __llvm_profile_write_buffer_internal(
char *Buffer, const __llvm_profile_data *DataBegin,
- const __llvm_profile_data *DataEnd, const uint64_t *CountersBegin,
- const uint64_t *CountersEnd, const char *NamesBegin, const char *NamesEnd) {
+ const __llvm_profile_data *DataEnd, const char *CountersBegin,
+ const char *CountersEnd, const char *NamesBegin, const char *NamesEnd) {
ProfDataWriter BufferWriter;
initBufferWriter(&BufferWriter, Buffer);
return lprofWriteDataImpl(&BufferWriter, DataBegin, DataEnd, CountersBegin,
{0}, 0, 0, 0, PNS_unknown};
static int ProfileMergeRequested = 0;
-static int isProfileMergeRequested() { return ProfileMergeRequested; }
+static int getProfileFileSizeForMerging(FILE *ProfileFile,
+ uint64_t *ProfileFileSize);
+
+#if defined(__APPLE__)
+static const int ContinuousModeSupported = 1;
+static const int UseBiasVar = 0;
+static const char *FileOpenMode = "a+b";
+static void *BiasAddr = NULL;
+static void *BiasDefaultAddr = NULL;
+static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) {
+ /* Get the sizes of various profile data sections. Taken from
+ * __llvm_profile_get_size_for_buffer(). */
+ const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
+ const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
+ const char *CountersBegin = __llvm_profile_begin_counters();
+ const char *CountersEnd = __llvm_profile_end_counters();
+ const char *NamesBegin = __llvm_profile_begin_names();
+ const char *NamesEnd = __llvm_profile_end_names();
+ const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char);
+ uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
+ uint64_t CountersSize =
+ __llvm_profile_get_counters_size(CountersBegin, CountersEnd);
+
+ /* Check that the counter and data sections in this image are
+ * page-aligned. */
+ unsigned PageSize = getpagesize();
+ if ((intptr_t)CountersBegin % PageSize != 0) {
+ PROF_ERR("Counters section not page-aligned (start = %p, pagesz = %u).\n",
+ CountersBegin, PageSize);
+ return 1;
+ }
+ if ((intptr_t)DataBegin % PageSize != 0) {
+ PROF_ERR("Data section not page-aligned (start = %p, pagesz = %u).\n",
+ DataBegin, PageSize);
+ return 1;
+ }
+ int Fileno = fileno(File);
+ /* Determine how much padding is needed before/after the counters and
+ * after the names. */
+ uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters,
+ PaddingBytesAfterNames;
+ __llvm_profile_get_padding_sizes_for_counters(
+ DataSize, CountersSize, NamesSize, &PaddingBytesBeforeCounters,
+ &PaddingBytesAfterCounters, &PaddingBytesAfterNames);
+
+ uint64_t PageAlignedCountersLength = CountersSize + PaddingBytesAfterCounters;
+ uint64_t FileOffsetToCounters = CurrentFileOffset +
+ sizeof(__llvm_profile_header) + DataSize +
+ PaddingBytesBeforeCounters;
+ void *CounterMmap = mmap((void *)CountersBegin, PageAlignedCountersLength,
+ PROT_READ | PROT_WRITE, MAP_FIXED | MAP_SHARED,
+ Fileno, FileOffsetToCounters);
+ if (CounterMmap != CountersBegin) {
+ PROF_ERR(
+ "Continuous counter sync mode is enabled, but mmap() failed (%s).\n"
+ " - CountersBegin: %p\n"
+ " - PageAlignedCountersLength: %" PRIu64 "\n"
+ " - Fileno: %d\n"
+ " - FileOffsetToCounters: %" PRIu64 "\n",
+ strerror(errno), CountersBegin, PageAlignedCountersLength, Fileno,
+ FileOffsetToCounters);
+ return 1;
+ }
+ return 0;
+}
+#elif defined(__ELF__) || defined(_WIN32)
+
+#define INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR \
+ INSTR_PROF_CONCAT(INSTR_PROF_PROFILE_COUNTER_BIAS_VAR, _default)
+COMPILER_RT_VISIBILITY intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR = 0;
+
+/* This variable is a weak external reference which could be used to detect
+ * whether or not the compiler defined this symbol. */
+#if defined(_MSC_VER)
+COMPILER_RT_VISIBILITY extern intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR;
+#if defined(_M_IX86) || defined(__i386__)
+#define WIN_SYM_PREFIX "_"
+#else
+#define WIN_SYM_PREFIX
+#endif
+#pragma comment( \
+ linker, "/alternatename:" WIN_SYM_PREFIX INSTR_PROF_QUOTE( \
+ INSTR_PROF_PROFILE_COUNTER_BIAS_VAR) "=" WIN_SYM_PREFIX \
+ INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR))
+#else
+COMPILER_RT_VISIBILITY extern intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR
+ __attribute__((weak, alias(INSTR_PROF_QUOTE(
+ INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR))));
+#endif
+static const int ContinuousModeSupported = 1;
+static const int UseBiasVar = 1;
+/* TODO: If there are two DSOs, the second DSO initilization will truncate the
+ * first profile file. */
+static const char *FileOpenMode = "w+b";
+/* This symbol is defined by the compiler when runtime counter relocation is
+ * used and runtime provides a weak alias so we can check if it's defined. */
+static void *BiasAddr = &INSTR_PROF_PROFILE_COUNTER_BIAS_VAR;
+static void *BiasDefaultAddr = &INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR;
+static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) {
+ /* Get the sizes of various profile data sections. Taken from
+ * __llvm_profile_get_size_for_buffer(). */
+ const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
+ const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
+ const char *CountersBegin = __llvm_profile_begin_counters();
+ const char *CountersEnd = __llvm_profile_end_counters();
+ uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
+ /* Get the file size. */
+ uint64_t FileSize = 0;
+ if (getProfileFileSizeForMerging(File, &FileSize))
+ return 1;
+
+ /* Map the profile. */
+ char *Profile = (char *)mmap(NULL, FileSize, PROT_READ | PROT_WRITE,
+ MAP_SHARED, fileno(File), 0);
+ if (Profile == MAP_FAILED) {
+ PROF_ERR("Unable to mmap profile: %s\n", strerror(errno));
+ return 1;
+ }
+ const uint64_t CountersOffsetInBiasMode =
+ sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) + DataSize;
+ /* Update the profile fields based on the current mapping. */
+ INSTR_PROF_PROFILE_COUNTER_BIAS_VAR =
+ (intptr_t)Profile - (uintptr_t)CountersBegin + CountersOffsetInBiasMode;
+
+ /* Return the memory allocated for counters to OS. */
+ lprofReleaseMemoryPagesToOS((uintptr_t)CountersBegin, (uintptr_t)CountersEnd);
+ return 0;
+}
+#else
+static const int ContinuousModeSupported = 0;
+static const int UseBiasVar = 0;
+static const char *FileOpenMode = "a+b";
+static void *BiasAddr = NULL;
+static void *BiasDefaultAddr = NULL;
+static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) {
+ return 0;
+}
+#endif
+
+static int isProfileMergeRequested(void) { return ProfileMergeRequested; }
static void setProfileMergeRequested(int EnableMerge) {
ProfileMergeRequested = EnableMerge;
}
static FILE *ProfileFile = NULL;
-static FILE *getProfileFile() { return ProfileFile; }
+static FILE *getProfileFile(void) { return ProfileFile; }
static void setProfileFile(FILE *File) { ProfileFile = File; }
-COMPILER_RT_VISIBILITY void __llvm_profile_set_file_object(FILE *File,
- int EnableMerge) {
- if (__llvm_profile_is_continuous_mode_enabled()) {
- PROF_WARN("__llvm_profile_set_file_object(fd=%d) not supported, because "
- "continuous sync mode (%%c) is enabled",
- fileno(File));
- return;
- }
- setProfileFile(File);
- setProfileMergeRequested(EnableMerge);
-}
-
-static int getCurFilenameLength();
+static int getCurFilenameLength(void);
static const char *getCurFilename(char *FilenameBuf, int ForceUseBuf);
-static unsigned doMerging() {
+static unsigned doMerging(void) {
return lprofCurFilename.MergePoolSize || isProfileMergeRequested();
}
return IO;
}
-static void setupIOBuffer() {
+static void setupIOBuffer(void) {
const char *BufferSzStr = 0;
BufferSzStr = getenv("LLVM_VP_BUFFER_SIZE");
if (BufferSzStr && BufferSzStr[0]) {
fclose(File);
}
-// TODO: Move these functions into InstrProfilingPlatform* files.
-#if defined(__APPLE__)
-static void assertIsZero(int *i) {
- if (*i)
- PROF_WARN("Expected flag to be 0, but got: %d\n", *i);
-}
-
/* Write a partial profile to \p Filename, which is required to be backed by
* the open file object \p File. */
static int writeProfileWithFileObject(const char *Filename, FILE *File) {
return rc;
}
-/* Unlock the profile \p File and clear the unlock flag. */
-static void unlockProfile(int *ProfileRequiresUnlock, FILE *File) {
- if (!*ProfileRequiresUnlock) {
- PROF_WARN("%s", "Expected to require profile unlock\n");
- }
-
- lprofUnlockFileHandle(File);
- *ProfileRequiresUnlock = 0;
-}
-
static void initializeProfileForContinuousMode(void) {
if (!__llvm_profile_is_continuous_mode_enabled())
return;
-
- /* Get the sizes of various profile data sections. Taken from
- * __llvm_profile_get_size_for_buffer(). */
- const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
- const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
- const uint64_t *CountersBegin = __llvm_profile_begin_counters();
- const uint64_t *CountersEnd = __llvm_profile_end_counters();
- const char *NamesBegin = __llvm_profile_begin_names();
- const char *NamesEnd = __llvm_profile_end_names();
- const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char);
- uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
- uint64_t CountersSize = CountersEnd - CountersBegin;
-
- /* Check that the counter and data sections in this image are page-aligned. */
- unsigned PageSize = getpagesize();
- if ((intptr_t)CountersBegin % PageSize != 0) {
- PROF_ERR("Counters section not page-aligned (start = %p, pagesz = %u).\n",
- CountersBegin, PageSize);
+ if (!ContinuousModeSupported) {
+ PROF_ERR("%s\n", "continuous mode is unsupported on this platform");
return;
}
- if ((intptr_t)DataBegin % PageSize != 0) {
- PROF_ERR("Data section not page-aligned (start = %p, pagesz = %u).\n",
- DataBegin, PageSize);
+ if (UseBiasVar && BiasAddr == BiasDefaultAddr) {
+ PROF_ERR("%s\n", "__llvm_profile_counter_bias is undefined");
return;
}
+ /* Get the sizes of counter section. */
+ uint64_t CountersSize = __llvm_profile_get_counters_size(
+ __llvm_profile_begin_counters(), __llvm_profile_end_counters());
+
int Length = getCurFilenameLength();
char *FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
const char *Filename = getCurFilename(FilenameBuf, 0);
return;
FILE *File = NULL;
- off_t CurrentFileOffset = 0;
- off_t OffsetModPage = 0;
-
- /* Whether an exclusive lock on the profile must be dropped after init.
- * Use a cleanup to warn if the unlock does not occur. */
- COMPILER_RT_CLEANUP(assertIsZero) int ProfileRequiresUnlock = 0;
-
- if (!doMerging()) {
- /* We are not merging profiles, so open the raw profile in append mode. */
- File = fopen(Filename, "a+b");
- if (!File)
- return;
-
- /* Check that the offset within the file is page-aligned. */
- CurrentFileOffset = ftello(File);
- OffsetModPage = CurrentFileOffset % PageSize;
- if (OffsetModPage != 0) {
- PROF_ERR("Continuous counter sync mode is enabled, but raw profile is not"
- "page-aligned. CurrentFileOffset = %" PRIu64 ", pagesz = %u.\n",
- (uint64_t)CurrentFileOffset, PageSize);
- return;
- }
-
- /* Grow the profile so that mmap() can succeed. Leak the file handle, as
- * the file should stay open. */
- if (writeProfileWithFileObject(Filename, File) != 0)
- return;
- } else {
+ uint64_t CurrentFileOffset = 0;
+ if (doMerging()) {
/* We are merging profiles. Map the counter section as shared memory into
* the profile, i.e. into each participating process. An increment in one
* process should be visible to every other process with the same counter
if (!File)
return;
- ProfileRequiresUnlock = 1;
-
- uint64_t ProfileFileSize;
- if (getProfileFileSizeForMerging(File, &ProfileFileSize) == -1)
- return unlockProfile(&ProfileRequiresUnlock, File);
-
- if (ProfileFileSize == 0) {
- /* Grow the profile so that mmap() can succeed. Leak the file handle, as
- * the file should stay open. */
- if (writeProfileWithFileObject(Filename, File) != 0)
- return unlockProfile(&ProfileRequiresUnlock, File);
- } else {
- /* The merged profile has a non-zero length. Check that it is compatible
- * with the data in this process. */
- char *ProfileBuffer;
- if (mmapProfileForMerging(File, ProfileFileSize, &ProfileBuffer) == -1 ||
- munmap(ProfileBuffer, ProfileFileSize) == -1)
- return unlockProfile(&ProfileRequiresUnlock, File);
- }
- }
-
- /* mmap() the profile counters so long as there is at least one counter.
- * If there aren't any counters, mmap() would fail with EINVAL. */
- if (CountersSize > 0) {
- int Fileno = fileno(File);
-
- /* Determine how much padding is needed before/after the counters and after
- * the names. */
- uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters,
- PaddingBytesAfterNames;
- __llvm_profile_get_padding_sizes_for_counters(
- DataSize, CountersSize, NamesSize, &PaddingBytesBeforeCounters,
- &PaddingBytesAfterCounters, &PaddingBytesAfterNames);
-
- uint64_t PageAlignedCountersLength =
- (CountersSize * sizeof(uint64_t)) + PaddingBytesAfterCounters;
- uint64_t FileOffsetToCounters =
- CurrentFileOffset + sizeof(__llvm_profile_header) +
- (DataSize * sizeof(__llvm_profile_data)) + PaddingBytesBeforeCounters;
-
- uint64_t *CounterMmap = (uint64_t *)mmap(
- (void *)CountersBegin, PageAlignedCountersLength, PROT_READ | PROT_WRITE,
- MAP_FIXED | MAP_SHARED, Fileno, FileOffsetToCounters);
- if (CounterMmap != CountersBegin) {
- PROF_ERR(
- "Continuous counter sync mode is enabled, but mmap() failed (%s).\n"
- " - CountersBegin: %p\n"
- " - PageAlignedCountersLength: %" PRIu64 "\n"
- " - Fileno: %d\n"
- " - FileOffsetToCounters: %" PRIu64 "\n",
- strerror(errno), CountersBegin, PageAlignedCountersLength, Fileno,
- FileOffsetToCounters);
- }
- }
-
- if (ProfileRequiresUnlock)
- unlockProfile(&ProfileRequiresUnlock, File);
-}
-#elif defined(__ELF__) || defined(_WIN32)
-
-#define INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR \
- INSTR_PROF_CONCAT(INSTR_PROF_PROFILE_COUNTER_BIAS_VAR, _default)
-intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR = 0;
-
-/* This variable is a weak external reference which could be used to detect
- * whether or not the compiler defined this symbol. */
-#if defined(_MSC_VER)
-COMPILER_RT_VISIBILITY extern intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR;
-#if defined(_M_IX86) || defined(__i386__)
-#define WIN_SYM_PREFIX "_"
-#else
-#define WIN_SYM_PREFIX
-#endif
-#pragma comment( \
- linker, "/alternatename:" WIN_SYM_PREFIX INSTR_PROF_QUOTE( \
- INSTR_PROF_PROFILE_COUNTER_BIAS_VAR) "=" WIN_SYM_PREFIX \
- INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR))
-#else
-COMPILER_RT_VISIBILITY extern intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR
- __attribute__((weak, alias(INSTR_PROF_QUOTE(
- INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR))));
-#endif
-
-static int writeMMappedFile(FILE *OutputFile, char **Profile) {
- if (!OutputFile)
- return -1;
-
- /* Write the data into a file. */
- setupIOBuffer();
- ProfDataWriter fileWriter;
- initFileWriter(&fileWriter, OutputFile);
- if (lprofWriteData(&fileWriter, NULL, 0)) {
- PROF_ERR("Failed to write profile: %s\n", strerror(errno));
- return -1;
- }
- fflush(OutputFile);
-
- /* Get the file size. */
- uint64_t FileSize = ftell(OutputFile);
-
- /* Map the profile. */
- *Profile = (char *)mmap(
- NULL, FileSize, PROT_READ | PROT_WRITE, MAP_SHARED, fileno(OutputFile), 0);
- if (*Profile == MAP_FAILED) {
- PROF_ERR("Unable to mmap profile: %s\n", strerror(errno));
- return -1;
- }
-
- return 0;
-}
-
-static void initializeProfileForContinuousMode(void) {
- if (!__llvm_profile_is_continuous_mode_enabled())
- return;
-
- /* This symbol is defined by the compiler when runtime counter relocation is
- * used and runtime provides a weak alias so we can check if it's defined. */
- void *BiasAddr = &INSTR_PROF_PROFILE_COUNTER_BIAS_VAR;
- void *BiasDefaultAddr = &INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR;
- if (BiasAddr == BiasDefaultAddr) {
- PROF_ERR("%s\n", "__llvm_profile_counter_bias is undefined");
- return;
- }
-
- /* Get the sizes of various profile data sections. Taken from
- * __llvm_profile_get_size_for_buffer(). */
- const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
- const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
- const uint64_t *CountersBegin = __llvm_profile_begin_counters();
- const uint64_t *CountersEnd = __llvm_profile_end_counters();
- uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
- const uint64_t CountersOffset = sizeof(__llvm_profile_header) +
- __llvm_write_binary_ids(NULL) +
- (DataSize * sizeof(__llvm_profile_data));
-
- int Length = getCurFilenameLength();
- char *FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
- const char *Filename = getCurFilename(FilenameBuf, 0);
- if (!Filename)
- return;
-
- FILE *File = NULL;
- char *Profile = NULL;
-
- if (!doMerging()) {
- File = fopen(Filename, "w+b");
- if (!File)
- return;
-
- if (writeMMappedFile(File, &Profile) == -1) {
- fclose(File);
- return;
- }
- } else {
- File = lprofOpenFileEx(Filename);
- if (!File)
- return;
-
uint64_t ProfileFileSize = 0;
if (getProfileFileSizeForMerging(File, &ProfileFileSize) == -1) {
lprofUnlockFileHandle(File);
fclose(File);
return;
}
-
- if (!ProfileFileSize) {
- if (writeMMappedFile(File, &Profile) == -1) {
+ if (ProfileFileSize == 0) {
+ /* Grow the profile so that mmap() can succeed. Leak the file handle, as
+ * the file should stay open. */
+ if (writeProfileWithFileObject(Filename, File) != 0) {
+ lprofUnlockFileHandle(File);
fclose(File);
return;
}
} else {
/* The merged profile has a non-zero length. Check that it is compatible
* with the data in this process. */
- if (mmapProfileForMerging(File, ProfileFileSize, &Profile) == -1) {
+ char *ProfileBuffer;
+ if (mmapProfileForMerging(File, ProfileFileSize, &ProfileBuffer) == -1) {
+ lprofUnlockFileHandle(File);
fclose(File);
return;
}
+ (void)munmap(ProfileBuffer, ProfileFileSize);
+ }
+ } else {
+ File = fopen(Filename, FileOpenMode);
+ if (!File)
+ return;
+ /* Check that the offset within the file is page-aligned. */
+ CurrentFileOffset = ftell(File);
+ unsigned PageSize = getpagesize();
+ if (CurrentFileOffset % PageSize != 0) {
+ PROF_ERR("Continuous counter sync mode is enabled, but raw profile is not"
+ "page-aligned. CurrentFileOffset = %" PRIu64 ", pagesz = %u.\n",
+ (uint64_t)CurrentFileOffset, PageSize);
+ return;
+ }
+ if (writeProfileWithFileObject(Filename, File) != 0) {
+ fclose(File);
+ return;
}
-
- lprofUnlockFileHandle(File);
}
- /* Update the profile fields based on the current mapping. */
- INSTR_PROF_PROFILE_COUNTER_BIAS_VAR =
- (intptr_t)Profile - (uintptr_t)CountersBegin +
- CountersOffset;
+ /* mmap() the profile counters so long as there is at least one counter.
+ * If there aren't any counters, mmap() would fail with EINVAL. */
+ if (CountersSize > 0)
+ mmapForContinuousMode(CurrentFileOffset, File);
- /* Return the memory allocated for counters to OS. */
- lprofReleaseMemoryPagesToOS((uintptr_t)CountersBegin, (uintptr_t)CountersEnd);
-}
-#else
-static void initializeProfileForContinuousMode(void) {
- PROF_ERR("%s\n", "continuous mode is unsupported on this platform");
+ if (doMerging()) {
+ lprofUnlockFileHandle(File);
+ fclose(File);
+ }
}
-#endif
static const char *DefaultProfileName = "default.profraw";
static void resetFilenameToDefault(void) {
* filename with PID and hostname substitutions. */
/* The length to hold uint64_t followed by 3 digits pool id including '_' */
#define SIGLEN 24
-static int getCurFilenameLength() {
+static int getCurFilenameLength(void) {
int Len;
if (!lprofCurFilename.FilenamePat || !lprofCurFilename.FilenamePat[0])
return 0;
return atexit(writeFileWithoutReturn);
}
+COMPILER_RT_VISIBILITY int __llvm_profile_set_file_object(FILE *File,
+ int EnableMerge) {
+ if (__llvm_profile_is_continuous_mode_enabled()) {
+ if (!EnableMerge) {
+ PROF_WARN("__llvm_profile_set_file_object(fd=%d) not supported in "
+ "continuous sync mode when merging is disabled\n",
+ fileno(File));
+ return 1;
+ }
+ if (lprofLockFileHandle(File) != 0) {
+ PROF_WARN("Data may be corrupted during profile merging : %s\n",
+ "Fail to obtain file lock due to system limit.");
+ }
+ uint64_t ProfileFileSize = 0;
+ if (getProfileFileSizeForMerging(File, &ProfileFileSize) == -1) {
+ lprofUnlockFileHandle(File);
+ return 1;
+ }
+ if (ProfileFileSize == 0) {
+ FreeHook = &free;
+ setupIOBuffer();
+ ProfDataWriter fileWriter;
+ initFileWriter(&fileWriter, File);
+ if (lprofWriteData(&fileWriter, 0, 0)) {
+ lprofUnlockFileHandle(File);
+ PROF_ERR("Failed to write file \"%d\": %s\n", fileno(File),
+ strerror(errno));
+ return 1;
+ }
+ fflush(File);
+ } else {
+ /* The merged profile has a non-zero length. Check that it is compatible
+ * with the data in this process. */
+ char *ProfileBuffer;
+ if (mmapProfileForMerging(File, ProfileFileSize, &ProfileBuffer) == -1) {
+ lprofUnlockFileHandle(File);
+ return 1;
+ }
+ (void)munmap(ProfileBuffer, ProfileFileSize);
+ }
+ mmapForContinuousMode(0, File);
+ lprofUnlockFileHandle(File);
+ } else {
+ setProfileFile(File);
+ setProfileMergeRequested(EnableMerge);
+ }
+ return 0;
+}
+
#endif
static unsigned ProfileDumped = 0;
-COMPILER_RT_VISIBILITY unsigned lprofProfileDumped() {
+COMPILER_RT_VISIBILITY unsigned lprofProfileDumped(void) {
return ProfileDumped;
}
*/
uint64_t __llvm_profile_get_size_for_buffer_internal(
const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd,
- const uint64_t *CountersBegin, const uint64_t *CountersEnd,
- const char *NamesBegin, const char *NamesEnd);
+ const char *CountersBegin, const char *CountersEnd, const char *NamesBegin,
+ const char *NamesEnd);
/*!
* \brief Write instrumentation data to the given buffer, given explicit
*/
int __llvm_profile_write_buffer_internal(
char *Buffer, const __llvm_profile_data *DataBegin,
- const __llvm_profile_data *DataEnd, const uint64_t *CountersBegin,
- const uint64_t *CountersEnd, const char *NamesBegin, const char *NamesEnd);
+ const __llvm_profile_data *DataEnd, const char *CountersBegin,
+ const char *CountersEnd, const char *NamesBegin, const char *NamesEnd);
/*!
* The data structure describing the data to be written by the
uint32_t N);
} VPDataReaderType;
-/* Write profile data to destinitation. If SkipNameDataWrite is set to 1,
- the name data is already in destintation, we just skip over it. */
+/* Write profile data to destination. If SkipNameDataWrite is set to 1,
+ the name data is already in destination, we just skip over it. */
int lprofWriteData(ProfDataWriter *Writer, VPDataReaderType *VPDataReader,
int SkipNameDataWrite);
int lprofWriteDataImpl(ProfDataWriter *Writer,
const __llvm_profile_data *DataBegin,
const __llvm_profile_data *DataEnd,
- const uint64_t *CountersBegin,
- const uint64_t *CountersEnd,
+ const char *CountersBegin, const char *CountersEnd,
VPDataReaderType *VPDataReader, const char *NamesBegin,
const char *NamesEnd, int SkipNameDataWrite);
void (*VPMergeHook)(ValueProfData *, __llvm_profile_data *);
COMPILER_RT_VISIBILITY
-uint64_t lprofGetLoadModuleSignature() {
+uint64_t lprofGetLoadModuleSignature(void) {
/* A very fast way to compute a module signature. */
uint64_t Version = __llvm_profile_get_version();
- uint64_t CounterSize = (uint64_t)(__llvm_profile_end_counters() -
- __llvm_profile_begin_counters());
- uint64_t DataSize = __llvm_profile_get_data_size(__llvm_profile_begin_data(),
- __llvm_profile_end_data());
+ uint64_t NumCounters = __llvm_profile_get_num_counters(
+ __llvm_profile_begin_counters(), __llvm_profile_end_counters());
+ uint64_t NumData = __llvm_profile_get_num_data(__llvm_profile_begin_data(),
+ __llvm_profile_end_data());
uint64_t NamesSize =
(uint64_t)(__llvm_profile_end_names() - __llvm_profile_begin_names());
uint64_t NumVnodes =
(uint64_t)(__llvm_profile_end_vnodes() - __llvm_profile_begin_vnodes());
const __llvm_profile_data *FirstD = __llvm_profile_begin_data();
- return (NamesSize << 40) + (CounterSize << 30) + (DataSize << 20) +
- (NumVnodes << 10) + (DataSize > 0 ? FirstD->NameRef : 0) + Version;
+ return (NamesSize << 40) + (NumCounters << 30) + (NumData << 20) +
+ (NumVnodes << 10) + (NumData > 0 ? FirstD->NameRef : 0) + Version +
+ __llvm_profile_get_magic();
}
/* Returns 1 if profile is not structurally compatible. */
if (Header->Magic != __llvm_profile_get_magic() ||
Header->Version != __llvm_profile_get_version() ||
Header->DataSize !=
- __llvm_profile_get_data_size(__llvm_profile_begin_data(),
- __llvm_profile_end_data()) ||
- Header->CountersSize != (uint64_t)(__llvm_profile_end_counters() -
- __llvm_profile_begin_counters()) ||
+ __llvm_profile_get_num_data(__llvm_profile_begin_data(),
+ __llvm_profile_end_data()) ||
+ Header->CountersSize !=
+ __llvm_profile_get_num_counters(__llvm_profile_begin_counters(),
+ __llvm_profile_end_counters()) ||
Header->NamesSize != (uint64_t)(__llvm_profile_end_names() -
__llvm_profile_begin_names()) ||
Header->ValueKindLast != IPVK_Last)
return 1;
- if (ProfileSize < sizeof(__llvm_profile_header) + Header->BinaryIdsSize +
- Header->DataSize * sizeof(__llvm_profile_data) +
- Header->NamesSize + Header->CountersSize)
+ if (ProfileSize <
+ sizeof(__llvm_profile_header) + Header->BinaryIdsSize +
+ Header->DataSize * sizeof(__llvm_profile_data) + Header->NamesSize +
+ Header->CountersSize * __llvm_profile_counter_entry_size())
return 1;
for (SrcData = SrcDataStart,
return 0;
}
+static uintptr_t signextIfWin64(void *V) {
+#ifdef _WIN64
+ return (uintptr_t)(int32_t)(uintptr_t)V;
+#else
+ return (uintptr_t)V;
+#endif
+}
+
COMPILER_RT_VISIBILITY
int __llvm_profile_merge_from_buffer(const char *ProfileData,
uint64_t ProfileSize) {
+ if (__llvm_profile_get_version() & VARIANT_MASK_DBG_CORRELATE) {
+ PROF_ERR(
+ "%s\n",
+ "Debug info correlation does not support profile merging at runtime. "
+ "Instead, merge raw profiles using the llvm-profdata tool.");
+ return 1;
+ }
+
__llvm_profile_data *SrcDataStart, *SrcDataEnd, *SrcData, *DstData;
__llvm_profile_header *Header = (__llvm_profile_header *)ProfileData;
- uint64_t *SrcCountersStart;
+ char *SrcCountersStart;
const char *SrcNameStart;
const char *SrcValueProfDataStart, *SrcValueProfData;
+ uintptr_t CountersDelta = Header->CountersDelta;
SrcDataStart =
(__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header) +
Header->BinaryIdsSize);
SrcDataEnd = SrcDataStart + Header->DataSize;
- SrcCountersStart = (uint64_t *)SrcDataEnd;
- SrcNameStart = (const char *)(SrcCountersStart + Header->CountersSize);
+ SrcCountersStart = (char *)SrcDataEnd;
+ SrcNameStart = SrcCountersStart +
+ Header->CountersSize * __llvm_profile_counter_entry_size();
SrcValueProfDataStart =
SrcNameStart + Header->NamesSize +
__llvm_profile_get_num_padding_bytes(Header->NamesSize);
- if (SrcNameStart < (const char *)SrcCountersStart)
+ if (SrcNameStart < SrcCountersStart)
return 1;
for (SrcData = SrcDataStart,
DstData = (__llvm_profile_data *)__llvm_profile_begin_data(),
SrcValueProfData = SrcValueProfDataStart;
SrcData < SrcDataEnd; ++SrcData, ++DstData) {
- uint64_t *DstCounters = (uint64_t *)DstData->CounterPtr;
+ // For the in-memory destination, CounterPtr is the distance from the start
+ // address of the data to the start address of the counter. On WIN64,
+ // CounterPtr is a truncated 32-bit value due to COFF limitation. Sign
+ // extend CounterPtr to get the original value.
+ char *DstCounters =
+ (char *)((uintptr_t)DstData + signextIfWin64(DstData->CounterPtr));
unsigned NVK = 0;
+ // SrcData is a serialized representation of the memory image. We need to
+ // compute the in-buffer counter offset from the in-memory address distance.
+ // The initial CountersDelta is the in-memory address difference
+ // start(__llvm_prf_cnts)-start(__llvm_prf_data), so SrcData->CounterPtr -
+ // CountersDelta computes the offset into the in-buffer counter section.
+ //
+ // On WIN64, CountersDelta is truncated as well, so no need for signext.
+ char *SrcCounters =
+ SrcCountersStart + ((uintptr_t)SrcData->CounterPtr - CountersDelta);
+ // CountersDelta needs to be decreased as we advance to the next data
+ // record.
+ CountersDelta -= sizeof(*SrcData);
unsigned NC = SrcData->NumCounters;
if (NC == 0)
return 1;
- uint64_t *SrcCounters = SrcCountersStart + ((size_t)SrcData->CounterPtr -
- Header->CountersDelta) /
- sizeof(uint64_t);
- if (SrcCounters < SrcCountersStart ||
- (const char *)SrcCounters >= SrcNameStart ||
- (const char *)(SrcCounters + NC) > SrcNameStart)
+ if (SrcCounters < SrcCountersStart || SrcCounters >= SrcNameStart ||
+ (SrcCounters + __llvm_profile_counter_entry_size() * NC) > SrcNameStart)
return 1;
- for (unsigned I = 0; I < NC; I++)
- DstCounters[I] += SrcCounters[I];
+ for (unsigned I = 0; I < NC; I++) {
+ if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) {
+ // A value of zero signifies the function is covered.
+ DstCounters[I] &= SrcCounters[I];
+ } else {
+ ((uint64_t *)DstCounters)[I] += ((uint64_t *)SrcCounters)[I];
+ }
+ }
/* Now merge value profile data. */
if (!VPMergeHook)
* user has not specified one. Set this up by moving the runtime's copy of this
* symbol to an object file within the archive.
*/
-COMPILER_RT_WEAK char INSTR_PROF_PROFILE_NAME_VAR[1] = {0};
+COMPILER_RT_WEAK COMPILER_RT_VISIBILITY char INSTR_PROF_PROFILE_NAME_VAR[1] = {0};
COMPILER_RT_VISIBILITY
extern char NamesEnd __asm("section$end$__DATA$" INSTR_PROF_NAME_SECT_NAME);
COMPILER_RT_VISIBILITY
-extern uint64_t
+extern char
CountersStart __asm("section$start$__DATA$" INSTR_PROF_CNTS_SECT_NAME);
COMPILER_RT_VISIBILITY
-extern uint64_t
- CountersEnd __asm("section$end$__DATA$" INSTR_PROF_CNTS_SECT_NAME);
+extern char CountersEnd __asm("section$end$__DATA$" INSTR_PROF_CNTS_SECT_NAME);
COMPILER_RT_VISIBILITY
extern uint32_t
OrderFileStart __asm("section$start$__DATA$" INSTR_PROF_ORDERFILE_SECT_NAME);
COMPILER_RT_VISIBILITY
const char *__llvm_profile_end_names(void) { return &NamesEnd; }
COMPILER_RT_VISIBILITY
-uint64_t *__llvm_profile_begin_counters(void) { return &CountersStart; }
+char *__llvm_profile_begin_counters(void) { return &CountersStart; }
COMPILER_RT_VISIBILITY
-uint64_t *__llvm_profile_end_counters(void) { return &CountersEnd; }
+char *__llvm_profile_end_counters(void) { return &CountersEnd; }
COMPILER_RT_VISIBILITY
uint32_t *__llvm_profile_begin_orderfile(void) { return &OrderFileStart; }
/* This variable is an external reference to symbol defined by the compiler. */
COMPILER_RT_VISIBILITY extern intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR;
-COMPILER_RT_VISIBILITY unsigned lprofProfileDumped() {
+COMPILER_RT_VISIBILITY unsigned lprofProfileDumped(void) {
return 1;
}
COMPILER_RT_VISIBILITY void lprofSetProfileDumped(unsigned Value) {}
int ret = vsnprintf(s, sizeof(s), fmt, ap);
va_end(ap);
- __sanitizer_log_write(s, ret + 1);
+ __sanitizer_log_write(s, ret);
}
struct lprofVMOWriterCtx {
const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
- const uint64_t *CountersBegin = __llvm_profile_begin_counters();
- const uint64_t *CountersEnd = __llvm_profile_end_counters();
+ const char *CountersBegin = __llvm_profile_begin_counters();
+ const char *CountersEnd = __llvm_profile_end_counters();
const uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
- const uint64_t CountersOffset = sizeof(__llvm_profile_header) +
- __llvm_write_binary_ids(NULL) +
- (DataSize * sizeof(__llvm_profile_data));
- uint64_t CountersSize = CountersEnd - CountersBegin;
+ const uint64_t CountersOffset =
+ sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) + DataSize;
+ uint64_t CountersSize =
+ __llvm_profile_get_counters_size(CountersBegin, CountersEnd);
/* Don't publish a VMO if there are no counters. */
if (!CountersSize)
* also consumes the VMO handle. */
__sanitizer_publish_data(ProfileSinkName, Vmo);
- /* Use the dumpfile symbolizer markup element to write the name of VMO. */
- lprofWrite("LLVM Profile: {{{dumpfile:%s:%s}}}\n", ProfileSinkName, VmoName);
-
/* Update the profile fields based on the current mapping. */
INSTR_PROF_PROFILE_COUNTER_BIAS_VAR =
(intptr_t)Mapping - (uintptr_t)CountersBegin + CountersOffset;
\*===----------------------------------------------------------------------===*/
#if defined(__linux__) || defined(__FreeBSD__) || defined(__Fuchsia__) || \
- (defined(__sun__) && defined(__svr4__)) || defined(__NetBSD__)
+ (defined(__sun__) && defined(__svr4__)) || defined(__NetBSD__) || \
+ defined(_AIX)
+#if !defined(_AIX)
#include <elf.h>
#include <link.h>
+#endif
#include <stdlib.h>
#include <string.h>
COMPILER_RT_WEAK;
extern __llvm_profile_data PROF_DATA_STOP COMPILER_RT_VISIBILITY
COMPILER_RT_WEAK;
-extern uint64_t PROF_CNTS_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
-extern uint64_t PROF_CNTS_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
+extern char PROF_CNTS_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
+extern char PROF_CNTS_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
extern uint32_t PROF_ORDERFILE_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
extern char PROF_NAME_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
extern char PROF_NAME_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
COMPILER_RT_VISIBILITY const char *__llvm_profile_end_names(void) {
return &PROF_NAME_STOP;
}
-COMPILER_RT_VISIBILITY uint64_t *__llvm_profile_begin_counters(void) {
+COMPILER_RT_VISIBILITY char *__llvm_profile_begin_counters(void) {
return &PROF_CNTS_START;
}
-COMPILER_RT_VISIBILITY uint64_t *__llvm_profile_end_counters(void) {
+COMPILER_RT_VISIBILITY char *__llvm_profile_end_counters(void) {
return &PROF_CNTS_STOP;
}
COMPILER_RT_VISIBILITY uint32_t *__llvm_profile_begin_orderfile(void) {
* have a fixed length.
*/
static int WriteOneBinaryId(ProfDataWriter *Writer, uint64_t BinaryIdLen,
- const uint8_t *BinaryIdData) {
+ const uint8_t *BinaryIdData,
+ uint64_t BinaryIdPadding) {
ProfDataIOVec BinaryIdIOVec[] = {
{&BinaryIdLen, sizeof(uint64_t), 1, 0},
- {BinaryIdData, sizeof(uint8_t), BinaryIdLen, 0}};
+ {BinaryIdData, sizeof(uint8_t), BinaryIdLen, 0},
+ {NULL, sizeof(uint8_t), BinaryIdPadding, 1},
+ };
if (Writer->Write(Writer, BinaryIdIOVec,
sizeof(BinaryIdIOVec) / sizeof(*BinaryIdIOVec)))
return -1;
static int WriteBinaryIdForNote(ProfDataWriter *Writer,
const ElfW(Nhdr) * Note) {
int BinaryIdSize = 0;
-
const char *NoteName = (const char *)Note + sizeof(ElfW(Nhdr));
if (Note->n_type == NT_GNU_BUILD_ID && Note->n_namesz == 4 &&
memcmp(NoteName, "GNU\0", 4) == 0) {
-
uint64_t BinaryIdLen = Note->n_descsz;
const uint8_t *BinaryIdData =
(const uint8_t *)(NoteName + RoundUp(Note->n_namesz, 4));
- if (Writer != NULL &&
- WriteOneBinaryId(Writer, BinaryIdLen, BinaryIdData) == -1)
+ uint8_t BinaryIdPadding = __llvm_profile_get_num_padding_bytes(BinaryIdLen);
+ if (Writer != NULL && WriteOneBinaryId(Writer, BinaryIdLen, BinaryIdData,
+ BinaryIdPadding) == -1)
return -1;
- BinaryIdSize = sizeof(BinaryIdLen) + BinaryIdLen;
+ BinaryIdSize = sizeof(BinaryIdLen) + BinaryIdLen + BinaryIdPadding;
}
return BinaryIdSize;
*/
static int WriteBinaryIds(ProfDataWriter *Writer, const ElfW(Nhdr) * Note,
const ElfW(Nhdr) * NotesEnd) {
- int TotalBinaryIdsSize = 0;
+ int BinaryIdsSize = 0;
while (Note < NotesEnd) {
- int Result = WriteBinaryIdForNote(Writer, Note);
- if (Result == -1)
+ int OneBinaryIdSize = WriteBinaryIdForNote(Writer, Note);
+ if (OneBinaryIdSize == -1)
return -1;
- TotalBinaryIdsSize += Result;
+ BinaryIdsSize += OneBinaryIdSize;
/* Calculate the offset of the next note in notes section. */
size_t NoteOffset = sizeof(ElfW(Nhdr)) + RoundUp(Note->n_namesz, 4) +
Note = (const ElfW(Nhdr) *)((const char *)(Note) + NoteOffset);
}
- return TotalBinaryIdsSize;
+ return BinaryIdsSize;
}
/*
const ElfW(Phdr) *ProgramHeader =
(const ElfW(Phdr) *)((uintptr_t)ElfHeader + ElfHeader->e_phoff);
+ int TotalBinaryIdsSize = 0;
uint32_t I;
/* Iterate through entries in the program header. */
for (I = 0; I < ElfHeader->e_phnum; I++) {
- /* Look for the notes section in program header entries. */
+ /* Look for the notes segment in program header entries. */
if (ProgramHeader[I].p_type != PT_NOTE)
continue;
- const ElfW(Nhdr) *Note =
- (const ElfW(Nhdr) *)((uintptr_t)ElfHeader + ProgramHeader[I].p_offset);
- const ElfW(Nhdr) *NotesEnd =
- (const ElfW(Nhdr) *)((const char *)(Note) + ProgramHeader[I].p_filesz);
- return WriteBinaryIds(Writer, Note, NotesEnd);
+ /* There can be multiple notes segment, and examine each of them. */
+ const ElfW(Nhdr) * Note;
+ const ElfW(Nhdr) * NotesEnd;
+ /*
+ * When examining notes in file, use p_offset, which is the offset within
+ * the elf file, to find the start of notes.
+ */
+ if (ProgramHeader[I].p_memsz == 0 ||
+ ProgramHeader[I].p_memsz == ProgramHeader[I].p_filesz) {
+ Note = (const ElfW(Nhdr) *)((uintptr_t)ElfHeader +
+ ProgramHeader[I].p_offset);
+ NotesEnd = (const ElfW(Nhdr) *)((const char *)(Note) +
+ ProgramHeader[I].p_filesz);
+ } else {
+ /*
+ * When examining notes in memory, use p_vaddr, which is the address of
+ * section after loaded to memory, to find the start of notes.
+ */
+ Note =
+ (const ElfW(Nhdr) *)((uintptr_t)ElfHeader + ProgramHeader[I].p_vaddr);
+ NotesEnd =
+ (const ElfW(Nhdr) *)((const char *)(Note) + ProgramHeader[I].p_memsz);
+ }
+
+ int BinaryIdsSize = WriteBinaryIds(Writer, Note, NotesEnd);
+ if (TotalBinaryIdsSize == -1)
+ return -1;
+
+ TotalBinaryIdsSize += BinaryIdsSize;
}
- return 0;
+ return TotalBinaryIdsSize;
}
#else /* !NT_GNU_BUILD_ID */
/*
}
#endif
+#if defined(_AIX)
+// Empty stubs to allow linking object files using the registration-based scheme
+COMPILER_RT_VISIBILITY
+void __llvm_profile_register_function(void *Data_) {}
+
+COMPILER_RT_VISIBILITY
+void __llvm_profile_register_names_function(void *NamesStart,
+ uint64_t NamesSize) {}
+
+// The __start_SECNAME and __stop_SECNAME symbols (for SECNAME \in
+// {"__llvm_prf_cnts", "__llvm_prf_data", "__llvm_prf_name", "__llvm_prf_vnds"})
+// are always live when linking on AIX, regardless if the .o's being linked
+// reference symbols from the profile library (for example when no files were
+// compiled with -fprofile-generate). That's because these symbols are kept
+// alive through references in constructor functions that are always live in the
+// default linking model on AIX (-bcdtors:all). The __start_SECNAME and
+// __stop_SECNAME symbols are only resolved by the linker when the SECNAME
+// section exists. So for the scenario where the user objects have no such
+// section (i.e. when they are compiled with -fno-profile-generate), we always
+// define these zero length variables in each of the above 4 sections.
+static int dummy_cnts[0] COMPILER_RT_SECTION(
+ COMPILER_RT_SEG INSTR_PROF_CNTS_SECT_NAME);
+static int dummy_data[0] COMPILER_RT_SECTION(
+ COMPILER_RT_SEG INSTR_PROF_DATA_SECT_NAME);
+static const int dummy_name[0] COMPILER_RT_SECTION(
+ COMPILER_RT_SEG INSTR_PROF_NAME_SECT_NAME);
+static int dummy_vnds[0] COMPILER_RT_SECTION(
+ COMPILER_RT_SEG INSTR_PROF_VNODES_SECT_NAME);
+
+// To avoid GC'ing of the dummy variables by the linker, reference them in an
+// array and reference the array in the runtime registration code
+// (InstrProfilingRuntime.cpp)
+COMPILER_RT_VISIBILITY
+void *__llvm_profile_keep[] = {(void *)&dummy_cnts, (void *)&dummy_data,
+ (void *)&dummy_name, (void *)&dummy_vnds};
+#endif
+
#endif
\*===----------------------------------------------------------------------===*/
#if !defined(__APPLE__) && !defined(__linux__) && !defined(__FreeBSD__) && \
- !(defined(__sun__) && defined(__svr4__)) && !defined(__NetBSD__) && \
- !defined(_WIN32)
+ !defined(__Fuchsia__) && !(defined(__sun__) && defined(__svr4__)) && \
+ !defined(__NetBSD__) && !defined(_WIN32) && !defined(_AIX)
#include <stdlib.h>
#include <stdio.h>
static const __llvm_profile_data *DataLast = NULL;
static const char *NamesFirst = NULL;
static const char *NamesLast = NULL;
-static uint64_t *CountersFirst = NULL;
-static uint64_t *CountersLast = NULL;
+static char *CountersFirst = NULL;
+static char *CountersLast = NULL;
static uint32_t *OrderFileFirst = NULL;
static const void *getMinAddr(const void *A1, const void *A2) {
if (!DataFirst) {
DataFirst = Data;
DataLast = Data + 1;
- CountersFirst = Data->CounterPtr;
- CountersLast = (uint64_t *)Data->CounterPtr + Data->NumCounters;
+ CountersFirst = (char *)((uintptr_t)Data_ + Data->CounterPtr);
+ CountersLast =
+ CountersFirst + Data->NumCounters * __llvm_profile_counter_entry_size();
return;
}
DataFirst = (const __llvm_profile_data *)getMinAddr(DataFirst, Data);
- CountersFirst = (uint64_t *)getMinAddr(CountersFirst, Data->CounterPtr);
+ CountersFirst = (char *)getMinAddr(
+ CountersFirst, (char *)((uintptr_t)Data_ + Data->CounterPtr));
DataLast = (const __llvm_profile_data *)getMaxAddr(DataLast, Data + 1);
- CountersLast = (uint64_t *)getMaxAddr(
- CountersLast, (uint64_t *)Data->CounterPtr + Data->NumCounters);
+ CountersLast = (char *)getMaxAddr(
+ CountersLast,
+ (char *)((uintptr_t)Data_ + Data->CounterPtr) +
+ Data->NumCounters * __llvm_profile_counter_entry_size());
}
COMPILER_RT_VISIBILITY
COMPILER_RT_VISIBILITY
const char *__llvm_profile_end_names(void) { return NamesLast; }
COMPILER_RT_VISIBILITY
-uint64_t *__llvm_profile_begin_counters(void) { return CountersFirst; }
+char *__llvm_profile_begin_counters(void) { return CountersFirst; }
COMPILER_RT_VISIBILITY
-uint64_t *__llvm_profile_end_counters(void) { return CountersLast; }
+char *__llvm_profile_end_counters(void) { return CountersLast; }
/* TODO: correctly set up OrderFileFirst. */
COMPILER_RT_VISIBILITY
uint32_t *__llvm_profile_begin_orderfile(void) { return OrderFileFirst; }
const char COMPILER_RT_SECTION(".lprfn$A") NamesStart = '\0';
const char COMPILER_RT_SECTION(".lprfn$Z") NamesEnd = '\0';
-uint64_t COMPILER_RT_SECTION(".lprfc$A") CountersStart;
-uint64_t COMPILER_RT_SECTION(".lprfc$Z") CountersEnd;
+char COMPILER_RT_SECTION(".lprfc$A") CountersStart;
+char COMPILER_RT_SECTION(".lprfc$Z") CountersEnd;
uint32_t COMPILER_RT_SECTION(".lorderfile$A") OrderFileStart;
ValueProfNode COMPILER_RT_SECTION(".lprfnd$A") VNodesStart;
const char *__llvm_profile_begin_names(void) { return &NamesStart + 1; }
const char *__llvm_profile_end_names(void) { return &NamesEnd; }
-uint64_t *__llvm_profile_begin_counters(void) { return &CountersStart + 1; }
-uint64_t *__llvm_profile_end_counters(void) { return &CountersEnd; }
+char *__llvm_profile_begin_counters(void) { return &CountersStart + 1; }
+char *__llvm_profile_end_counters(void) { return &CountersEnd; }
uint32_t *__llvm_profile_begin_orderfile(void) { return &OrderFileStart; }
ValueProfNode *__llvm_profile_begin_vnodes(void) { return &VNodesStart + 1; }
#include "InstrProfiling.h"
-/* int __llvm_profile_runtime */
-COMPILER_RT_VISIBILITY int INSTR_PROF_PROFILE_RUNTIME_VAR;
+static int RegisterRuntime() {
+ __llvm_profile_initialize();
+#ifdef _AIX
+ extern COMPILER_RT_VISIBILITY void *__llvm_profile_keep[];
+ (void)*(void *volatile *)__llvm_profile_keep;
+#endif
+ return 0;
}
-namespace {
-
-class RegisterRuntime {
-public:
- RegisterRuntime() {
- __llvm_profile_initialize();
- }
-};
-
-RegisterRuntime Registration;
-
+/* int __llvm_profile_runtime */
+COMPILER_RT_VISIBILITY int INSTR_PROF_PROFILE_RUNTIME_VAR = RegisterRuntime();
}
#endif
#if defined(__Fuchsia__)
+#include <zircon/process.h>
#include <zircon/syscalls.h>
#endif
+#if defined(__FreeBSD__)
+#include <signal.h>
+#include <sys/procctl.h>
+#endif
+
#include "InstrProfiling.h"
#include "InstrProfilingUtil.h"
-COMPILER_RT_WEAK unsigned lprofDirMode = 0755;
+COMPILER_RT_VISIBILITY unsigned lprofDirMode = 0755;
COMPILER_RT_VISIBILITY
void __llvm_profile_recursive_mkdir(char *path) {
return Sep;
}
-COMPILER_RT_VISIBILITY int lprofSuspendSigKill() {
+COMPILER_RT_VISIBILITY int lprofSuspendSigKill(void) {
#if defined(__linux__)
int PDeachSig = 0;
/* Temporarily suspend getting SIGKILL upon exit of the parent process. */
if (prctl(PR_GET_PDEATHSIG, &PDeachSig) == 0 && PDeachSig == SIGKILL)
prctl(PR_SET_PDEATHSIG, 0);
return (PDeachSig == SIGKILL);
+#elif defined(__FreeBSD__)
+ int PDeachSig = 0, PDisableSig = 0;
+ if (procctl(P_PID, 0, PROC_PDEATHSIG_STATUS, &PDeachSig) == 0 &&
+ PDeachSig == SIGKILL)
+ procctl(P_PID, 0, PROC_PDEATHSIG_CTL, &PDisableSig);
+ return (PDeachSig == SIGKILL);
#else
return 0;
#endif
}
-COMPILER_RT_VISIBILITY void lprofRestoreSigKill() {
+COMPILER_RT_VISIBILITY void lprofRestoreSigKill(void) {
#if defined(__linux__)
prctl(PR_SET_PDEATHSIG, SIGKILL);
+#elif defined(__FreeBSD__)
+ int PEnableSig = SIGKILL;
+ procctl(P_PID, 0, PROC_PDEATHSIG_CTL, &PEnableSig);
#endif
}
COMPILER_RT_VISIBILITY int lprofReleaseMemoryPagesToOS(uintptr_t Begin,
uintptr_t End) {
+#if defined(__ve__)
+ // VE doesn't support madvise.
+ return 0;
+#else
size_t PageSize = getpagesize();
uintptr_t BeginAligned = lprofRoundUpTo((uintptr_t)Begin, PageSize);
uintptr_t EndAligned = lprofRoundDownTo((uintptr_t)End, PageSize);
#endif
}
return 0;
+#endif
}
COMPILER_RT_VISIBILITY uint32_t VPMaxNumValsPerSite =
INSTR_PROF_DEFAULT_NUM_VAL_PER_SITE;
-COMPILER_RT_VISIBILITY void lprofSetupValueProfiler() {
+COMPILER_RT_VISIBILITY void lprofSetupValueProfiler(void) {
const char *Str = 0;
Str = getenv("LLVM_VP_MAX_NUM_VALS_PER_SITE");
if (Str && Str[0]) {
/*
* A wrapper struct that represents value profile runtime data.
* Like InstrProfRecord class which is used by profiling host tools,
- * ValueProfRuntimeRecord also implements the abstract intefaces defined in
+ * ValueProfRuntimeRecord also implements the abstract interfaces defined in
* ValueProfRecordClosure so that the runtime data can be serialized using
* shared C implementation.
*/
getFirstValueProfRecord, getNumValueDataForSiteWrapper,
getValueProfDataSizeWrapper, getNextNValueData};
-COMPILER_RT_VISIBILITY VPDataReaderType *lprofGetVPDataReader() {
+COMPILER_RT_VISIBILITY VPDataReaderType *lprofGetVPDataReader(void) {
return &TheVPDataReader;
}
* user has not specified one. Set this up by moving the runtime's copy of this
* symbol to an object file within the archive.
*/
-COMPILER_RT_WEAK uint64_t INSTR_PROF_RAW_VERSION_VAR = INSTR_PROF_RAW_VERSION;
+COMPILER_RT_VISIBILITY COMPILER_RT_WEAK uint64_t INSTR_PROF_RAW_VERSION_VAR =
+ INSTR_PROF_RAW_VERSION;
COMPILER_RT_VISIBILITY uint8_t *DynamicBufferIOBuffer = 0;
COMPILER_RT_VISIBILITY uint32_t VPBufferSize = 0;
-/* The buffer writer is reponsponsible in keeping writer state
+/* The buffer writer is responsible in keeping writer state
* across the call.
*/
COMPILER_RT_VISIBILITY uint32_t lprofBufferWriter(ProfDataWriter *This,
/* Match logic in __llvm_profile_write_buffer(). */
const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
- const uint64_t *CountersBegin = __llvm_profile_begin_counters();
- const uint64_t *CountersEnd = __llvm_profile_end_counters();
+ const char *CountersBegin = __llvm_profile_begin_counters();
+ const char *CountersEnd = __llvm_profile_end_counters();
const char *NamesBegin = __llvm_profile_begin_names();
const char *NamesEnd = __llvm_profile_end_names();
return lprofWriteDataImpl(Writer, DataBegin, DataEnd, CountersBegin,
COMPILER_RT_VISIBILITY int
lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
const __llvm_profile_data *DataEnd,
- const uint64_t *CountersBegin, const uint64_t *CountersEnd,
+ const char *CountersBegin, const char *CountersEnd,
VPDataReaderType *VPDataReader, const char *NamesBegin,
const char *NamesEnd, int SkipNameDataWrite) {
+ int DebugInfoCorrelate =
+ (__llvm_profile_get_version() & VARIANT_MASK_DBG_CORRELATE) != 0ULL;
/* Calculate size of sections. */
- const uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
- const uint64_t CountersSize = CountersEnd - CountersBegin;
- const uint64_t NamesSize = NamesEnd - NamesBegin;
+ const uint64_t DataSectionSize =
+ DebugInfoCorrelate ? 0 : __llvm_profile_get_data_size(DataBegin, DataEnd);
+ const uint64_t NumData =
+ DebugInfoCorrelate ? 0 : __llvm_profile_get_num_data(DataBegin, DataEnd);
+ const uint64_t CountersSectionSize =
+ __llvm_profile_get_counters_size(CountersBegin, CountersEnd);
+ const uint64_t NumCounters =
+ __llvm_profile_get_num_counters(CountersBegin, CountersEnd);
+ const uint64_t NamesSize = DebugInfoCorrelate ? 0 : NamesEnd - NamesBegin;
/* Create the header. */
__llvm_profile_header Header;
- if (!DataSize)
- return 0;
-
/* Determine how much padding is needed before/after the counters and after
* the names. */
uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters,
PaddingBytesAfterNames;
__llvm_profile_get_padding_sizes_for_counters(
- DataSize, CountersSize, NamesSize, &PaddingBytesBeforeCounters,
- &PaddingBytesAfterCounters, &PaddingBytesAfterNames);
-
+ DataSectionSize, CountersSectionSize, NamesSize,
+ &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters,
+ &PaddingBytesAfterNames);
+
+ {
+ // TODO: Unfortunately the header's fields are named DataSize and
+ // CountersSize when they should be named NumData and NumCounters,
+ // respectively.
+ const uint64_t CountersSize = NumCounters;
+ const uint64_t DataSize = NumData;
/* Initialize header structure. */
#define INSTR_PROF_RAW_HEADER(Type, Name, Init) Header.Name = Init;
#include "profile/InstrProfData.inc"
+ }
+
+ /* On WIN64, label differences are truncated 32-bit values. Truncate
+ * CountersDelta to match. */
+#ifdef _WIN64
+ Header.CountersDelta = (uint32_t)Header.CountersDelta;
+#endif
+
+ /* The data and names sections are omitted in lightweight mode. */
+ if (DebugInfoCorrelate) {
+ Header.CountersDelta = 0;
+ Header.NamesDelta = 0;
+ }
/* Write the profile header. */
ProfDataIOVec IOVec[] = {{&Header, sizeof(__llvm_profile_header), 1, 0}};
/* Write the profile data. */
ProfDataIOVec IOVecData[] = {
- {DataBegin, sizeof(__llvm_profile_data), DataSize, 0},
+ {DebugInfoCorrelate ? NULL : DataBegin, sizeof(uint8_t), DataSectionSize,
+ 0},
{NULL, sizeof(uint8_t), PaddingBytesBeforeCounters, 1},
- {CountersBegin, sizeof(uint64_t), CountersSize, 0},
+ {CountersBegin, sizeof(uint8_t), CountersSectionSize, 0},
{NULL, sizeof(uint8_t), PaddingBytesAfterCounters, 1},
- {SkipNameDataWrite ? NULL : NamesBegin, sizeof(uint8_t), NamesSize, 0},
+ {(SkipNameDataWrite || DebugInfoCorrelate) ? NULL : NamesBegin,
+ sizeof(uint8_t), NamesSize, 0},
{NULL, sizeof(uint8_t), PaddingBytesAfterNames, 1}};
if (Writer->Write(Writer, IOVecData, sizeof(IOVecData) / sizeof(*IOVecData)))
return -1;
off_t offset) {
#if SANITIZER_NETBSD
return __mmap(addr, length, prot, flags, fd, 0, offset);
-#elif defined(__x86_64__) && (SANITIZER_FREEBSD)
+#elif SANITIZER_FREEBSD && (defined(__aarch64__) || defined(__x86_64__))
return (void *)__syscall(SYS_mmap, addr, length, prot, flags, fd, offset);
#else
return (void *)syscall(SYS_mmap, addr, length, prot, flags, fd, offset);
sanitizer_mac.cpp
sanitizer_mutex.cpp
sanitizer_netbsd.cpp
- sanitizer_persistent_allocator.cpp
sanitizer_platform_limits_freebsd.cpp
sanitizer_platform_limits_linux.cpp
sanitizer_platform_limits_netbsd.cpp
sanitizer_solaris.cpp
sanitizer_stoptheworld_fuchsia.cpp
sanitizer_stoptheworld_mac.cpp
+ sanitizer_stoptheworld_win.cpp
sanitizer_suppressions.cpp
sanitizer_tls_get_addr.cpp
sanitizer_thread_registry.cpp
set(SANITIZER_SYMBOLIZER_SOURCES
sanitizer_allocator_report.cpp
sanitizer_chained_origin_depot.cpp
+ sanitizer_stack_store.cpp
sanitizer_stackdepot.cpp
sanitizer_stacktrace.cpp
sanitizer_stacktrace_libcdep.cpp
sancov_flags.inc
sanitizer_addrhashmap.h
sanitizer_allocator.h
- sanitizer_allocator_bytemap.h
sanitizer_allocator_checks.h
sanitizer_allocator_combined.h
+ sanitizer_allocator_dlsym.h
sanitizer_allocator_interface.h
sanitizer_allocator_internal.h
sanitizer_allocator_local_cache.h
sanitizer_dbghelp.h
sanitizer_deadlock_detector.h
sanitizer_deadlock_detector_interface.h
+ sanitizer_dense_map.h
+ sanitizer_dense_map_info.h
sanitizer_errno.h
sanitizer_errno_codes.h
sanitizer_file.h
sanitizer_flag_parser.h
sanitizer_flags.h
sanitizer_flags.inc
+ sanitizer_flat_map.h
sanitizer_freebsd.h
sanitizer_fuchsia.h
sanitizer_getauxval.h
sanitizer_interceptors_ioctl_netbsd.inc
sanitizer_interface_internal.h
sanitizer_internal_defs.h
+ sanitizer_leb128.h
sanitizer_lfstack.h
sanitizer_libc.h
sanitizer_libignore.h
sanitizer_linux.h
sanitizer_list.h
sanitizer_local_address_space_view.h
+ sanitizer_lzw.h
sanitizer_mac.h
sanitizer_malloc_mac.inc
sanitizer_mutex.h
- sanitizer_persistent_allocator.h
sanitizer_placement_new.h
sanitizer_platform.h
sanitizer_platform_interceptors.h
sanitizer_report_decorator.h
sanitizer_ring_buffer.h
sanitizer_signal_interceptors.inc
+ sanitizer_stack_store.h
sanitizer_stackdepot.h
sanitizer_stackdepotbase.h
sanitizer_stacktrace.h
sanitizer_syscall_linux_arm.inc
sanitizer_syscall_linux_x86_64.inc
sanitizer_syscall_linux_riscv64.inc
+ sanitizer_syscall_linux_loongarch64.inc
sanitizer_syscalls_netbsd.inc
sanitizer_thread_registry.h
sanitizer_thread_safety.h
HAVE_RPC_XDR_H=${HAVE_RPC_XDR_H})
set(SANITIZER_CFLAGS ${SANITIZER_COMMON_CFLAGS})
+
+# Too many existing bugs, needs cleanup.
+append_list_if(COMPILER_RT_HAS_WNO_FORMAT -Wno-format SANITIZER_CFLAGS)
+
append_rtti_flag(OFF SANITIZER_CFLAGS)
append_list_if(SANITIZER_LIMIT_FRAME_SIZE -Wframe-larger-than=570
add_compiler_rt_object_libraries(SanitizerCommonWeakInterception
${SANITIZER_COMMON_SUPPORTED_OS}
ARCHS ${SANITIZER_COMMON_SUPPORTED_ARCH}
- SOURCES sanitizer_win_weak_interception.cpp
+ SOURCES
+ sanitizer_win_weak_interception.cpp
CFLAGS ${SANITIZER_CFLAGS} -DSANITIZER_DYNAMIC
DEFS ${SANITIZER_COMMON_DEFINITIONS})
add_compiler_rt_object_libraries(SancovWeakInterception
${SANITIZER_COMMON_SUPPORTED_OS}
ARCHS ${SANITIZER_COMMON_SUPPORTED_ARCH}
- SOURCES sanitizer_coverage_win_weak_interception.cpp
+ SOURCES
+ sanitizer_coverage_win_weak_interception.cpp
CFLAGS ${SANITIZER_CFLAGS} -DSANITIZER_DYNAMIC
DEFS ${SANITIZER_COMMON_DEFINITIONS})
add_compiler_rt_object_libraries(SanitizerCommonDllThunk
${SANITIZER_COMMON_SUPPORTED_OS}
ARCHS ${SANITIZER_COMMON_SUPPORTED_ARCH}
- SOURCES sanitizer_win_dll_thunk.cpp
+ SOURCES
+ sanitizer_win_dll_thunk.cpp
CFLAGS ${SANITIZER_CFLAGS} -DSANITIZER_DLL_THUNK
DEFS ${SANITIZER_COMMON_DEFINITIONS})
add_compiler_rt_object_libraries(SancovDllThunk
${SANITIZER_COMMON_SUPPORTED_OS}
ARCHS ${SANITIZER_COMMON_SUPPORTED_ARCH}
- SOURCES sanitizer_coverage_win_dll_thunk.cpp
- sanitizer_coverage_win_sections.cpp
+ SOURCES
+ sanitizer_coverage_win_dll_thunk.cpp
+ sanitizer_coverage_win_sections.cpp
CFLAGS ${SANITIZER_CFLAGS} -DSANITIZER_DLL_THUNK
DEFS ${SANITIZER_COMMON_DEFINITIONS})
add_compiler_rt_object_libraries(SanitizerCommonDynamicRuntimeThunk
${SANITIZER_COMMON_SUPPORTED_OS}
ARCHS ${SANITIZER_COMMON_SUPPORTED_ARCH}
- SOURCES sanitizer_win_dynamic_runtime_thunk.cpp
+ SOURCES
+ sanitizer_win_dynamic_runtime_thunk.cpp
CFLAGS ${SANITIZER_CFLAGS} ${DYNAMIC_RUNTIME_THUNK_CFLAGS}
DEFS ${SANITIZER_COMMON_DEFINITIONS})
add_compiler_rt_object_libraries(SancovDynamicRuntimeThunk
${SANITIZER_COMMON_SUPPORTED_OS}
ARCHS ${SANITIZER_COMMON_SUPPORTED_ARCH}
- SOURCES sanitizer_coverage_win_dynamic_runtime_thunk.cpp
- sanitizer_coverage_win_sections.cpp
+ SOURCES
+ sanitizer_coverage_win_dynamic_runtime_thunk.cpp
+ sanitizer_coverage_win_sections.cpp
CFLAGS ${SANITIZER_CFLAGS} ${DYNAMIC_RUNTIME_THUNK_CFLAGS}
DEFS ${SANITIZER_COMMON_DEFINITIONS})
endif()
#endif
SANCOV_FLAG(bool, symbolize, true,
- "If set, converage information will be symbolized by sancov tool "
+ "If set, coverage information will be symbolized by sancov tool "
"after dumping.")
SANCOV_FLAG(bool, help, false, "Print flags help.")
// the current thread has exclusive access to the data
// if !h.exists() then the element never existed
// }
+// {
+// Map::Handle h(&m, addr, false, true);
+// this will create a new element or return a handle to an existing element
+// if !h.created() this thread does *not* have exclusive access to the data
+// }
template<typename T, uptr kSize>
class AddrHashMap {
private:
static const uptr kBucketSize = 3;
struct Bucket {
- RWMutex mtx;
+ Mutex mtx;
atomic_uintptr_t add;
Cell cells[kBucketSize];
};
bool create_;
};
+ typedef void (*ForEachCallback)(const uptr key, const T &val, void *arg);
+ // ForEach acquires a lock on each bucket while iterating over
+ // elements. Note that this only ensures that the structure of the hashmap is
+ // unchanged, there may be a data race to the element itself.
+ void ForEach(ForEachCallback cb, void *arg);
+
private:
friend class Handle;
Bucket *table_;
uptr calcHash(uptr addr);
};
+template <typename T, uptr kSize>
+void AddrHashMap<T, kSize>::ForEach(ForEachCallback cb, void *arg) {
+ for (uptr n = 0; n < kSize; n++) {
+ Bucket *bucket = &table_[n];
+
+ ReadLock lock(&bucket->mtx);
+
+ for (uptr i = 0; i < kBucketSize; i++) {
+ Cell *c = &bucket->cells[i];
+ uptr addr1 = atomic_load(&c->addr, memory_order_acquire);
+ if (addr1 != 0)
+ cb(addr1, c->val, arg);
+ }
+
+ // Iterate over any additional cells.
+ if (AddBucket *add =
+ (AddBucket *)atomic_load(&bucket->add, memory_order_acquire)) {
+ for (uptr i = 0; i < add->size; i++) {
+ Cell *c = &add->cells[i];
+ uptr addr1 = atomic_load(&c->addr, memory_order_acquire);
+ if (addr1 != 0)
+ cb(addr1, c->val, arg);
+ }
+ }
+ }
+}
+
template<typename T, uptr kSize>
AddrHashMap<T, kSize>::Handle::Handle(AddrHashMap<T, kSize> *map, uptr addr) {
map_ = map;
}
template <typename T, uptr kSize>
-void AddrHashMap<T, kSize>::acquire(Handle *h) NO_THREAD_SAFETY_ANALYSIS {
+void AddrHashMap<T, kSize>::acquire(Handle *h)
+ SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
uptr addr = h->addr_;
uptr hash = calcHash(addr);
Bucket *b = &table_[hash];
}
template <typename T, uptr kSize>
- void AddrHashMap<T, kSize>::release(Handle *h) NO_THREAD_SAFETY_ANALYSIS {
+ void AddrHashMap<T, kSize>::release(Handle *h)
+ SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
if (!h->cell_)
return;
Bucket *b = h->bucket_;
#include "sanitizer_allocator_internal.h"
#include "sanitizer_atomic.h"
#include "sanitizer_common.h"
+#include "sanitizer_platform.h"
namespace __sanitizer {
const char *PrimaryAllocatorName = "SizeClassAllocator";
const char *SecondaryAllocatorName = "LargeMmapAllocator";
-// ThreadSanitizer for Go uses libc malloc/free.
-#if defined(SANITIZER_USE_MALLOC)
-# if SANITIZER_LINUX && !SANITIZER_ANDROID
-extern "C" void *__libc_malloc(uptr size);
-# if !SANITIZER_GO
-extern "C" void *__libc_memalign(uptr alignment, uptr size);
-# endif
-extern "C" void *__libc_realloc(void *ptr, uptr size);
-extern "C" void __libc_free(void *ptr);
-# else
-# include <stdlib.h>
-# define __libc_malloc malloc
-# if !SANITIZER_GO
-static void *__libc_memalign(uptr alignment, uptr size) {
- void *p;
- uptr error = posix_memalign(&p, alignment, size);
- if (error) return nullptr;
- return p;
-}
-# endif
-# define __libc_realloc realloc
-# define __libc_free free
-# endif
-
-static void *RawInternalAlloc(uptr size, InternalAllocatorCache *cache,
- uptr alignment) {
- (void)cache;
-#if !SANITIZER_GO
- if (alignment == 0)
- return __libc_malloc(size);
- else
- return __libc_memalign(alignment, size);
-#else
- // Windows does not provide __libc_memalign/posix_memalign. It provides
- // __aligned_malloc, but the allocated blocks can't be passed to free,
- // they need to be passed to __aligned_free. InternalAlloc interface does
- // not account for such requirement. Alignemnt does not seem to be used
- // anywhere in runtime, so just call __libc_malloc for now.
- DCHECK_EQ(alignment, 0);
- return __libc_malloc(size);
-#endif
-}
-
-static void *RawInternalRealloc(void *ptr, uptr size,
- InternalAllocatorCache *cache) {
- (void)cache;
- return __libc_realloc(ptr, size);
-}
-
-static void RawInternalFree(void *ptr, InternalAllocatorCache *cache) {
- (void)cache;
- __libc_free(ptr);
-}
-
-InternalAllocator *internal_allocator() {
- return 0;
-}
-
-#else // SANITIZER_GO || defined(SANITIZER_USE_MALLOC)
-
static ALIGNED(64) char internal_alloc_placeholder[sizeof(InternalAllocator)];
static atomic_uint8_t internal_allocator_initialized;
static StaticSpinMutex internal_alloc_init_mu;
internal_allocator()->Deallocate(cache, ptr);
}
-#endif // SANITIZER_GO || defined(SANITIZER_USE_MALLOC)
-
static void NORETURN ReportInternalAllocatorOutOfMemory(uptr requested_size) {
SetAllocatorOutOfMemory();
Report("FATAL: %s: internal allocator is out of memory trying to allocate "
RawInternalFree(addr, cache);
}
+void InternalAllocatorLock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
+ internal_allocator_cache_mu.Lock();
+ internal_allocator()->ForceLock();
+}
+
+void InternalAllocatorUnlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
+ internal_allocator()->ForceUnlock();
+ internal_allocator_cache_mu.Unlock();
+}
+
// LowLevelAllocator
constexpr uptr kLowLevelAllocatorDefaultAlignment = 8;
static uptr low_level_alloc_min_alignment = kLowLevelAllocatorDefaultAlignment;
"allocator_may_return_null=1\n");
}
+static atomic_uint8_t rss_limit_exceeded;
+
+bool IsRssLimitExceeded() {
+ return atomic_load(&rss_limit_exceeded, memory_order_relaxed);
+}
+
+void SetRssLimitExceeded(bool limit_exceeded) {
+ atomic_store(&rss_limit_exceeded, limit_exceeded, memory_order_relaxed);
+}
+
} // namespace __sanitizer
#define SANITIZER_ALLOCATOR_H
#include "sanitizer_common.h"
+#include "sanitizer_flat_map.h"
#include "sanitizer_internal_defs.h"
#include "sanitizer_lfstack.h"
#include "sanitizer_libc.h"
void PrintHintAllocatorCannotReturnNull();
-// Allocators call these callbacks on mmap/munmap.
-struct NoOpMapUnmapCallback {
- void OnMap(uptr p, uptr size) const { }
- void OnUnmap(uptr p, uptr size) const { }
-};
-
// Callback type for iterating over chunks.
typedef void (*ForEachChunkCallback)(uptr chunk, void *arg);
#include "sanitizer_allocator_size_class_map.h"
#include "sanitizer_allocator_stats.h"
#include "sanitizer_allocator_primary64.h"
-#include "sanitizer_allocator_bytemap.h"
#include "sanitizer_allocator_primary32.h"
#include "sanitizer_allocator_local_cache.h"
#include "sanitizer_allocator_secondary.h"
#include "sanitizer_allocator_combined.h"
+bool IsRssLimitExceeded();
+void SetRssLimitExceeded(bool limit_exceeded);
+
} // namespace __sanitizer
#endif // SANITIZER_ALLOCATOR_H
return new_p;
}
- bool PointerIsMine(void *p) {
+ bool PointerIsMine(const void *p) const {
if (primary_.PointerIsMine(p))
return true;
return secondary_.PointerIsMine(p);
}
- bool FromPrimary(void *p) {
- return primary_.PointerIsMine(p);
- }
+ bool FromPrimary(const void *p) const { return primary_.PointerIsMine(p); }
void *GetMetaData(const void *p) {
if (primary_.PointerIsMine(p))
// This function does the same as GetBlockBegin, but is much faster.
// Must be called with the allocator locked.
- void *GetBlockBeginFastLocked(void *p) {
+ void *GetBlockBeginFastLocked(const void *p) {
if (primary_.PointerIsMine(p))
return primary_.GetBlockBegin(p);
return secondary_.GetBlockBeginFastLocked(p);
// ForceLock() and ForceUnlock() are needed to implement Darwin malloc zone
// introspection API.
- void ForceLock() NO_THREAD_SAFETY_ANALYSIS {
+ void ForceLock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
primary_.ForceLock();
secondary_.ForceLock();
}
- void ForceUnlock() NO_THREAD_SAFETY_ANALYSIS {
+ void ForceUnlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
secondary_.ForceUnlock();
primary_.ForceUnlock();
}
--- /dev/null
+//===-- sanitizer_allocator_dlsym.h -----------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Hack: Sanitizer initializer calls dlsym which may need to allocate and call
+// back into uninitialized sanitizer.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_ALLOCATOR_DLSYM_H
+#define SANITIZER_ALLOCATOR_DLSYM_H
+
+#include "sanitizer_allocator_internal.h"
+
+namespace __sanitizer {
+
+template <typename Details>
+struct DlSymAllocator {
+ static bool Use() {
+ // Fuchsia doesn't use dlsym-based interceptors.
+ return !SANITIZER_FUCHSIA && UNLIKELY(Details::UseImpl());
+ }
+
+ static bool PointerIsMine(const void *ptr) {
+ // Fuchsia doesn't use dlsym-based interceptors.
+ return !SANITIZER_FUCHSIA &&
+ UNLIKELY(internal_allocator()->FromPrimary(ptr));
+ }
+
+ static void *Allocate(uptr size_in_bytes) {
+ void *ptr = InternalAlloc(size_in_bytes, nullptr, kWordSize);
+ CHECK(internal_allocator()->FromPrimary(ptr));
+ Details::OnAllocate(ptr,
+ internal_allocator()->GetActuallyAllocatedSize(ptr));
+ return ptr;
+ }
+
+ static void *Callocate(SIZE_T nmemb, SIZE_T size) {
+ void *ptr = InternalCalloc(nmemb, size);
+ CHECK(internal_allocator()->FromPrimary(ptr));
+ Details::OnAllocate(ptr,
+ internal_allocator()->GetActuallyAllocatedSize(ptr));
+ return ptr;
+ }
+
+ static void Free(void *ptr) {
+ uptr size = internal_allocator()->GetActuallyAllocatedSize(ptr);
+ Details::OnFree(ptr, size);
+ InternalFree(ptr);
+ }
+
+ static void *Realloc(void *ptr, uptr new_size) {
+ if (!ptr)
+ return Allocate(new_size);
+ CHECK(internal_allocator()->FromPrimary(ptr));
+ if (!new_size) {
+ Free(ptr);
+ return nullptr;
+ }
+ uptr size = internal_allocator()->GetActuallyAllocatedSize(ptr);
+ uptr memcpy_size = Min(new_size, size);
+ void *new_ptr = Allocate(new_size);
+ if (new_ptr)
+ internal_memcpy(new_ptr, ptr, memcpy_size);
+ Free(ptr);
+ return new_ptr;
+ }
+
+ static void OnAllocate(const void *ptr, uptr size) {}
+ static void OnFree(const void *ptr, uptr size) {}
+};
+
+} // namespace __sanitizer
+
+#endif // SANITIZER_ALLOCATOR_DLSYM_H
void *InternalCalloc(uptr count, uptr size,
InternalAllocatorCache *cache = nullptr);
void InternalFree(void *p, InternalAllocatorCache *cache = nullptr);
+void InternalAllocatorLock();
+void InternalAllocatorUnlock();
InternalAllocator *internal_allocator();
} // namespace __sanitizer
sci->free_list.push_front(b);
}
- bool PointerIsMine(const void *p) {
+ bool PointerIsMine(const void *p) const {
uptr mem = reinterpret_cast<uptr>(p);
if (SANITIZER_SIGN_EXTENDED_ADDRESSES)
mem &= (kSpaceSize - 1);
return GetSizeClass(p) != 0;
}
- uptr GetSizeClass(const void *p) {
- return possible_regions[ComputeRegionId(reinterpret_cast<uptr>(p))];
+ uptr GetSizeClass(const void *p) const {
+ uptr id = ComputeRegionId(reinterpret_cast<uptr>(p));
+ return possible_regions.contains(id) ? possible_regions[id] : 0;
}
void *GetBlockBegin(const void *p) {
// ForceLock() and ForceUnlock() are needed to implement Darwin malloc zone
// introspection API.
- void ForceLock() NO_THREAD_SAFETY_ANALYSIS {
+ void ForceLock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
for (uptr i = 0; i < kNumClasses; i++) {
GetSizeClassInfo(i)->mutex.Lock();
}
}
- void ForceUnlock() NO_THREAD_SAFETY_ANALYSIS {
+ void ForceUnlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
for (int i = kNumClasses - 1; i >= 0; i--) {
GetSizeClassInfo(i)->mutex.Unlock();
}
// Iterate over all existing chunks.
// The allocator must be locked when calling this function.
- void ForEachChunk(ForEachChunkCallback callback, void *arg) {
+ void ForEachChunk(ForEachChunkCallback callback, void *arg) const {
for (uptr region = 0; region < kNumPossibleRegions; region++)
- if (possible_regions[region]) {
+ if (possible_regions.contains(region) && possible_regions[region]) {
uptr chunk_size = ClassIdToSize(possible_regions[region]);
uptr max_chunks_in_region = kRegionSize / (chunk_size + kMetadataSize);
uptr region_beg = region * kRegionSize;
return res;
}
- uptr ComputeRegionBeg(uptr mem) {
- return mem & ~(kRegionSize - 1);
- }
+ uptr ComputeRegionBeg(uptr mem) const { return mem & ~(kRegionSize - 1); }
uptr AllocateRegion(AllocatorStats *stat, uptr class_id) {
DCHECK_LT(class_id, kNumClasses);
MapUnmapCallback().OnMap(res, kRegionSize);
stat->Add(AllocatorStatMapped, kRegionSize);
CHECK(IsAligned(res, kRegionSize));
- possible_regions.set(ComputeRegionId(res), static_cast<u8>(class_id));
+ possible_regions[ComputeRegionId(res)] = class_id;
return res;
}
void ForceReleaseToOS() {
MemoryMapperT memory_mapper(*this);
for (uptr class_id = 1; class_id < kNumClasses; class_id++) {
- BlockingMutexLock l(&GetRegionInfo(class_id)->mutex);
+ Lock l(&GetRegionInfo(class_id)->mutex);
MaybeReleaseToOS(&memory_mapper, class_id, true /*force*/);
}
}
uptr region_beg = GetRegionBeginBySizeClass(class_id);
CompactPtrT *free_array = GetFreeArray(region_beg);
- BlockingMutexLock l(®ion->mutex);
+ Lock l(®ion->mutex);
uptr old_num_chunks = region->num_freed_chunks;
uptr new_num_freed_chunks = old_num_chunks + n_chunks;
// Failure to allocate free array space while releasing memory is non
uptr region_beg = GetRegionBeginBySizeClass(class_id);
CompactPtrT *free_array = GetFreeArray(region_beg);
- BlockingMutexLock l(®ion->mutex);
+ Lock l(®ion->mutex);
#if SANITIZER_WINDOWS
/* On Windows unmapping of memory during __sanitizer_purge_allocator is
explicit and immediate, so unmapped regions must be explicitly mapped back
CHECK(kMetadataSize);
uptr class_id = GetSizeClass(p);
uptr size = ClassIdToSize(class_id);
+ if (!size)
+ return nullptr;
uptr chunk_idx = GetChunkIdx(reinterpret_cast<uptr>(p), size);
uptr region_beg = GetRegionBeginBySizeClass(class_id);
return reinterpret_cast<void *>(GetMetadataEnd(region_beg) -
UnmapWithCallbackOrDie((uptr)address_range.base(), address_range.size());
}
- static void FillMemoryProfile(uptr start, uptr rss, bool file, uptr *stats,
- uptr stats_size) {
- for (uptr class_id = 0; class_id < stats_size; class_id++)
+ static void FillMemoryProfile(uptr start, uptr rss, bool file, uptr *stats) {
+ for (uptr class_id = 0; class_id < kNumClasses; class_id++)
if (stats[class_id] == start)
stats[class_id] = rss;
}
Printf(
"%s %02zd (%6zd): mapped: %6zdK allocs: %7zd frees: %7zd inuse: %6zd "
"num_freed_chunks %7zd avail: %6zd rss: %6zdK releases: %6zd "
- "last released: %6zdK region: 0x%zx\n",
+ "last released: %6lldK region: 0x%zx\n",
region->exhausted ? "F" : " ", class_id, ClassIdToSize(class_id),
region->mapped_user >> 10, region->stats.n_allocated,
region->stats.n_freed, in_use, region->num_freed_chunks, avail_chunks,
uptr rss_stats[kNumClasses];
for (uptr class_id = 0; class_id < kNumClasses; class_id++)
rss_stats[class_id] = SpaceBeg() + kRegionSize * class_id;
- GetMemoryProfile(FillMemoryProfile, rss_stats, kNumClasses);
+ GetMemoryProfile(FillMemoryProfile, rss_stats);
uptr total_mapped = 0;
uptr total_rss = 0;
// ForceLock() and ForceUnlock() are needed to implement Darwin malloc zone
// introspection API.
- void ForceLock() NO_THREAD_SAFETY_ANALYSIS {
+ void ForceLock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
for (uptr i = 0; i < kNumClasses; i++) {
GetRegionInfo(i)->mutex.Lock();
}
}
- void ForceUnlock() NO_THREAD_SAFETY_ANALYSIS {
+ void ForceUnlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
for (int i = (int)kNumClasses - 1; i >= 0; i--) {
GetRegionInfo(i)->mutex.Unlock();
}
static const uptr kRegionSize = kSpaceSize / kNumClassesRounded;
// FreeArray is the array of free-d chunks (stored as 4-byte offsets).
- // In the worst case it may reguire kRegionSize/SizeClassMap::kMinSize
+ // In the worst case it may require kRegionSize/SizeClassMap::kMinSize
// elements, but in reality this will not happen. For simplicity we
// dedicate 1/8 of the region's virtual space to FreeArray.
static const uptr kFreeArraySize = kRegionSize / 8;
};
struct ALIGNED(SANITIZER_CACHE_LINE_SIZE) RegionInfo {
- BlockingMutex mutex;
+ Mutex mutex;
uptr num_freed_chunks; // Number of elements in the freearray.
uptr mapped_free_array; // Bytes mapped for freearray.
uptr allocated_user; // Bytes allocated for user memory.
void NORETURN ReportOutOfMemory(uptr requested_size, const StackTrace *stack) {
{
ScopedAllocatorErrorReport report("out-of-memory", stack);
- Report("ERROR: %s: allocator is out of memory trying to allocate 0x%zx "
- "bytes\n", SanitizerToolName, requested_size);
+ ERROR_OOM("allocator is trying to allocate 0x%zx bytes\n", requested_size);
}
Die();
}
return res;
}
- bool PointerIsMine(const void *p) {
+ bool PointerIsMine(const void *p) const {
return GetBlockBegin(p) != nullptr;
}
return GetHeader(p) + 1;
}
- void *GetBlockBegin(const void *ptr) {
+ void *GetBlockBegin(const void *ptr) const {
uptr p = reinterpret_cast<uptr>(ptr);
SpinMutexLock l(&mutex_);
uptr nearest_chunk = 0;
// This function does the same as GetBlockBegin, but is much faster.
// Must be called with the allocator locked.
- void *GetBlockBeginFastLocked(void *ptr) {
+ void *GetBlockBeginFastLocked(const void *ptr) {
mutex_.CheckLocked();
uptr p = reinterpret_cast<uptr>(ptr);
uptr n = n_chunks_;
// ForceLock() and ForceUnlock() are needed to implement Darwin malloc zone
// introspection API.
- void ForceLock() ACQUIRE(mutex_) { mutex_.Lock(); }
+ void ForceLock() SANITIZER_ACQUIRE(mutex_) { mutex_.Lock(); }
- void ForceUnlock() RELEASE(mutex_) { mutex_.Unlock(); }
+ void ForceUnlock() SANITIZER_RELEASE(mutex_) { mutex_.Unlock(); }
// Iterate over all existing chunks.
// The allocator must be locked when calling this function.
return GetHeader(reinterpret_cast<uptr>(p));
}
- void *GetUser(const Header *h) {
+ void *GetUser(const Header *h) const {
CHECK(IsAligned((uptr)h, page_size_));
return reinterpret_cast<void*>(reinterpret_cast<uptr>(h) + page_size_);
}
struct Stats {
uptr n_allocs, n_frees, currently_allocated, max_allocated, by_size_log[64];
} stats;
- StaticSpinMutex mutex_;
+ mutable StaticSpinMutex mutex_;
};
uptr cached = MaxCachedHint(s) * s;
if (i == kBatchClassID)
d = p = l = 0;
- Printf("c%02zd => s: %zd diff: +%zd %02zd%% l %zd "
- "cached: %zd %zd; id %zd\n",
- i, Size(i), d, p, l, MaxCachedHint(s), cached, ClassID(s));
+ Printf(
+ "c%02zu => s: %zu diff: +%zu %02zu%% l %zu cached: %zu %zu; id %zu\n",
+ i, Size(i), d, p, l, MaxCachedHint(s), cached, ClassID(s));
total_cached += cached;
prev_s = s;
}
- Printf("Total cached: %zd\n", total_cached);
+ Printf("Total cached: %zu\n", total_cached);
}
static void Validate() {
//
//===----------------------------------------------------------------------===//
//
-// Various support for assemebler.
+// Various support for assembler.
//
//===----------------------------------------------------------------------===//
#if defined(__ELF__) && (defined(__GNU__) || defined(__FreeBSD__) || \
defined(__Fuchsia__) || defined(__linux__))
// clang-format off
-#define NO_EXEC_STACK_DIRECTIVE .section .note.GNU-stack,"",%progbits // NOLINT
+#define NO_EXEC_STACK_DIRECTIVE .section .note.GNU-stack,"",%progbits
// clang-format on
#else
#define NO_EXEC_STACK_DIRECTIVE
#endif
+
+#if (defined(__x86_64__) || defined(__i386__)) && defined(__has_include) && __has_include(<cet.h>)
+#include <cet.h>
+#endif
+#ifndef _CET_ENDBR
+#define _CET_ENDBR
+#endif
inline bool atomic_compare_exchange_strong(volatile T *a, typename T::Type *cmp,
typename T::Type xchg,
memory_order mo) {
- typedef typename T::Type Type;
- Type cmpv = *cmp;
- Type prev;
- prev = __sync_val_compare_and_swap(&a->val_dont_use, cmpv, xchg);
- if (prev == cmpv) return true;
- *cmp = prev;
- return false;
+ // Transitioned from __sync_val_compare_and_swap to support targets like
+ // SPARC V8 that cannot inline atomic cmpxchg. __atomic_compare_exchange
+ // can then be resolved from libatomic. __ATOMIC_SEQ_CST is used to best
+ // match the __sync builtin memory order.
+ return __atomic_compare_exchange(&a->val_dont_use, cmp, &xchg, false,
+ __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
}
template<typename T>
// This include provides explicit template instantiations for atomic_uint64_t
// on MIPS32, which does not directly support 8 byte atomics. It has to
// proceed the template definitions above.
-#if defined(_MIPS_SIM) && defined(_ABIO32)
- #include "sanitizer_atomic_clang_mips.h"
+#if defined(_MIPS_SIM) && defined(_ABIO32) && _MIPS_SIM == _ABIO32
+# include "sanitizer_atomic_clang_mips.h"
#endif
#undef ATOMIC_ORDER
// MIPS32 does not support atomics > 4 bytes. To address this lack of
// functionality, the sanitizer library provides helper methods which use an
-// internal spin lock mechanism to emulate atomic oprations when the size is
+// internal spin lock mechanism to emulate atomic operations when the size is
// 8 bytes.
static void __spin_lock(volatile int *lock) {
while (__sync_lock_test_and_set(lock, 1))
#include "sanitizer_chained_origin_depot.h"
+#include "sanitizer_stackdepotbase.h"
+
namespace __sanitizer {
-bool ChainedOriginDepot::ChainedOriginDepotNode::eq(
- u32 hash, const args_type &args) const {
- return here_id == args.here_id && prev_id == args.prev_id;
-}
+namespace {
+struct ChainedOriginDepotDesc {
+ u32 here_id;
+ u32 prev_id;
+};
-uptr ChainedOriginDepot::ChainedOriginDepotNode::storage_size(
- const args_type &args) {
- return sizeof(ChainedOriginDepotNode);
+struct ChainedOriginDepotNode {
+ using hash_type = u32;
+ u32 link;
+ u32 here_id;
+ u32 prev_id;
+
+ typedef ChainedOriginDepotDesc args_type;
+
+ bool eq(hash_type hash, const args_type &args) const;
+
+ static uptr allocated() { return 0; }
+
+ static hash_type hash(const args_type &args);
+
+ static bool is_valid(const args_type &args);
+
+ void store(u32 id, const args_type &args, hash_type other_hash);
+
+ args_type load(u32 id) const;
+
+ struct Handle {
+ const ChainedOriginDepotNode *node_ = nullptr;
+ u32 id_ = 0;
+ Handle(const ChainedOriginDepotNode *node, u32 id) : node_(node), id_(id) {}
+ bool valid() const { return node_; }
+ u32 id() const { return id_; }
+ int here_id() const { return node_->here_id; }
+ int prev_id() const { return node_->prev_id; }
+ };
+
+ static Handle get_handle(u32 id);
+
+ typedef Handle handle_type;
+};
+
+} // namespace
+
+static StackDepotBase<ChainedOriginDepotNode, 4, 20> depot;
+
+bool ChainedOriginDepotNode::eq(hash_type hash, const args_type &args) const {
+ return here_id == args.here_id && prev_id == args.prev_id;
}
/* This is murmur2 hash for the 64->32 bit case.
split, or one of two reserved values (-1) or (-2). Either case can
dominate depending on the workload.
*/
-u32 ChainedOriginDepot::ChainedOriginDepotNode::hash(const args_type &args) {
+ChainedOriginDepotNode::hash_type ChainedOriginDepotNode::hash(
+ const args_type &args) {
const u32 m = 0x5bd1e995;
const u32 seed = 0x9747b28c;
const u32 r = 24;
return h;
}
-bool ChainedOriginDepot::ChainedOriginDepotNode::is_valid(
- const args_type &args) {
- return true;
-}
+bool ChainedOriginDepotNode::is_valid(const args_type &args) { return true; }
-void ChainedOriginDepot::ChainedOriginDepotNode::store(const args_type &args,
- u32 other_hash) {
+void ChainedOriginDepotNode::store(u32 id, const args_type &args,
+ hash_type other_hash) {
here_id = args.here_id;
prev_id = args.prev_id;
}
-ChainedOriginDepot::ChainedOriginDepotNode::args_type
-ChainedOriginDepot::ChainedOriginDepotNode::load() const {
+ChainedOriginDepotNode::args_type ChainedOriginDepotNode::load(u32 id) const {
args_type ret = {here_id, prev_id};
return ret;
}
-ChainedOriginDepot::ChainedOriginDepotNode::Handle
-ChainedOriginDepot::ChainedOriginDepotNode::get_handle() {
- return Handle(this);
+ChainedOriginDepotNode::Handle ChainedOriginDepotNode::get_handle(u32 id) {
+ return Handle(&depot.nodes[id], id);
}
ChainedOriginDepot::ChainedOriginDepot() {}
-StackDepotStats *ChainedOriginDepot::GetStats() { return depot.GetStats(); }
+StackDepotStats ChainedOriginDepot::GetStats() const {
+ return depot.GetStats();
+}
bool ChainedOriginDepot::Put(u32 here_id, u32 prev_id, u32 *new_id) {
ChainedOriginDepotDesc desc = {here_id, prev_id};
bool inserted;
- ChainedOriginDepotNode::Handle h = depot.Put(desc, &inserted);
- *new_id = h.valid() ? h.id() : 0;
+ *new_id = depot.Put(desc, &inserted);
return inserted;
}
void ChainedOriginDepot::UnlockAll() { depot.UnlockAll(); }
+void ChainedOriginDepot::TestOnlyUnmap() { depot.TestOnlyUnmap(); }
+
} // namespace __sanitizer
#define SANITIZER_CHAINED_ORIGIN_DEPOT_H
#include "sanitizer_common.h"
-#include "sanitizer_stackdepotbase.h"
namespace __sanitizer {
ChainedOriginDepot();
// Gets the statistic of the origin chain storage.
- StackDepotStats *GetStats();
+ StackDepotStats GetStats() const;
// Stores a chain with StackDepot ID here_id and previous chain ID prev_id.
// If successful, returns true and the new chain id new_id.
void LockAll();
void UnlockAll();
+ void TestOnlyUnmap();
private:
- struct ChainedOriginDepotDesc {
- u32 here_id;
- u32 prev_id;
- };
-
- struct ChainedOriginDepotNode {
- ChainedOriginDepotNode *link;
- u32 id;
- u32 here_id;
- u32 prev_id;
-
- typedef ChainedOriginDepotDesc args_type;
-
- bool eq(u32 hash, const args_type &args) const;
-
- static uptr storage_size(const args_type &args);
-
- static u32 hash(const args_type &args);
-
- static bool is_valid(const args_type &args);
-
- void store(const args_type &args, u32 other_hash);
-
- args_type load() const;
-
- struct Handle {
- ChainedOriginDepotNode *node_;
- Handle() : node_(nullptr) {}
- explicit Handle(ChainedOriginDepotNode *node) : node_(node) {}
- bool valid() { return node_; }
- u32 id() { return node_->id; }
- int here_id() { return node_->here_id; }
- int prev_id() { return node_->prev_id; }
- };
-
- Handle get_handle();
-
- typedef Handle handle_type;
- };
-
- StackDepotBase<ChainedOriginDepotNode, 4, 20> depot;
-
ChainedOriginDepot(const ChainedOriginDepot &) = delete;
void operator=(const ChainedOriginDepot &) = delete;
};
//===----------------------------------------------------------------------===//
#include "sanitizer_common.h"
+
#include "sanitizer_allocator_interface.h"
#include "sanitizer_allocator_internal.h"
#include "sanitizer_atomic.h"
#include "sanitizer_flags.h"
+#include "sanitizer_interface_internal.h"
#include "sanitizer_libc.h"
#include "sanitizer_placement_new.h"
Die();
}
recursion_count++;
- Report("ERROR: %s failed to "
- "%s 0x%zx (%zd) bytes of %s (error code: %d)\n",
- SanitizerToolName, mmap_type, size, size, mem_type, err);
+ if (ErrorIsOOM(err)) {
+ ERROR_OOM("failed to %s 0x%zx (%zd) bytes of %s (error code: %d)\n",
+ mmap_type, size, size, mem_type, err);
+ } else {
+ Report(
+ "ERROR: %s failed to "
+ "%s 0x%zx (%zd) bytes of %s (error code: %d)\n",
+ SanitizerToolName, mmap_type, size, size, mem_type, err);
+ }
#if !SANITIZER_GO
DumpProcessMap();
#endif
set(module_name, base_address);
arch_ = arch;
internal_memcpy(uuid_, uuid, sizeof(uuid_));
+ uuid_size_ = kModuleUUIDSize;
instrumented_ = instrumented;
}
+void LoadedModule::setUuid(const char *uuid, uptr size) {
+ if (size > kModuleUUIDSize)
+ size = kModuleUUIDSize;
+ internal_memcpy(uuid_, uuid, size);
+ uuid_size_ = size;
+}
+
void LoadedModule::clear() {
InternalFree(full_name_);
base_address_ = 0;
- max_executable_address_ = 0;
+ max_address_ = 0;
full_name_ = nullptr;
arch_ = kModuleArchUnknown;
internal_memset(uuid_, 0, kModuleUUIDSize);
AddressRange *r =
new(mem) AddressRange(beg, end, executable, writable, name);
ranges_.push_back(r);
- if (executable && end > max_executable_address_)
- max_executable_address_ = end;
+ max_address_ = Max(max_address_, end);
}
bool LoadedModule::containsAddress(uptr address) const {
static MallocFreeHook MFHooks[kMaxMallocFreeHooks];
-void RunMallocHooks(const void *ptr, uptr size) {
+void RunMallocHooks(void *ptr, uptr size) {
+ __sanitizer_malloc_hook(ptr, size);
for (int i = 0; i < kMaxMallocFreeHooks; i++) {
auto hook = MFHooks[i].malloc_hook;
- if (!hook) return;
+ if (!hook)
+ break;
hook(ptr, size);
}
}
-void RunFreeHooks(const void *ptr) {
+void RunFreeHooks(void *ptr) {
+ __sanitizer_free_hook(ptr);
for (int i = 0; i < kMaxMallocFreeHooks; i++) {
auto hook = MFHooks[i].free_hook;
- if (!hook) return;
+ if (!hook)
+ break;
hook(ptr);
}
}
}
void SleepForMillis(unsigned millis) { internal_usleep((u64)millis * 1000); }
+void WaitForDebugger(unsigned seconds, const char *label) {
+ if (seconds) {
+ Report("Sleeping for %u second(s) %s\n", seconds, label);
+ SleepForSeconds(seconds);
+ }
+}
+
} // namespace __sanitizer
using namespace __sanitizer;
void (*free_hook)(const void *)) {
return InstallMallocFreeHooks(malloc_hook, free_hook);
}
+
+// Provide default (no-op) implementation of malloc hooks.
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_malloc_hook, void *ptr,
+ uptr size) {
+ (void)ptr;
+ (void)size;
+}
+
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_free_hook, void *ptr) {
+ (void)ptr;
+}
+
} // extern "C"
#define SANITIZER_COMMON_H
#include "sanitizer_flags.h"
-#include "sanitizer_interface_internal.h"
#include "sanitizer_internal_defs.h"
#include "sanitizer_libc.h"
#include "sanitizer_list.h"
void MprotectMallocZones(void *addr, int prot);
+#if SANITIZER_WINDOWS
+// Zero previously mmap'd memory. Currently used only on Windows.
+bool ZeroMmapFixedRegion(uptr fixed_addr, uptr size) WARN_UNUSED_RESULT;
+#endif
+
#if SANITIZER_LINUX
// Unmap memory. Currently only used on Linux.
void UnmapFromTo(uptr from, uptr to);
bool DontDumpShadowMemory(uptr addr, uptr length);
// Check if the built VMA size matches the runtime one.
void CheckVMASize();
-void RunMallocHooks(const void *ptr, uptr size);
-void RunFreeHooks(const void *ptr);
+void RunMallocHooks(void *ptr, uptr size);
+void RunFreeHooks(void *ptr);
class ReservedAddressRange {
public:
};
typedef void (*fill_profile_f)(uptr start, uptr rss, bool file,
- /*out*/uptr *stats, uptr stats_size);
+ /*out*/ uptr *stats);
// Parse the contents of /proc/self/smaps and generate a memory profile.
-// |cb| is a tool-specific callback that fills the |stats| array containing
-// |stats_size| elements.
-void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size);
+// |cb| is a tool-specific callback that fills the |stats| array.
+void GetMemoryProfile(fill_profile_f cb, uptr *stats);
+void ParseUnixMemoryProfile(fill_profile_f cb, uptr *stats, char *smaps,
+ uptr smaps_len);
// Simple low-level (mmap-based) allocator for internal use. Doesn't have
// constructor, so all instances of LowLevelAllocator should be
void RawWrite(const char *buffer);
bool ColorizeReports();
void RemoveANSIEscapeSequencesFromString(char *buffer);
-void Printf(const char *format, ...);
-void Report(const char *format, ...);
+void Printf(const char *format, ...) FORMAT(1, 2);
+void Report(const char *format, ...) FORMAT(1, 2);
void SetPrintfAndReportCallback(void (*callback)(const char *));
#define VReport(level, ...) \
do { \
// Lock sanitizer error reporting and protects against nested errors.
class ScopedErrorReportLock {
public:
- ScopedErrorReportLock() ACQUIRE(mutex_) { Lock(); }
- ~ScopedErrorReportLock() RELEASE(mutex_) { Unlock(); }
+ ScopedErrorReportLock() SANITIZER_ACQUIRE(mutex_) { Lock(); }
+ ~ScopedErrorReportLock() SANITIZER_RELEASE(mutex_) { Unlock(); }
- static void Lock() ACQUIRE(mutex_);
- static void Unlock() RELEASE(mutex_);
- static void CheckLocked() CHECK_LOCKED(mutex_);
+ static void Lock() SANITIZER_ACQUIRE(mutex_);
+ static void Unlock() SANITIZER_RELEASE(mutex_);
+ static void CheckLocked() SANITIZER_CHECK_LOCKED(mutex_);
private:
static atomic_uintptr_t reporting_thread_;
bool AddressSpaceIsUnlimited();
void SetAddressSpaceUnlimited();
void AdjustStackSize(void *attr);
-void PlatformPrepareForSandboxing(__sanitizer_sandbox_arguments *args);
+void PlatformPrepareForSandboxing(void *args);
void SetSandboxingCallback(void (*f)());
void InitializeCoverage(bool enabled, const char *coverage_dir);
uptr GetTlsSize();
// Other
+void WaitForDebugger(unsigned seconds, const char *label);
void SleepForSeconds(unsigned seconds);
void SleepForMillis(unsigned millis);
u64 NanoTime();
const char *mmap_type, error_t err,
bool raw_report = false);
+// Returns true if the platform-specific error reported is an OOM error.
+bool ErrorIsOOM(error_t err);
+
+// This reports an error in the form:
+//
+// `ERROR: {{SanitizerToolName}}: out of memory: {{err_msg}}`
+//
+// Downstream tools that read sanitizer output will know that errors starting
+// in this format are specifically OOM errors.
+#define ERROR_OOM(err_msg, ...) \
+ Report("ERROR: %s: out of memory: " err_msg, SanitizerToolName, __VA_ARGS__)
+
// Specific tools may override behavior of "Die" function to do tool-specific
// job.
typedef void (*DieCallbackType)(void);
void SetCheckUnwindCallback(void (*callback)());
-// Callback will be called if soft_rss_limit_mb is given and the limit is
-// exceeded (exceeded==true) or if rss went down below the limit
-// (exceeded==false).
-// The callback should be registered once at the tool init time.
-void SetSoftRssLimitExceededCallback(void (*Callback)(bool exceeded));
-
// Functions related to signal handling.
typedef void (*SignalHandlerType)(int, void *, void *);
HandleSignalMode GetHandleSignalMode(int signum);
void ReportErrorSummary(const char *error_type, const StackTrace *trace,
const char *alt_tool_name = nullptr);
-void ReportMmapWriteExec(int prot);
+void ReportMmapWriteExec(int prot, int mflags);
// Math
#if SANITIZER_WINDOWS && !defined(__clang__) && !defined(__GNUC__)
return up;
}
-inline bool IsPowerOfTwo(uptr x) {
- return (x & (x - 1)) == 0;
-}
+inline constexpr bool IsPowerOfTwo(uptr x) { return (x & (x - 1)) == 0; }
inline uptr RoundUpToPowerOfTwo(uptr size) {
CHECK(size);
return 1ULL << (up + 1);
}
-inline uptr RoundUpTo(uptr size, uptr boundary) {
+inline constexpr uptr RoundUpTo(uptr size, uptr boundary) {
RAW_CHECK(IsPowerOfTwo(boundary));
return (size + boundary - 1) & ~(boundary - 1);
}
-inline uptr RoundDownTo(uptr x, uptr boundary) {
+inline constexpr uptr RoundDownTo(uptr x, uptr boundary) {
return x & ~(boundary - 1);
}
-inline bool IsAligned(uptr a, uptr alignment) {
+inline constexpr bool IsAligned(uptr a, uptr alignment) {
return (a & (alignment - 1)) == 0;
}
constexpr T Max(T a, T b) {
return a > b ? a : b;
}
+template <class T>
+constexpr T Abs(T a) {
+ return a < 0 ? -a : a;
+}
template<class T> void Swap(T& a, T& b) {
T tmp = a;
a = b;
buffer_.resize(1);
buffer_[0] = '\0';
}
- void append(const char *format, ...);
+ void append(const char *format, ...) FORMAT(2, 3);
const char *data() const { return buffer_.data(); }
char *data() { return buffer_.data(); }
// Works like std::lower_bound: finds the first element that is not less
// than the val.
-template <class Container,
+template <class Container, class T,
class Compare = CompareLess<typename Container::value_type>>
-uptr InternalLowerBound(const Container &v,
- const typename Container::value_type &val,
- Compare comp = {}) {
+uptr InternalLowerBound(const Container &v, const T &val, Compare comp = {}) {
uptr first = 0;
uptr last = v.size();
while (last > first) {
kModuleArchARMV7S,
kModuleArchARMV7K,
kModuleArchARM64,
- kModuleArchRISCV64
+ kModuleArchLoongArch64,
+ kModuleArchRISCV64,
+ kModuleArchHexagon
};
// Sorts and removes duplicates from the container.
v.resize(last + 1);
}
+constexpr uptr kDefaultFileMaxSize = FIRST_32_SECOND_64(1 << 26, 1 << 28);
+
// Opens the file 'file_name" and reads up to 'max_len' bytes.
// The resulting buffer is mmaped and stored in '*buff'.
// Returns true if file was successfully opened and read.
bool ReadFileToVector(const char *file_name,
InternalMmapVectorNoCtor<char> *buff,
- uptr max_len = 1 << 26, error_t *errno_p = nullptr);
+ uptr max_len = kDefaultFileMaxSize,
+ error_t *errno_p = nullptr);
// Opens the file 'file_name" and reads up to 'max_len' bytes.
// This function is less I/O efficient than ReadFileToVector as it may reread
// The total number of read bytes is stored in '*read_len'.
// Returns true if file was successfully opened and read.
bool ReadFileToBuffer(const char *file_name, char **buff, uptr *buff_size,
- uptr *read_len, uptr max_len = 1 << 26,
+ uptr *read_len, uptr max_len = kDefaultFileMaxSize,
error_t *errno_p = nullptr);
+int GetModuleAndOffsetForPc(uptr pc, char *module_name, uptr module_name_len,
+ uptr *pc_offset);
+
// When adding a new architecture, don't forget to also update
// script/asan_symbolize.py and sanitizer_symbolizer_libcdep.cpp.
inline const char *ModuleArchToString(ModuleArch arch) {
return "armv7k";
case kModuleArchARM64:
return "arm64";
+ case kModuleArchLoongArch64:
+ return "loongarch64";
case kModuleArchRISCV64:
return "riscv64";
+ case kModuleArchHexagon:
+ return "hexagon";
}
CHECK(0 && "Invalid module arch");
return "";
}
-const uptr kModuleUUIDSize = 16;
+const uptr kModuleUUIDSize = 32;
const uptr kMaxSegName = 16;
// Represents a binary loaded into virtual memory (e.g. this can be an
LoadedModule()
: full_name_(nullptr),
base_address_(0),
- max_executable_address_(0),
+ max_address_(0),
arch_(kModuleArchUnknown),
+ uuid_size_(0),
instrumented_(false) {
internal_memset(uuid_, 0, kModuleUUIDSize);
ranges_.clear();
void set(const char *module_name, uptr base_address);
void set(const char *module_name, uptr base_address, ModuleArch arch,
u8 uuid[kModuleUUIDSize], bool instrumented);
+ void setUuid(const char *uuid, uptr size);
void clear();
void addAddressRange(uptr beg, uptr end, bool executable, bool writable,
const char *name = nullptr);
const char *full_name() const { return full_name_; }
uptr base_address() const { return base_address_; }
- uptr max_executable_address() const { return max_executable_address_; }
+ uptr max_address() const { return max_address_; }
ModuleArch arch() const { return arch_; }
const u8 *uuid() const { return uuid_; }
+ uptr uuid_size() const { return uuid_size_; }
bool instrumented() const { return instrumented_; }
struct AddressRange {
private:
char *full_name_; // Owned.
uptr base_address_;
- uptr max_executable_address_;
+ uptr max_address_;
ModuleArch arch_;
+ uptr uuid_size_;
u8 uuid_[kModuleUUIDSize];
bool instrumented_;
IntrusiveList<AddressRange> ranges_;
#define SANITIZER_WIN_TRACE 0
#endif
-#if SANITIZER_MAC || SANITIZER_WIN_TRACE
+#if SANITIZER_APPLE || SANITIZER_WIN_TRACE
void LogFullErrorReport(const char *buffer);
#else
inline void LogFullErrorReport(const char *buffer) {}
#endif
-#if SANITIZER_LINUX || SANITIZER_MAC
+#if SANITIZER_LINUX || SANITIZER_APPLE
void WriteOneLineToSyslog(const char *s);
void LogMessageOnPrintf(const char *str);
#else
uptr sp;
uptr bp;
bool is_memory_access;
- enum WriteFlag { UNKNOWN, READ, WRITE } write_flag;
+ enum WriteFlag { Unknown, Read, Write } write_flag;
// In some cases the kernel cannot provide the true faulting address; `addr`
// will be zero then. This field allows to distinguish between these cases
};
void InitializePlatformEarly();
-void MaybeReexec();
template <typename Fn>
class RunOnDestruction {
T *end_ = nullptr;
};
-#define PRINTF_128(v) \
- (*((u8 *)&v + 0)), (*((u8 *)&v + 1)), (*((u8 *)&v + 2)), (*((u8 *)&v + 3)), \
- (*((u8 *)&v + 4)), (*((u8 *)&v + 5)), (*((u8 *)&v + 6)), \
- (*((u8 *)&v + 7)), (*((u8 *)&v + 8)), (*((u8 *)&v + 9)), \
- (*((u8 *)&v + 10)), (*((u8 *)&v + 11)), (*((u8 *)&v + 12)), \
- (*((u8 *)&v + 13)), (*((u8 *)&v + 14)), (*((u8 *)&v + 15))
-
} // namespace __sanitizer
inline void *operator new(__sanitizer::operator_new_size_type size,
- __sanitizer::LowLevelAllocator &alloc) { // NOLINT
+ __sanitizer::LowLevelAllocator &alloc) {
return alloc.Allocate(size);
}
// COMMON_INTERCEPTOR_FD_RELEASE
// COMMON_INTERCEPTOR_FD_ACCESS
// COMMON_INTERCEPTOR_SET_THREAD_NAME
-// COMMON_INTERCEPTOR_ON_DLOPEN
+// COMMON_INTERCEPTOR_DLOPEN
// COMMON_INTERCEPTOR_ON_EXIT
-// COMMON_INTERCEPTOR_MUTEX_PRE_LOCK
-// COMMON_INTERCEPTOR_MUTEX_POST_LOCK
-// COMMON_INTERCEPTOR_MUTEX_UNLOCK
-// COMMON_INTERCEPTOR_MUTEX_REPAIR
// COMMON_INTERCEPTOR_SET_PTHREAD_NAME
// COMMON_INTERCEPTOR_HANDLE_RECVMSG
// COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED
extern const short *_tolower_tab_;
#endif
+#if SANITIZER_MUSL && \
+ (defined(__i386__) || defined(__arm__) || SANITIZER_MIPS32 || SANITIZER_PPC32)
+// musl 1.2.0 on existing 32-bit architectures uses new symbol names for the
+// time-related functions that take 64-bit time_t values. See
+// https://musl.libc.org/time64.html
+#define adjtime __adjtime64
+#define adjtimex __adjtimex_time64
+#define aio_suspend __aio_suspend_time64
+#define clock_adjtime __clock_adjtime64
+#define clock_getres __clock_getres_time64
+#define clock_gettime __clock_gettime64
+#define clock_nanosleep __clock_nanosleep_time64
+#define clock_settime __clock_settime64
+#define cnd_timedwait __cnd_timedwait_time64
+#define ctime __ctime64
+#define ctime_r __ctime64_r
+#define difftime __difftime64
+#define dlsym __dlsym_time64
+#define fstatat __fstatat_time64
+#define fstat __fstat_time64
+#define ftime __ftime64
+#define futimens __futimens_time64
+#define futimesat __futimesat_time64
+#define futimes __futimes_time64
+#define getitimer __getitimer_time64
+#define getrusage __getrusage_time64
+#define gettimeofday __gettimeofday_time64
+#define gmtime __gmtime64
+#define gmtime_r __gmtime64_r
+#define localtime __localtime64
+#define localtime_r __localtime64_r
+#define lstat __lstat_time64
+#define lutimes __lutimes_time64
+#define mktime __mktime64
+#define mq_timedreceive __mq_timedreceive_time64
+#define mq_timedsend __mq_timedsend_time64
+#define mtx_timedlock __mtx_timedlock_time64
+#define nanosleep __nanosleep_time64
+#define ppoll __ppoll_time64
+#define pselect __pselect_time64
+#define pthread_cond_timedwait __pthread_cond_timedwait_time64
+#define pthread_mutex_timedlock __pthread_mutex_timedlock_time64
+#define pthread_rwlock_timedrdlock __pthread_rwlock_timedrdlock_time64
+#define pthread_rwlock_timedwrlock __pthread_rwlock_timedwrlock_time64
+#define pthread_timedjoin_np __pthread_timedjoin_np_time64
+#define recvmmsg __recvmmsg_time64
+#define sched_rr_get_interval __sched_rr_get_interval_time64
+#define select __select_time64
+#define semtimedop __semtimedop_time64
+#define sem_timedwait __sem_timedwait_time64
+#define setitimer __setitimer_time64
+#define settimeofday __settimeofday_time64
+#define sigtimedwait __sigtimedwait_time64
+#define stat __stat_time64
+#define stime __stime64
+#define thrd_sleep __thrd_sleep_time64
+#define timegm __timegm_time64
+#define timerfd_gettime __timerfd_gettime64
+#define timerfd_settime __timerfd_settime64
+#define timer_gettime __timer_gettime64
+#define timer_settime __timer_settime64
+#define timespec_get __timespec_get_time64
+#define time __time64
+#define utimensat __utimensat_time64
+#define utimes __utimes_time64
+#define utime __utime64
+#define wait3 __wait3_time64
+#define wait4 __wait4_time64
+#endif
+
// Platform-specific options.
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 0
#elif SANITIZER_WINDOWS64
#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 0
#else
#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 1
-#endif // SANITIZER_MAC
+#endif // SANITIZER_APPLE
#ifndef COMMON_INTERCEPTOR_INITIALIZE_RANGE
#define COMMON_INTERCEPTOR_INITIALIZE_RANGE(p, size) {}
#define COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd) {}
#endif
-#ifndef COMMON_INTERCEPTOR_MUTEX_PRE_LOCK
-#define COMMON_INTERCEPTOR_MUTEX_PRE_LOCK(ctx, m) {}
-#endif
-
-#ifndef COMMON_INTERCEPTOR_MUTEX_POST_LOCK
-#define COMMON_INTERCEPTOR_MUTEX_POST_LOCK(ctx, m) {}
-#endif
-
-#ifndef COMMON_INTERCEPTOR_MUTEX_UNLOCK
-#define COMMON_INTERCEPTOR_MUTEX_UNLOCK(ctx, m) {}
-#endif
-
-#ifndef COMMON_INTERCEPTOR_MUTEX_REPAIR
-#define COMMON_INTERCEPTOR_MUTEX_REPAIR(ctx, m) {}
-#endif
-
-#ifndef COMMON_INTERCEPTOR_MUTEX_INVALID
-#define COMMON_INTERCEPTOR_MUTEX_INVALID(ctx, m) {}
-#endif
-
#ifndef COMMON_INTERCEPTOR_HANDLE_RECVMSG
#define COMMON_INTERCEPTOR_HANDLE_RECVMSG(ctx, msg) ((void)(msg))
#endif
#define COMMON_INTERCEPTOR_READ_STRING(ctx, s, n) \
COMMON_INTERCEPTOR_READ_RANGE((ctx), (s), \
- common_flags()->strict_string_checks ? (REAL(strlen)(s)) + 1 : (n) )
+ common_flags()->strict_string_checks ? (internal_strlen(s)) + 1 : (n) )
-#ifndef COMMON_INTERCEPTOR_ON_DLOPEN
-#define COMMON_INTERCEPTOR_ON_DLOPEN(filename, flag) \
- CheckNoDeepBind(filename, flag);
+#ifndef COMMON_INTERCEPTOR_DLOPEN
+#define COMMON_INTERCEPTOR_DLOPEN(filename, flag) \
+ ({ CheckNoDeepBind(filename, flag); REAL(dlopen)(filename, flag); })
#endif
#ifndef COMMON_INTERCEPTOR_GET_TLS_RANGE
if (common_flags()->intercept_strndup) { \
COMMON_INTERCEPTOR_READ_STRING(ctx, s, Min(size, copy_length + 1)); \
} \
- COMMON_INTERCEPTOR_COPY_STRING(ctx, new_mem, s, copy_length); \
- internal_memcpy(new_mem, s, copy_length); \
- new_mem[copy_length] = '\0'; \
+ if (new_mem) { \
+ COMMON_INTERCEPTOR_COPY_STRING(ctx, new_mem, s, copy_length); \
+ internal_memcpy(new_mem, s, copy_length); \
+ new_mem[copy_length] = '\0'; \
+ } \
return new_mem;
#endif
if (domainname) COMMON_INTERCEPTOR_READ_STRING(ctx, domainname, 0);
char *domain = REAL(textdomain)(domainname);
if (domain) {
- COMMON_INTERCEPTOR_INITIALIZE_RANGE(domain, REAL(strlen)(domain) + 1);
+ COMMON_INTERCEPTOR_INITIALIZE_RANGE(domain, internal_strlen(domain) + 1);
}
return domain;
}
#if SANITIZER_INTERCEPT_STRSTR || SANITIZER_INTERCEPT_STRCASESTR
static inline void StrstrCheck(void *ctx, char *r, const char *s1,
const char *s2) {
- uptr len1 = REAL(strlen)(s1);
- uptr len2 = REAL(strlen)(s2);
+ uptr len1 = internal_strlen(s1);
+ uptr len2 = internal_strlen(s2);
COMMON_INTERCEPTOR_READ_STRING(ctx, s1, r ? r - s1 + len2 : len1 + 1);
COMMON_INTERCEPTOR_READ_RANGE(ctx, s2, len2 + 1);
}
// for subsequent calls). We do not need to check strtok's result.
// As the delimiters can change, we check them every call.
if (str != nullptr) {
- COMMON_INTERCEPTOR_READ_RANGE(ctx, str, REAL(strlen)(str) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, str, internal_strlen(str) + 1);
}
COMMON_INTERCEPTOR_READ_RANGE(ctx, delimiters,
- REAL(strlen)(delimiters) + 1);
+ internal_strlen(delimiters) + 1);
return REAL(strtok)(str, delimiters);
} else {
// However, when strict_string_checks is disabled we cannot check the
COMMON_INTERCEPTOR_READ_RANGE(ctx, delimiters, 1);
char *result = REAL(strtok)(str, delimiters);
if (result != nullptr) {
- COMMON_INTERCEPTOR_READ_RANGE(ctx, result, REAL(strlen)(result) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, result, internal_strlen(result) + 1);
} else if (str != nullptr) {
// No delimiter were found, it's safe to assume that the entire str was
// scanned.
- COMMON_INTERCEPTOR_READ_RANGE(ctx, str, REAL(strlen)(str) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, str, internal_strlen(str) + 1);
}
return result;
}
if (common_flags()->intercept_strchr) {
// Keep strlen as macro argument, as macro may ignore it.
COMMON_INTERCEPTOR_READ_STRING(ctx, s,
- (result ? result - s : REAL(strlen)(s)) + 1);
+ (result ? result - s : internal_strlen(s)) + 1);
}
return result;
}
return internal_strrchr(s, c);
COMMON_INTERCEPTOR_ENTER(ctx, strrchr, s, c);
if (common_flags()->intercept_strchr)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, s, REAL(strlen)(s) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, s, internal_strlen(s) + 1);
return REAL(strrchr)(s, c);
}
#define INIT_STRRCHR COMMON_INTERCEPT_FUNCTION(strrchr)
COMMON_INTERCEPTOR_ENTER(ctx, strspn, s1, s2);
SIZE_T r = REAL(strspn)(s1, s2);
if (common_flags()->intercept_strspn) {
- COMMON_INTERCEPTOR_READ_RANGE(ctx, s2, REAL(strlen)(s2) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, s2, internal_strlen(s2) + 1);
COMMON_INTERCEPTOR_READ_STRING(ctx, s1, r + 1);
}
return r;
COMMON_INTERCEPTOR_ENTER(ctx, strcspn, s1, s2);
SIZE_T r = REAL(strcspn)(s1, s2);
if (common_flags()->intercept_strspn) {
- COMMON_INTERCEPTOR_READ_RANGE(ctx, s2, REAL(strlen)(s2) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, s2, internal_strlen(s2) + 1);
COMMON_INTERCEPTOR_READ_STRING(ctx, s1, r + 1);
}
return r;
COMMON_INTERCEPTOR_ENTER(ctx, strpbrk, s1, s2);
char *r = REAL(strpbrk)(s1, s2);
if (common_flags()->intercept_strpbrk) {
- COMMON_INTERCEPTOR_READ_RANGE(ctx, s2, REAL(strlen)(s2) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, s2, internal_strlen(s2) + 1);
COMMON_INTERCEPTOR_READ_STRING(ctx, s1,
- r ? r - s1 + 1 : REAL(strlen)(s1) + 1);
+ r ? r - s1 + 1 : internal_strlen(s1) + 1);
}
return r;
}
// https://github.com/google/sanitizers/issues/321.
char *res = REAL(fgets)(s, size, file);
if (res)
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, s, REAL(strlen)(s) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, s, internal_strlen(s) + 1);
return res;
}
#define INIT_FGETS COMMON_INTERCEPT_FUNCTION(fgets)
// libc file streams can call user-supplied functions, see fopencookie.
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, fputs, s, file);
- if (!SANITIZER_MAC || s) { // `fputs(NULL, file)` is supported on Darwin.
- COMMON_INTERCEPTOR_READ_RANGE(ctx, s, REAL(strlen)(s) + 1);
+ if (!SANITIZER_APPLE || s) { // `fputs(NULL, file)` is supported on Darwin.
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, s, internal_strlen(s) + 1);
}
return REAL(fputs)(s, file);
}
// libc file streams can call user-supplied functions, see fopencookie.
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, puts, s);
- if (!SANITIZER_MAC || s) { // `puts(NULL)` is supported on Darwin.
- COMMON_INTERCEPTOR_READ_RANGE(ctx, s, REAL(strlen)(s) + 1);
+ if (!SANITIZER_APPLE || s) { // `puts(NULL)` is supported on Darwin.
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, s, internal_strlen(s) + 1);
}
return REAL(puts)(s);
}
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, prctl, option, arg2, arg3, arg4, arg5);
static const int PR_SET_NAME = 15;
+ static const int PR_SET_VMA = 0x53564d41;
+ static const int PR_SCHED_CORE = 62;
+ static const int PR_SCHED_CORE_GET = 0;
+ if (option == PR_SET_VMA && arg2 == 0UL) {
+ char *name = (char *)arg5;
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
+ }
int res = REAL(prctl(option, arg2, arg3, arg4, arg5));
if (option == PR_SET_NAME) {
char buff[16];
internal_strncpy(buff, (char *)arg2, 15);
buff[15] = 0;
COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, buff);
+ } else if (res != -1 && option == PR_SCHED_CORE && arg2 == PR_SCHED_CORE_GET) {
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (u64*)(arg5), sizeof(u64));
}
return res;
}
// Can not use COMMON_INTERCEPTOR_WRITE_RANGE here, because tm->tm_zone
// can point to shared memory and tsan would report a data race.
COMMON_INTERCEPTOR_INITIALIZE_RANGE(tm->tm_zone,
- REAL(strlen(tm->tm_zone)) + 1);
+ internal_strlen(tm->tm_zone) + 1);
}
#endif
}
char *res = REAL(ctime)(timep);
if (res) {
COMMON_INTERCEPTOR_READ_RANGE(ctx, timep, sizeof(*timep));
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
}
return res;
}
char *res = REAL(ctime_r)(timep, result);
if (res) {
COMMON_INTERCEPTOR_READ_RANGE(ctx, timep, sizeof(*timep));
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
}
return res;
}
char *res = REAL(asctime)(tm);
if (res) {
COMMON_INTERCEPTOR_READ_RANGE(ctx, tm, sizeof(*tm));
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
}
return res;
}
char *res = REAL(asctime_r)(tm, result);
if (res) {
COMMON_INTERCEPTOR_READ_RANGE(ctx, tm, sizeof(*tm));
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
}
return res;
}
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, strptime, s, format, tm);
if (format)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, format, REAL(strlen)(format) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, format, internal_strlen(format) + 1);
// FIXME: under ASan the call below may write to freed memory and corrupt
// its metadata. See
// https://github.com/google/sanitizers/issues/321.
const ioctl_desc *desc = ioctl_lookup(request);
ioctl_desc decoded_desc;
if (!desc) {
- VPrintf(2, "Decoding unknown ioctl 0x%x\n", request);
+ VPrintf(2, "Decoding unknown ioctl 0x%lx\n", request);
if (!ioctl_decode(request, &decoded_desc))
- Printf("WARNING: failed decoding unknown ioctl 0x%x\n", request);
+ Printf("WARNING: failed decoding unknown ioctl 0x%lx\n", request);
else
desc = &decoded_desc;
}
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd, sizeof(*pwd));
if (pwd->pw_name)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_name,
- REAL(strlen)(pwd->pw_name) + 1);
+ internal_strlen(pwd->pw_name) + 1);
if (pwd->pw_passwd)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_passwd,
- REAL(strlen)(pwd->pw_passwd) + 1);
+ internal_strlen(pwd->pw_passwd) + 1);
#if !SANITIZER_ANDROID
if (pwd->pw_gecos)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_gecos,
- REAL(strlen)(pwd->pw_gecos) + 1);
+ internal_strlen(pwd->pw_gecos) + 1);
#endif
-#if SANITIZER_MAC || SANITIZER_FREEBSD || SANITIZER_NETBSD
+#if SANITIZER_APPLE || SANITIZER_FREEBSD || SANITIZER_NETBSD
if (pwd->pw_class)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_class,
- REAL(strlen)(pwd->pw_class) + 1);
+ internal_strlen(pwd->pw_class) + 1);
#endif
if (pwd->pw_dir)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_dir,
- REAL(strlen)(pwd->pw_dir) + 1);
+ internal_strlen(pwd->pw_dir) + 1);
if (pwd->pw_shell)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_shell,
- REAL(strlen)(pwd->pw_shell) + 1);
+ internal_strlen(pwd->pw_shell) + 1);
}
}
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, grp, sizeof(*grp));
if (grp->gr_name)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, grp->gr_name,
- REAL(strlen)(grp->gr_name) + 1);
+ internal_strlen(grp->gr_name) + 1);
if (grp->gr_passwd)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, grp->gr_passwd,
- REAL(strlen)(grp->gr_passwd) + 1);
+ internal_strlen(grp->gr_passwd) + 1);
char **p = grp->gr_mem;
for (; *p; ++p) {
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *p, REAL(strlen)(*p) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *p, internal_strlen(*p) + 1);
}
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, grp->gr_mem,
(p - grp->gr_mem + 1) * sizeof(*p));
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, getpwnam, name);
if (name)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
__sanitizer_passwd *res = REAL(getpwnam)(name);
unpoison_passwd(ctx, res);
return res;
INTERCEPTOR(__sanitizer_group *, getgrnam, const char *name) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, getgrnam, name);
- COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
__sanitizer_group *res = REAL(getgrnam)(name);
unpoison_group(ctx, res);
return res;
char *buf, SIZE_T buflen, __sanitizer_passwd **result) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, getpwnam_r, name, pwd, buf, buflen, result);
- COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
// FIXME: under ASan the call below may write to freed memory and corrupt
// its metadata. See
// https://github.com/google/sanitizers/issues/321.
char *buf, SIZE_T buflen, __sanitizer_group **result) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, getgrnam_r, name, grp, buf, buflen, result);
- COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
// FIXME: under ASan the call below may write to freed memory and corrupt
// its metadata. See
// https://github.com/google/sanitizers/issues/321.
return res;
}
-#define INIT_CLOCK_GETCPUCLOCKID \
- COMMON_INTERCEPT_FUNCTION(clock_getcpuclockid);
+INTERCEPTOR(int, pthread_getcpuclockid, uptr thread,
+ __sanitizer_clockid_t *clockid) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, pthread_getcpuclockid, thread, clockid);
+ int res = REAL(pthread_getcpuclockid)(thread, clockid);
+ if (!res && clockid) {
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, clockid, sizeof *clockid);
+ }
+ return res;
+}
+
+#define INIT_CLOCK_GETCPUCLOCKID \
+ COMMON_INTERCEPT_FUNCTION(clock_getcpuclockid); \
+ COMMON_INTERCEPT_FUNCTION(pthread_getcpuclockid);
#else
#define INIT_CLOCK_GETCPUCLOCKID
#endif
ctx, pglob->gl_pathv, (pglob->gl_pathc + 1) * sizeof(*pglob->gl_pathv));
for (SIZE_T i = 0; i < pglob->gl_pathc; ++i) {
char *p = pglob->gl_pathv[i];
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, REAL(strlen)(p) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, internal_strlen(p) + 1);
}
}
static void *wrapped_gl_opendir(const char *s) {
COMMON_INTERCEPTOR_UNPOISON_PARAM(1);
- COMMON_INTERCEPTOR_INITIALIZE_RANGE(s, REAL(strlen)(s) + 1);
+ COMMON_INTERCEPTOR_INITIALIZE_RANGE(s, internal_strlen(s) + 1);
return pglob_copy->gl_opendir(s);
}
static int wrapped_gl_lstat(const char *s, void *st) {
COMMON_INTERCEPTOR_UNPOISON_PARAM(2);
- COMMON_INTERCEPTOR_INITIALIZE_RANGE(s, REAL(strlen)(s) + 1);
+ COMMON_INTERCEPTOR_INITIALIZE_RANGE(s, internal_strlen(s) + 1);
return pglob_copy->gl_lstat(s, st);
}
static int wrapped_gl_stat(const char *s, void *st) {
COMMON_INTERCEPTOR_UNPOISON_PARAM(2);
- COMMON_INTERCEPTOR_INITIALIZE_RANGE(s, REAL(strlen)(s) + 1);
+ COMMON_INTERCEPTOR_INITIALIZE_RANGE(s, internal_strlen(s) + 1);
return pglob_copy->gl_stat(s, st);
}
#define INIT_GLOB64
#endif // SANITIZER_INTERCEPT_GLOB64
+#if SANITIZER_INTERCEPT___B64_TO
+INTERCEPTOR(int, __b64_ntop, unsigned char const *src, SIZE_T srclength,
+ char *target, SIZE_T targsize) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, __b64_ntop, src, srclength, target, targsize);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, src, srclength);
+ int res = REAL(__b64_ntop)(src, srclength, target, targsize);
+ if (res >= 0)
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, target, res + 1);
+ return res;
+}
+INTERCEPTOR(int, __b64_pton, char const *src, char *target, SIZE_T targsize) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, __b64_pton, src, target, targsize);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, src, internal_strlen(src) + 1);
+ int res = REAL(__b64_pton)(src, target, targsize);
+ if (res >= 0)
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, target, res);
+ return res;
+}
+#define INIT___B64_TO \
+ COMMON_INTERCEPT_FUNCTION(__b64_ntop); \
+ COMMON_INTERCEPT_FUNCTION(__b64_pton);
+#else // SANITIZER_INTERCEPT___B64_TO
+#define INIT___B64_TO
+#endif // SANITIZER_INTERCEPT___B64_TO
+
+#if SANITIZER_INTERCEPT_DN_COMP_EXPAND
+# if __GLIBC_PREREQ(2, 34)
+// Changed with https://sourceware.org/git/?p=glibc.git;h=640bbdf
+# define DN_COMP_INTERCEPTOR_NAME dn_comp
+# define DN_EXPAND_INTERCEPTOR_NAME dn_expand
+# else
+# define DN_COMP_INTERCEPTOR_NAME __dn_comp
+# define DN_EXPAND_INTERCEPTOR_NAME __dn_expand
+# endif
+INTERCEPTOR(int, DN_COMP_INTERCEPTOR_NAME, unsigned char *exp_dn,
+ unsigned char *comp_dn, int length, unsigned char **dnptrs,
+ unsigned char **lastdnptr) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, DN_COMP_INTERCEPTOR_NAME, exp_dn, comp_dn,
+ length, dnptrs, lastdnptr);
+ int res = REAL(DN_COMP_INTERCEPTOR_NAME)(exp_dn, comp_dn, length, dnptrs,
+ lastdnptr);
+ if (res >= 0) {
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, comp_dn, res);
+ if (dnptrs && lastdnptr) {
+ unsigned char **p = dnptrs;
+ for (; p != lastdnptr && *p; ++p)
+ ;
+ if (p != lastdnptr)
+ ++p;
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dnptrs, (p - dnptrs) * sizeof(*p));
+ }
+ }
+ return res;
+}
+INTERCEPTOR(int, DN_EXPAND_INTERCEPTOR_NAME, unsigned char const *base,
+ unsigned char const *end, unsigned char const *src, char *dest,
+ int space) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, DN_EXPAND_INTERCEPTOR_NAME, base, end, src,
+ dest, space);
+ // TODO: add read check if __dn_comp intercept added
+ int res = REAL(DN_EXPAND_INTERCEPTOR_NAME)(base, end, src, dest, space);
+ if (res >= 0)
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dest, internal_strlen(dest) + 1);
+ return res;
+}
+# define INIT_DN_COMP_EXPAND \
+ COMMON_INTERCEPT_FUNCTION(DN_COMP_INTERCEPTOR_NAME); \
+ COMMON_INTERCEPT_FUNCTION(DN_EXPAND_INTERCEPTOR_NAME);
+#else // SANITIZER_INTERCEPT_DN_COMP_EXPAND
+# define INIT_DN_COMP_EXPAND
+#endif // SANITIZER_INTERCEPT_DN_COMP_EXPAND
+
+#if SANITIZER_INTERCEPT_POSIX_SPAWN
+
+template <class RealSpawnPtr>
+static int PosixSpawnImpl(void *ctx, RealSpawnPtr *real_posix_spawn, pid_t *pid,
+ const char *file_or_path, const void *file_actions,
+ const void *attrp, char *const argv[],
+ char *const envp[]) {
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, file_or_path,
+ internal_strlen(file_or_path) + 1);
+ if (argv) {
+ for (char *const *s = argv; ; ++s) {
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, s, sizeof(*s));
+ if (!*s) break;
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, *s, internal_strlen(*s) + 1);
+ }
+ }
+ if (envp) {
+ for (char *const *s = envp; ; ++s) {
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, s, sizeof(*s));
+ if (!*s) break;
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, *s, internal_strlen(*s) + 1);
+ }
+ }
+ int res =
+ real_posix_spawn(pid, file_or_path, file_actions, attrp, argv, envp);
+ if (res == 0)
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pid, sizeof(*pid));
+ return res;
+}
+INTERCEPTOR(int, posix_spawn, pid_t *pid, const char *path,
+ const void *file_actions, const void *attrp, char *const argv[],
+ char *const envp[]) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, posix_spawn, pid, path, file_actions, attrp,
+ argv, envp);
+ return PosixSpawnImpl(ctx, REAL(posix_spawn), pid, path, file_actions, attrp,
+ argv, envp);
+}
+INTERCEPTOR(int, posix_spawnp, pid_t *pid, const char *file,
+ const void *file_actions, const void *attrp, char *const argv[],
+ char *const envp[]) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, posix_spawnp, pid, file, file_actions, attrp,
+ argv, envp);
+ return PosixSpawnImpl(ctx, REAL(posix_spawnp), pid, file, file_actions, attrp,
+ argv, envp);
+}
+# define INIT_POSIX_SPAWN \
+ COMMON_INTERCEPT_FUNCTION(posix_spawn); \
+ COMMON_INTERCEPT_FUNCTION(posix_spawnp);
+#else // SANITIZER_INTERCEPT_POSIX_SPAWN
+# define INIT_POSIX_SPAWN
+#endif // SANITIZER_INTERCEPT_POSIX_SPAWN
+
#if SANITIZER_INTERCEPT_WAIT
// According to sys/wait.h, wait(), waitid(), waitpid() may have symbol version
// suffixes on Darwin. See the declaration of INTERCEPTOR_WITH_SUFFIX for
// its metadata. See
// https://github.com/google/sanitizers/issues/321.
char *res = REAL(inet_ntop)(af, src, dst, size);
- if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
+ if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
return res;
}
INTERCEPTOR(int, inet_pton, int af, const char *src, void *dst) {
INTERCEPTOR(int, inet_aton, const char *cp, void *dst) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, inet_aton, cp, dst);
- if (cp) COMMON_INTERCEPTOR_READ_RANGE(ctx, cp, REAL(strlen)(cp) + 1);
+ if (cp) COMMON_INTERCEPTOR_READ_RANGE(ctx, cp, internal_strlen(cp) + 1);
// FIXME: under ASan the call below may write to freed memory and corrupt
// its metadata. See
// https://github.com/google/sanitizers/issues/321.
struct __sanitizer_addrinfo **out) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, getaddrinfo, node, service, hints, out);
- if (node) COMMON_INTERCEPTOR_READ_RANGE(ctx, node, REAL(strlen)(node) + 1);
+ if (node) COMMON_INTERCEPTOR_READ_RANGE(ctx, node, internal_strlen(node) + 1);
if (service)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, service, REAL(strlen)(service) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, service, internal_strlen(service) + 1);
if (hints)
COMMON_INTERCEPTOR_READ_RANGE(ctx, hints, sizeof(__sanitizer_addrinfo));
// FIXME: under ASan the call below may write to freed memory and corrupt
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->ai_addr, p->ai_addrlen);
if (p->ai_canonname)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->ai_canonname,
- REAL(strlen)(p->ai_canonname) + 1);
+ internal_strlen(p->ai_canonname) + 1);
p = p->ai_next;
}
}
REAL(getnameinfo)(sockaddr, salen, host, hostlen, serv, servlen, flags);
if (res == 0) {
if (host && hostlen)
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, host, REAL(strlen)(host) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, host, internal_strlen(host) + 1);
if (serv && servlen)
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, serv, REAL(strlen)(serv) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, serv, internal_strlen(serv) + 1);
}
return res;
}
#endif
#if SANITIZER_INTERCEPT_GETSOCKNAME
-INTERCEPTOR(int, getsockname, int sock_fd, void *addr, int *addrlen) {
+INTERCEPTOR(int, getsockname, int sock_fd, void *addr, unsigned *addrlen) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, getsockname, sock_fd, addr, addrlen);
- COMMON_INTERCEPTOR_READ_RANGE(ctx, addrlen, sizeof(*addrlen));
- int addrlen_in = *addrlen;
+ unsigned addr_sz;
+ if (addrlen) {
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, addrlen, sizeof(*addrlen));
+ addr_sz = *addrlen;
+ }
// FIXME: under ASan the call below may write to freed memory and corrupt
// its metadata. See
// https://github.com/google/sanitizers/issues/321.
int res = REAL(getsockname)(sock_fd, addr, addrlen);
- if (res == 0) {
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, addr, Min(addrlen_in, *addrlen));
+ if (!res && addr && addrlen) {
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, addr, Min(addr_sz, *addrlen));
}
return res;
}
static void write_hostent(void *ctx, struct __sanitizer_hostent *h) {
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, h, sizeof(__sanitizer_hostent));
if (h->h_name)
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, h->h_name, REAL(strlen)(h->h_name) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, h->h_name, internal_strlen(h->h_name) + 1);
char **p = h->h_aliases;
while (*p) {
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *p, REAL(strlen)(*p) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *p, internal_strlen(*p) + 1);
++p;
}
COMMON_INTERCEPTOR_WRITE_RANGE(
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, getpeername, sockfd, addr, addrlen);
unsigned addr_sz;
- if (addrlen) addr_sz = *addrlen;
+ if (addrlen) {
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, addrlen, sizeof(*addrlen));
+ addr_sz = *addrlen;
+ }
// FIXME: under ASan the call below may write to freed memory and corrupt
// its metadata. See
// https://github.com/google/sanitizers/issues/321.
int res = REAL(getpeername)(sockfd, addr, addrlen);
- if (!res && addr && addrlen)
+ if (!res && addr && addrlen) {
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, addr, Min(addr_sz, *addrlen));
+ }
return res;
}
#define INIT_GETPEERNAME COMMON_INTERCEPT_FUNCTION(getpeername);
INTERCEPTOR(__sanitizer_dirent *, opendir, const char *path) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, opendir, path);
- COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
__sanitizer_dirent *res = REAL(opendir)(path);
if (res)
COMMON_INTERCEPTOR_DIR_ACQUIRE(ctx, path);
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, setlocale, category, locale);
if (locale)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, locale, REAL(strlen)(locale) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, locale, internal_strlen(locale) + 1);
char *res = REAL(setlocale)(category, locale);
if (res) {
- COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);
+ COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1);
unpoison_ctype_arrays(ctx);
}
return res;
// its metadata. See
// https://github.com/google/sanitizers/issues/321.
char *res = REAL(getcwd)(buf, size);
- if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
+ if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
return res;
}
#define INIT_GETCWD COMMON_INTERCEPT_FUNCTION(getcwd);
// its metadata. See
// https://github.com/google/sanitizers/issues/321.
char *res = REAL(get_current_dir_name)(fake);
- if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
+ if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
return res;
}
INTERCEPTOR(char *, realpath, const char *path, char *resolved_path) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, realpath, path, resolved_path);
- if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+ if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
// Workaround a bug in glibc where dlsym(RTLD_NEXT, ...) returns the oldest
// version of a versioned symbol. For realpath(), this gives us something
allocated_path = resolved_path = (char *)WRAP(malloc)(path_max + 1);
char *res = REAL(realpath)(path, resolved_path);
- if (allocated_path && !res) WRAP(free)(allocated_path);
- if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
+ if (allocated_path && !res)
+ WRAP(free)(allocated_path);
+ if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
return res;
}
-#define INIT_REALPATH COMMON_INTERCEPT_FUNCTION(realpath);
+# define INIT_REALPATH COMMON_INTERCEPT_FUNCTION(realpath);
#else
#define INIT_REALPATH
#endif
INTERCEPTOR(char *, canonicalize_file_name, const char *path) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, canonicalize_file_name, path);
- if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+ if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
char *res = REAL(canonicalize_file_name)(path);
- if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
+ if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
return res;
}
#define INIT_CANONICALIZE_FILE_NAME \
COMMON_INTERCEPTOR_ENTER(ctx, strerror, errnum);
COMMON_INTERCEPTOR_STRERROR();
char *res = REAL(strerror)(errnum);
- if (res) COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);
+ if (res) COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1);
return res;
}
#define INIT_STRERROR COMMON_INTERCEPT_FUNCTION(strerror);
// * GNU version returns message pointer, which points to either buf or some
// static storage.
#if ((_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600) && !_GNU_SOURCE) || \
- SANITIZER_MAC || SANITIZER_ANDROID || SANITIZER_NETBSD || \
+ SANITIZER_APPLE || SANITIZER_ANDROID || SANITIZER_NETBSD || \
SANITIZER_FREEBSD
// POSIX version. Spec is not clear on whether buf is NULL-terminated.
// At least on OSX, buf contents are valid even when the call fails.
// https://github.com/google/sanitizers/issues/321.
char *res = REAL(strerror_r)(errnum, buf, buflen);
if (res == buf)
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
else
- COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);
+ COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1);
return res;
}
#endif //(_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600) && !_GNU_SOURCE ||
- //SANITIZER_MAC
+ //SANITIZER_APPLE
#define INIT_STRERROR_R COMMON_INTERCEPT_FUNCTION(strerror_r);
#else
#define INIT_STRERROR_R
int res = REAL(__xpg_strerror_r)(errnum, buf, buflen);
// This version always returns a null-terminated string.
if (buf && buflen)
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, REAL(strlen)(buf) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, internal_strlen(buf) + 1);
return res;
}
#define INIT_XPG_STRERROR_R COMMON_INTERCEPT_FUNCTION(__xpg_strerror_r);
scandir_filter_f filter, scandir_compar_f compar) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, scandir, dirp, namelist, filter, compar);
- if (dirp) COMMON_INTERCEPTOR_READ_RANGE(ctx, dirp, REAL(strlen)(dirp) + 1);
+ if (dirp) COMMON_INTERCEPTOR_READ_RANGE(ctx, dirp, internal_strlen(dirp) + 1);
scandir_filter = filter;
scandir_compar = compar;
// FIXME: under ASan the call below may write to freed memory and corrupt
scandir64_filter_f filter, scandir64_compar_f compar) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, scandir64, dirp, namelist, filter, compar);
- if (dirp) COMMON_INTERCEPTOR_READ_RANGE(ctx, dirp, REAL(strlen)(dirp) + 1);
+ if (dirp) COMMON_INTERCEPTOR_READ_RANGE(ctx, dirp, internal_strlen(dirp) + 1);
scandir64_filter = filter;
scandir64_compar = compar;
// FIXME: under ASan the call below may write to freed memory and corrupt
INTERCEPTOR(int, wordexp, char *s, __sanitizer_wordexp_t *p, int flags) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, wordexp, s, p, flags);
- if (s) COMMON_INTERCEPTOR_READ_RANGE(ctx, s, REAL(strlen)(s) + 1);
+ if (s) COMMON_INTERCEPTOR_READ_RANGE(ctx, s, internal_strlen(s) + 1);
// FIXME: under ASan the call below may write to freed memory and corrupt
// its metadata. See
// https://github.com/google/sanitizers/issues/321.
int res = REAL(wordexp)(s, p, flags);
if (!res && p) {
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, sizeof(*p));
- if (p->we_wordc)
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->we_wordv,
- sizeof(*p->we_wordv) * p->we_wordc);
- for (uptr i = 0; i < p->we_wordc; ++i) {
+ uptr we_wordc =
+ ((flags & wordexp_wrde_dooffs) ? p->we_offs : 0) + p->we_wordc;
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->we_wordv,
+ sizeof(*p->we_wordv) * (we_wordc + 1));
+ for (uptr i = 0; i < we_wordc; ++i) {
char *w = p->we_wordv[i];
- if (w) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, w, REAL(strlen)(w) + 1);
+ if (w) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, w, internal_strlen(w) + 1);
}
}
return res;
if (res && size) {
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, size * sizeof(*res));
for (int i = 0; i < size; ++i)
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res[i], REAL(strlen(res[i])) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res[i], internal_strlen(res[i]) + 1);
}
return res;
}
#define INIT__EXIT
#endif
-#if SANITIZER_INTERCEPT_PTHREAD_MUTEX
-INTERCEPTOR(int, pthread_mutex_lock, void *m) {
- void *ctx;
- COMMON_INTERCEPTOR_ENTER(ctx, pthread_mutex_lock, m);
- COMMON_INTERCEPTOR_MUTEX_PRE_LOCK(ctx, m);
- int res = REAL(pthread_mutex_lock)(m);
- if (res == errno_EOWNERDEAD)
- COMMON_INTERCEPTOR_MUTEX_REPAIR(ctx, m);
- if (res == 0 || res == errno_EOWNERDEAD)
- COMMON_INTERCEPTOR_MUTEX_POST_LOCK(ctx, m);
- if (res == errno_EINVAL)
- COMMON_INTERCEPTOR_MUTEX_INVALID(ctx, m);
- return res;
-}
-
-INTERCEPTOR(int, pthread_mutex_unlock, void *m) {
- void *ctx;
- COMMON_INTERCEPTOR_ENTER(ctx, pthread_mutex_unlock, m);
- COMMON_INTERCEPTOR_MUTEX_UNLOCK(ctx, m);
- int res = REAL(pthread_mutex_unlock)(m);
- if (res == errno_EINVAL)
- COMMON_INTERCEPTOR_MUTEX_INVALID(ctx, m);
- return res;
-}
-
-#define INIT_PTHREAD_MUTEX_LOCK COMMON_INTERCEPT_FUNCTION(pthread_mutex_lock)
-#define INIT_PTHREAD_MUTEX_UNLOCK \
- COMMON_INTERCEPT_FUNCTION(pthread_mutex_unlock)
-#else
-#define INIT_PTHREAD_MUTEX_LOCK
-#define INIT_PTHREAD_MUTEX_UNLOCK
-#endif
-
-#if SANITIZER_INTERCEPT___PTHREAD_MUTEX
-INTERCEPTOR(int, __pthread_mutex_lock, void *m) {
- void *ctx;
- COMMON_INTERCEPTOR_ENTER(ctx, __pthread_mutex_lock, m);
- COMMON_INTERCEPTOR_MUTEX_PRE_LOCK(ctx, m);
- int res = REAL(__pthread_mutex_lock)(m);
- if (res == errno_EOWNERDEAD)
- COMMON_INTERCEPTOR_MUTEX_REPAIR(ctx, m);
- if (res == 0 || res == errno_EOWNERDEAD)
- COMMON_INTERCEPTOR_MUTEX_POST_LOCK(ctx, m);
- if (res == errno_EINVAL)
- COMMON_INTERCEPTOR_MUTEX_INVALID(ctx, m);
- return res;
-}
-
-INTERCEPTOR(int, __pthread_mutex_unlock, void *m) {
- void *ctx;
- COMMON_INTERCEPTOR_ENTER(ctx, __pthread_mutex_unlock, m);
- COMMON_INTERCEPTOR_MUTEX_UNLOCK(ctx, m);
- int res = REAL(__pthread_mutex_unlock)(m);
- if (res == errno_EINVAL)
- COMMON_INTERCEPTOR_MUTEX_INVALID(ctx, m);
- return res;
-}
-
-#define INIT___PTHREAD_MUTEX_LOCK \
- COMMON_INTERCEPT_FUNCTION(__pthread_mutex_lock)
-#define INIT___PTHREAD_MUTEX_UNLOCK \
- COMMON_INTERCEPT_FUNCTION(__pthread_mutex_unlock)
-#else
-#define INIT___PTHREAD_MUTEX_LOCK
-#define INIT___PTHREAD_MUTEX_UNLOCK
-#endif
-
#if SANITIZER_INTERCEPT___LIBC_MUTEX
-INTERCEPTOR(int, __libc_mutex_lock, void *m)
-ALIAS(WRAPPER_NAME(pthread_mutex_lock));
-
-INTERCEPTOR(int, __libc_mutex_unlock, void *m)
-ALIAS(WRAPPER_NAME(pthread_mutex_unlock));
-
INTERCEPTOR(int, __libc_thr_setcancelstate, int state, int *oldstate)
ALIAS(WRAPPER_NAME(pthread_setcancelstate));
-#define INIT___LIBC_MUTEX_LOCK COMMON_INTERCEPT_FUNCTION(__libc_mutex_lock)
-#define INIT___LIBC_MUTEX_UNLOCK COMMON_INTERCEPT_FUNCTION(__libc_mutex_unlock)
#define INIT___LIBC_THR_SETCANCELSTATE \
COMMON_INTERCEPT_FUNCTION(__libc_thr_setcancelstate)
#else
-#define INIT___LIBC_MUTEX_LOCK
-#define INIT___LIBC_MUTEX_UNLOCK
#define INIT___LIBC_THR_SETCANCELSTATE
#endif
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, mnt, sizeof(*mnt));
if (mnt->mnt_fsname)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, mnt->mnt_fsname,
- REAL(strlen)(mnt->mnt_fsname) + 1);
+ internal_strlen(mnt->mnt_fsname) + 1);
if (mnt->mnt_dir)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, mnt->mnt_dir,
- REAL(strlen)(mnt->mnt_dir) + 1);
+ internal_strlen(mnt->mnt_dir) + 1);
if (mnt->mnt_type)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, mnt->mnt_type,
- REAL(strlen)(mnt->mnt_type) + 1);
+ internal_strlen(mnt->mnt_type) + 1);
if (mnt->mnt_opts)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, mnt->mnt_opts,
- REAL(strlen)(mnt->mnt_opts) + 1);
+ internal_strlen(mnt->mnt_opts) + 1);
}
#endif
INTERCEPTOR(int, statfs, char *path, void *buf) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, statfs, path, buf);
- if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+ if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
// FIXME: under ASan the call below may write to freed memory and corrupt
// its metadata. See
// https://github.com/google/sanitizers/issues/321.
INTERCEPTOR(int, statfs64, char *path, void *buf) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, statfs64, path, buf);
- if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+ if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
// FIXME: under ASan the call below may write to freed memory and corrupt
// its metadata. See
// https://github.com/google/sanitizers/issues/321.
INTERCEPTOR(int, statvfs, char *path, void *buf) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, statvfs, path, buf);
- if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+ if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
// FIXME: under ASan the call below may write to freed memory and corrupt
// its metadata. See
// https://github.com/google/sanitizers/issues/321.
INTERCEPTOR(int, statvfs64, char *path, void *buf) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, statvfs64, path, buf);
- if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+ if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
// FIXME: under ASan the call below may write to freed memory and corrupt
// its metadata. See
// https://github.com/google/sanitizers/issues/321.
INTERCEPTOR(int, initgroups, char *user, u32 group) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, initgroups, user, group);
- if (user) COMMON_INTERCEPTOR_READ_RANGE(ctx, user, REAL(strlen)(user) + 1);
+ if (user) COMMON_INTERCEPTOR_READ_RANGE(ctx, user, internal_strlen(user) + 1);
int res = REAL(initgroups)(user, group);
return res;
}
COMMON_INTERCEPTOR_ENTER(ctx, ether_ntoa, addr);
if (addr) COMMON_INTERCEPTOR_READ_RANGE(ctx, addr, sizeof(*addr));
char *res = REAL(ether_ntoa)(addr);
- if (res) COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);
+ if (res) COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1);
return res;
}
INTERCEPTOR(__sanitizer_ether_addr *, ether_aton, char *buf) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, ether_aton, buf);
- if (buf) COMMON_INTERCEPTOR_READ_RANGE(ctx, buf, REAL(strlen)(buf) + 1);
+ if (buf) COMMON_INTERCEPTOR_READ_RANGE(ctx, buf, internal_strlen(buf) + 1);
__sanitizer_ether_addr *res = REAL(ether_aton)(buf);
if (res) COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, sizeof(*res));
return res;
// https://github.com/google/sanitizers/issues/321.
int res = REAL(ether_ntohost)(hostname, addr);
if (!res && hostname)
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, hostname, REAL(strlen)(hostname) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, hostname, internal_strlen(hostname) + 1);
return res;
}
INTERCEPTOR(int, ether_hostton, char *hostname, __sanitizer_ether_addr *addr) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, ether_hostton, hostname, addr);
if (hostname)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, hostname, REAL(strlen)(hostname) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, hostname, internal_strlen(hostname) + 1);
// FIXME: under ASan the call below may write to freed memory and corrupt
// its metadata. See
// https://github.com/google/sanitizers/issues/321.
char *hostname) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, ether_line, line, addr, hostname);
- if (line) COMMON_INTERCEPTOR_READ_RANGE(ctx, line, REAL(strlen)(line) + 1);
+ if (line) COMMON_INTERCEPTOR_READ_RANGE(ctx, line, internal_strlen(line) + 1);
// FIXME: under ASan the call below may write to freed memory and corrupt
// its metadata. See
// https://github.com/google/sanitizers/issues/321.
if (!res) {
if (addr) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, addr, sizeof(*addr));
if (hostname)
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, hostname, REAL(strlen)(hostname) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, hostname, internal_strlen(hostname) + 1);
}
return res;
}
// its metadata. See
// https://github.com/google/sanitizers/issues/321.
char *res = REAL(ether_ntoa_r)(addr, buf);
- if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
+ if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
return res;
}
INTERCEPTOR(__sanitizer_ether_addr *, ether_aton_r, char *buf,
__sanitizer_ether_addr *addr) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, ether_aton_r, buf, addr);
- if (buf) COMMON_INTERCEPTOR_READ_RANGE(ctx, buf, REAL(strlen)(buf) + 1);
+ if (buf) COMMON_INTERCEPTOR_READ_RANGE(ctx, buf, internal_strlen(buf) + 1);
// FIXME: under ASan the call below may write to freed memory and corrupt
// its metadata. See
// https://github.com/google/sanitizers/issues/321.
#define INIT_PTHREAD_ATTR_GETAFFINITY_NP
#endif
+#if SANITIZER_INTERCEPT_PTHREAD_GETAFFINITY_NP
+INTERCEPTOR(int, pthread_getaffinity_np, void *attr, SIZE_T cpusetsize,
+ void *cpuset) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, pthread_getaffinity_np, attr, cpusetsize,
+ cpuset);
+ // FIXME: under ASan the call below may write to freed memory and corrupt
+ // its metadata. See
+ // https://github.com/google/sanitizers/issues/321.
+ int res = REAL(pthread_getaffinity_np)(attr, cpusetsize, cpuset);
+ if (!res && cpusetsize && cpuset)
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cpuset, cpusetsize);
+ return res;
+}
+
+#define INIT_PTHREAD_GETAFFINITY_NP \
+ COMMON_INTERCEPT_FUNCTION(pthread_getaffinity_np);
+#else
+#define INIT_PTHREAD_GETAFFINITY_NP
+#endif
+
#if SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPSHARED
INTERCEPTOR_PTHREAD_MUTEXATTR_GET(pshared, sizeof(int))
#define INIT_PTHREAD_MUTEXATTR_GETPSHARED \
// FIXME: under ASan the call below may write to freed memory and corrupt
// its metadata. See
// https://github.com/google/sanitizers/issues/321.
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, s, REAL(strlen)(s) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, s, internal_strlen(s) + 1);
else
- COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);
+ COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1);
}
return res;
}
// its metadata. See
// https://github.com/google/sanitizers/issues/321.
char *res = REAL(tmpnam_r)(s);
- if (res && s) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, s, REAL(strlen)(s) + 1);
+ if (res && s) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, s, internal_strlen(s) + 1);
return res;
}
#define INIT_TMPNAM_R COMMON_INTERCEPT_FUNCTION(tmpnam_r);
COMMON_INTERCEPTOR_ENTER(ctx, ptsname, fd);
char *res = REAL(ptsname)(fd);
if (res != nullptr)
- COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);
+ COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1);
return res;
}
#define INIT_PTSNAME COMMON_INTERCEPT_FUNCTION(ptsname);
COMMON_INTERCEPTOR_ENTER(ctx, ptsname_r, fd, name, namesize);
int res = REAL(ptsname_r)(fd, name, namesize);
if (res == 0)
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, REAL(strlen)(name) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, internal_strlen(name) + 1);
return res;
}
#define INIT_PTSNAME_R COMMON_INTERCEPT_FUNCTION(ptsname_r);
COMMON_INTERCEPTOR_ENTER(ctx, ttyname, fd);
char *res = REAL(ttyname)(fd);
if (res != nullptr)
- COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);
+ COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1);
return res;
}
#define INIT_TTYNAME COMMON_INTERCEPT_FUNCTION(ttyname);
COMMON_INTERCEPTOR_ENTER(ctx, ttyname_r, fd, name, namesize);
int res = REAL(ttyname_r)(fd, name, namesize);
if (res == 0)
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, REAL(strlen)(name) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, internal_strlen(name) + 1);
return res;
}
#define INIT_TTYNAME_R COMMON_INTERCEPT_FUNCTION(ttyname_r);
INTERCEPTOR(char *, tempnam, char *dir, char *pfx) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, tempnam, dir, pfx);
- if (dir) COMMON_INTERCEPTOR_READ_RANGE(ctx, dir, REAL(strlen)(dir) + 1);
- if (pfx) COMMON_INTERCEPTOR_READ_RANGE(ctx, pfx, REAL(strlen)(pfx) + 1);
+ if (dir) COMMON_INTERCEPTOR_READ_RANGE(ctx, dir, internal_strlen(dir) + 1);
+ if (pfx) COMMON_INTERCEPTOR_READ_RANGE(ctx, pfx, internal_strlen(pfx) + 1);
char *res = REAL(tempnam)(dir, pfx);
- if (res) COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);
+ if (res) COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1);
return res;
}
#define INIT_TEMPNAM COMMON_INTERCEPT_FUNCTION(tempnam);
INTERCEPTOR(SSIZE_T, listxattr, const char *path, char *list, SIZE_T size) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, listxattr, path, list, size);
- if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+ if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
// FIXME: under ASan the call below may write to freed memory and corrupt
// its metadata. See
// https://github.com/google/sanitizers/issues/321.
INTERCEPTOR(SSIZE_T, llistxattr, const char *path, char *list, SIZE_T size) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, llistxattr, path, list, size);
- if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+ if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
// FIXME: under ASan the call below may write to freed memory and corrupt
// its metadata. See
// https://github.com/google/sanitizers/issues/321.
SIZE_T size) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, getxattr, path, name, value, size);
- if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
- if (name) COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
+ if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
+ if (name) COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
// FIXME: under ASan the call below may write to freed memory and corrupt
// its metadata. See
// https://github.com/google/sanitizers/issues/321.
SIZE_T size) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, lgetxattr, path, name, value, size);
- if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
- if (name) COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
+ if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
+ if (name) COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
// FIXME: under ASan the call below may write to freed memory and corrupt
// its metadata. See
// https://github.com/google/sanitizers/issues/321.
SIZE_T size) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, fgetxattr, fd, name, value, size);
- if (name) COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
+ if (name) COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
// FIXME: under ASan the call below may write to freed memory and corrupt
// its metadata. See
// https://github.com/google/sanitizers/issues/321.
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, sizeof(__sanitizer_ifaddrs));
if (p->ifa_name)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->ifa_name,
- REAL(strlen)(p->ifa_name) + 1);
+ internal_strlen(p->ifa_name) + 1);
if (p->ifa_addr)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->ifa_addr, struct_sockaddr_sz);
if (p->ifa_netmask)
// https://github.com/google/sanitizers/issues/321.
char *res = REAL(if_indextoname)(ifindex, ifname);
if (res && ifname)
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ifname, REAL(strlen)(ifname) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ifname, internal_strlen(ifname) + 1);
return res;
}
INTERCEPTOR(unsigned int, if_nametoindex, const char* ifname) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, if_nametoindex, ifname);
if (ifname)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, ifname, REAL(strlen)(ifname) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, ifname, internal_strlen(ifname) + 1);
return REAL(if_nametoindex)(ifname);
}
#define INIT_IF_INDEXTONAME \
COMMON_INTERCEPTOR_ENTER(ctx, xdr_string, xdrs, p, maxsize);
if (p && xdrs->x_op == __sanitizer_XDR_ENCODE) {
COMMON_INTERCEPTOR_READ_RANGE(ctx, p, sizeof(*p));
- COMMON_INTERCEPTOR_READ_RANGE(ctx, *p, REAL(strlen)(*p) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, *p, internal_strlen(*p) + 1);
}
// FIXME: under ASan the call below may write to freed memory and corrupt
// its metadata. See
if (p && xdrs->x_op == __sanitizer_XDR_DECODE) {
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, sizeof(*p));
if (res && *p)
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *p, REAL(strlen)(*p) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *p, internal_strlen(*p) + 1);
}
return res;
}
INTERCEPTOR(__sanitizer_FILE *, fopen, const char *path, const char *mode) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, fopen, path, mode);
- if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
- COMMON_INTERCEPTOR_READ_RANGE(ctx, mode, REAL(strlen)(mode) + 1);
+ if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, mode, internal_strlen(mode) + 1);
__sanitizer_FILE *res = REAL(fopen)(path, mode);
COMMON_INTERCEPTOR_FILE_OPEN(ctx, res, path);
if (res) unpoison_file(res);
INTERCEPTOR(__sanitizer_FILE *, fdopen, int fd, const char *mode) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, fdopen, fd, mode);
- COMMON_INTERCEPTOR_READ_RANGE(ctx, mode, REAL(strlen)(mode) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, mode, internal_strlen(mode) + 1);
__sanitizer_FILE *res = REAL(fdopen)(fd, mode);
if (res) unpoison_file(res);
return res;
__sanitizer_FILE *fp) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, freopen, path, mode, fp);
- if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
- COMMON_INTERCEPTOR_READ_RANGE(ctx, mode, REAL(strlen)(mode) + 1);
+ if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, mode, internal_strlen(mode) + 1);
COMMON_INTERCEPTOR_FILE_CLOSE(ctx, fp);
__sanitizer_FILE *res = REAL(freopen)(path, mode, fp);
COMMON_INTERCEPTOR_FILE_OPEN(ctx, res, path);
va_end(ap);
COMMON_INTERCEPTOR_ENTER(ctx, flopen, path, flags, mode);
if (path) {
- COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
}
return REAL(flopen)(path, flags, mode);
}
va_end(ap);
COMMON_INTERCEPTOR_ENTER(ctx, flopen, path, flags, mode);
if (path) {
- COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
}
return REAL(flopenat)(dirfd, path, flags, mode);
}
INTERCEPTOR(__sanitizer_FILE *, fopen64, const char *path, const char *mode) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, fopen64, path, mode);
- COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
- COMMON_INTERCEPTOR_READ_RANGE(ctx, mode, REAL(strlen)(mode) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, mode, internal_strlen(mode) + 1);
__sanitizer_FILE *res = REAL(fopen64)(path, mode);
COMMON_INTERCEPTOR_FILE_OPEN(ctx, res, path);
if (res) unpoison_file(res);
__sanitizer_FILE *fp) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, freopen64, path, mode, fp);
- if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
- COMMON_INTERCEPTOR_READ_RANGE(ctx, mode, REAL(strlen)(mode) + 1);
+ if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, mode, internal_strlen(mode) + 1);
COMMON_INTERCEPTOR_FILE_CLOSE(ctx, fp);
__sanitizer_FILE *res = REAL(freopen64)(path, mode, fp);
COMMON_INTERCEPTOR_FILE_OPEN(ctx, res, path);
void *ctx;
COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, dlopen, filename, flag);
if (filename) COMMON_INTERCEPTOR_READ_STRING(ctx, filename, 0);
- COMMON_INTERCEPTOR_ON_DLOPEN(filename, flag);
- void *res = REAL(dlopen)(filename, flag);
+ void *res = COMMON_INTERCEPTOR_DLOPEN(filename, flag);
Symbolizer::GetOrInit()->InvalidateModuleList();
COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, res);
return res;
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, getpass, prompt);
if (prompt)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, prompt, REAL(strlen)(prompt)+1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, prompt, internal_strlen(prompt)+1);
char *res = REAL(getpass)(prompt);
- if (res) COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res)+1);
+ if (res) COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res)+1);
return res;
}
COMMON_INTERCEPTOR_ENTER(ctx, sem_init, s, pshared, value);
// Workaround a bug in glibc's "old" semaphore implementation by
// zero-initializing the sem_t contents. This has to be done here because
- // interceptors bind to the lowest symbols version by default, hitting the
+ // interceptors bind to the lowest version before glibc 2.36, hitting the
// buggy code path while the non-sanitized build of the same code works fine.
REAL(memset)(s, 0, sizeof(*s));
int res = REAL(sem_init)(s, pshared, value);
}
return res;
}
-#define INIT_SEM \
- COMMON_INTERCEPT_FUNCTION(sem_init); \
- COMMON_INTERCEPT_FUNCTION(sem_destroy); \
- COMMON_INTERCEPT_FUNCTION(sem_wait); \
- COMMON_INTERCEPT_FUNCTION(sem_trywait); \
- COMMON_INTERCEPT_FUNCTION(sem_timedwait); \
- COMMON_INTERCEPT_FUNCTION(sem_post); \
- COMMON_INTERCEPT_FUNCTION(sem_getvalue);
+
+INTERCEPTOR(__sanitizer_sem_t *, sem_open, const char *name, int oflag, ...) {
+ void *ctx;
+ va_list ap;
+ va_start(ap, oflag);
+ u32 mode = va_arg(ap, u32);
+ u32 value = va_arg(ap, u32);
+ COMMON_INTERCEPTOR_ENTER(ctx, sem_open, name, oflag, mode, value);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
+ __sanitizer_sem_t *s = REAL(sem_open)(name, oflag, mode, value);
+ if (s)
+ COMMON_INTERCEPTOR_INITIALIZE_RANGE(s, sizeof(*s));
+ va_end(ap);
+ return s;
+}
+
+INTERCEPTOR(int, sem_unlink, const char *name) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, sem_unlink, name);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
+ return REAL(sem_unlink)(name);
+}
+
+# define INIT_SEM \
+ COMMON_INTERCEPT_FUNCTION(sem_init); \
+ COMMON_INTERCEPT_FUNCTION(sem_destroy); \
+ COMMON_INTERCEPT_FUNCTION(sem_wait); \
+ COMMON_INTERCEPT_FUNCTION(sem_trywait); \
+ COMMON_INTERCEPT_FUNCTION(sem_timedwait); \
+ COMMON_INTERCEPT_FUNCTION(sem_post); \
+ COMMON_INTERCEPT_FUNCTION(sem_getvalue); \
+ COMMON_INTERCEPT_FUNCTION(sem_open); \
+ COMMON_INTERCEPT_FUNCTION(sem_unlink);
#else
-#define INIT_SEM
-#endif // SANITIZER_INTERCEPT_SEM
+# define INIT_SEM
+#endif // SANITIZER_INTERCEPT_SEM
#if SANITIZER_INTERCEPT_PTHREAD_SETCANCEL
INTERCEPTOR(int, pthread_setcancelstate, int state, int *oldstate) {
COMMON_INTERCEPTOR_ENTER(ctx, ctermid, s);
char *res = REAL(ctermid)(s);
if (res) {
- COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);
+ COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1);
}
return res;
}
COMMON_INTERCEPTOR_ENTER(ctx, ctermid_r, s);
char *res = REAL(ctermid_r)(s);
if (res) {
- COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);
+ COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1);
}
return res;
}
#define INIT_STAT
#endif
+#if SANITIZER_INTERCEPT_STAT64
+INTERCEPTOR(int, stat64, const char *path, void *buf) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, stat64, path, buf);
+ if (common_flags()->intercept_stat)
+ COMMON_INTERCEPTOR_READ_STRING(ctx, path, 0);
+ int res = REAL(stat64)(path, buf);
+ if (!res)
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, __sanitizer::struct_stat64_sz);
+ return res;
+}
+#define INIT_STAT64 COMMON_INTERCEPT_FUNCTION(stat64)
+#else
+#define INIT_STAT64
+#endif
+
+
#if SANITIZER_INTERCEPT_LSTAT
INTERCEPTOR(int, lstat, const char *path, void *buf) {
void *ctx;
#define INIT_LSTAT
#endif
+#if SANITIZER_INTERCEPT_STAT64
+INTERCEPTOR(int, lstat64, const char *path, void *buf) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, lstat64, path, buf);
+ if (common_flags()->intercept_stat)
+ COMMON_INTERCEPTOR_READ_STRING(ctx, path, 0);
+ int res = REAL(lstat64)(path, buf);
+ if (!res)
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, __sanitizer::struct_stat64_sz);
+ return res;
+}
+#define INIT_LSTAT64 COMMON_INTERCEPT_FUNCTION(lstat64)
+#else
+#define INIT_LSTAT64
+#endif
+
#if SANITIZER_INTERCEPT___XSTAT
INTERCEPTOR(int, __xstat, int version, const char *path, void *buf) {
void *ctx;
INTERCEPTOR(wchar_t *, wcscat, wchar_t *dst, const wchar_t *src) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, wcscat, dst, src);
- SIZE_T src_size = REAL(wcslen)(src);
- SIZE_T dst_size = REAL(wcslen)(dst);
+ SIZE_T src_size = internal_wcslen(src);
+ SIZE_T dst_size = internal_wcslen(dst);
COMMON_INTERCEPTOR_READ_RANGE(ctx, src, (src_size + 1) * sizeof(wchar_t));
COMMON_INTERCEPTOR_READ_RANGE(ctx, dst, (dst_size + 1) * sizeof(wchar_t));
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst + dst_size,
INTERCEPTOR(wchar_t *, wcsncat, wchar_t *dst, const wchar_t *src, SIZE_T n) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, wcsncat, dst, src, n);
- SIZE_T src_size = REAL(wcsnlen)(src, n);
- SIZE_T dst_size = REAL(wcslen)(dst);
+ SIZE_T src_size = internal_wcsnlen(src, n);
+ SIZE_T dst_size = internal_wcslen(dst);
COMMON_INTERCEPTOR_READ_RANGE(ctx, src,
Min(src_size + 1, n) * sizeof(wchar_t));
COMMON_INTERCEPTOR_READ_RANGE(ctx, dst, (dst_size + 1) * sizeof(wchar_t));
INTERCEPTOR(wchar_t *, wcsdup, wchar_t *s) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, wcsdup, s);
- SIZE_T len = REAL(wcslen)(s);
+ SIZE_T len = internal_wcslen(s);
COMMON_INTERCEPTOR_READ_RANGE(ctx, s, sizeof(wchar_t) * (len + 1));
wchar_t *result = REAL(wcsdup)(s);
if (result)
#endif
#if SANITIZER_INTERCEPT_STRXFRM
-static SIZE_T RealStrLen(const char *str) { return REAL(strlen)(str); }
+static SIZE_T RealStrLen(const char *str) { return internal_strlen(str); }
-static SIZE_T RealStrLen(const wchar_t *str) { return REAL(wcslen)(str); }
+static SIZE_T RealStrLen(const wchar_t *str) { return internal_wcslen(str); }
#define STRXFRM_INTERCEPTOR_IMPL(strxfrm, dest, src, len, ...) \
{ \
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, acct, file);
if (file)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, file, REAL(strlen)(file) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, file, internal_strlen(file) + 1);
return REAL(acct)(file);
}
#define INIT_ACCT COMMON_INTERCEPT_FUNCTION(acct)
COMMON_INTERCEPTOR_ENTER(ctx, user_from_uid, uid, nouser);
user = REAL(user_from_uid)(uid, nouser);
if (user)
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, user, REAL(strlen)(user) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, user, internal_strlen(user) + 1);
return user;
}
#define INIT_USER_FROM_UID COMMON_INTERCEPT_FUNCTION(user_from_uid)
int res;
COMMON_INTERCEPTOR_ENTER(ctx, uid_from_user, name, uid);
if (name)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
res = REAL(uid_from_user)(name, uid);
if (uid)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, uid, sizeof(*uid));
COMMON_INTERCEPTOR_ENTER(ctx, group_from_gid, gid, nogroup);
group = REAL(group_from_gid)(gid, nogroup);
if (group)
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, group, REAL(strlen)(group) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, group, internal_strlen(group) + 1);
return group;
}
#define INIT_GROUP_FROM_GID COMMON_INTERCEPT_FUNCTION(group_from_gid)
int res;
COMMON_INTERCEPTOR_ENTER(ctx, gid_from_group, group, gid);
if (group)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, group, REAL(strlen)(group) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, group, internal_strlen(group) + 1);
res = REAL(gid_from_group)(group, gid);
if (gid)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, gid, sizeof(*gid));
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, access, path, mode);
if (path)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
return REAL(access)(path, mode);
}
#define INIT_ACCESS COMMON_INTERCEPT_FUNCTION(access)
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, faccessat, fd, path, mode, flags);
if (path)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
return REAL(faccessat)(fd, path, mode, flags);
}
#define INIT_FACCESSAT COMMON_INTERCEPT_FUNCTION(faccessat)
int res;
COMMON_INTERCEPTOR_ENTER(ctx, getgrouplist, name, basegid, groups, ngroups);
if (name)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
if (ngroups)
COMMON_INTERCEPTOR_READ_RANGE(ctx, ngroups, sizeof(*ngroups));
res = REAL(getgrouplist)(name, basegid, groups, ngroups);
COMMON_INTERCEPTOR_ENTER(ctx, getgroupmembership, name, basegid, groups,
maxgrp, ngroups);
if (name)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
res = REAL(getgroupmembership)(name, basegid, groups, maxgrp, ngroups);
if (!res && groups && ngroups) {
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, groups, sizeof(*groups) * (*ngroups));
INTERCEPTOR(SSIZE_T, readlink, const char *path, char *buf, SIZE_T bufsiz) {
void* ctx;
COMMON_INTERCEPTOR_ENTER(ctx, readlink, path, buf, bufsiz);
- COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
SSIZE_T res = REAL(readlink)(path, buf, bufsiz);
if (res > 0)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, res);
SIZE_T bufsiz) {
void* ctx;
COMMON_INTERCEPTOR_ENTER(ctx, readlinkat, dirfd, path, buf, bufsiz);
- COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
SSIZE_T res = REAL(readlinkat)(dirfd, path, buf, bufsiz);
if (res > 0)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, res);
void* ctx;
COMMON_INTERCEPTOR_ENTER(ctx, name_to_handle_at, dirfd, pathname, handle,
mount_id, flags);
- COMMON_INTERCEPTOR_READ_RANGE(ctx, pathname, REAL(strlen)(pathname) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, pathname, internal_strlen(pathname) + 1);
__sanitizer_file_handle *sanitizer_handle =
reinterpret_cast<__sanitizer_file_handle*>(handle);
ctx, src, Min(internal_strnlen(src, size), size - 1) + 1);
}
res = REAL(strlcpy)(dst, src, size);
- COMMON_INTERCEPTOR_COPY_STRING(ctx, dst, src, REAL(strlen)(dst) + 1);
+ COMMON_INTERCEPTOR_COPY_STRING(ctx, dst, src, internal_strlen(dst) + 1);
return res;
}
OFF_T off) {
void *ctx;
if (common_flags()->detect_write_exec)
- ReportMmapWriteExec(prot);
+ ReportMmapWriteExec(prot, flags);
if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
return (void *)internal_mmap(addr, sz, prot, flags, fd, off);
COMMON_INTERCEPTOR_ENTER(ctx, mmap, addr, sz, prot, flags, fd, off);
INTERCEPTOR(int, mprotect, void *addr, SIZE_T sz, int prot) {
void *ctx;
if (common_flags()->detect_write_exec)
- ReportMmapWriteExec(prot);
+ ReportMmapWriteExec(prot, 0);
if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
return (int)internal_mprotect(addr, sz, prot);
COMMON_INTERCEPTOR_ENTER(ctx, mprotect, addr, sz, prot);
OFF64_T off) {
void *ctx;
if (common_flags()->detect_write_exec)
- ReportMmapWriteExec(prot);
+ ReportMmapWriteExec(prot, flags);
if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
return (void *)internal_mmap(addr, sz, prot, flags, fd, off);
COMMON_INTERCEPTOR_ENTER(ctx, mmap64, addr, sz, prot, flags, fd, off);
COMMON_INTERCEPTOR_ENTER(ctx, devname, dev, type);
name = REAL(devname)(dev, type);
if (name)
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, REAL(strlen)(name) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, internal_strlen(name) + 1);
return name;
}
#define INIT_DEVNAME COMMON_INTERCEPT_FUNCTION(devname);
COMMON_INTERCEPTOR_ENTER(ctx, devname_r, dev, type, path, len);
DEVNAME_R_RETTYPE res = REAL(devname_r)(dev, type, path, len);
if (DEVNAME_R_SUCCESS(res))
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, path, REAL(strlen)(path) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, path, internal_strlen(path) + 1);
return res;
}
#define INIT_DEVNAME_R COMMON_INTERCEPT_FUNCTION(devname_r);
COMMON_INTERCEPTOR_ENTER(ctx, strmode, mode, bp);
REAL(strmode)(mode, bp);
if (bp)
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, bp, REAL(strlen)(bp) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, bp, internal_strlen(bp) + 1);
}
#define INIT_STRMODE COMMON_INTERCEPT_FUNCTION(strmode)
#else
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, getttynam, name);
if (name)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
struct __sanitizer_ttyent *ttyent = REAL(getttynam)(name);
if (ttyent)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ttyent, struct_ttyent_sz);
return ttyent;
}
+#define INIT_TTYENT \
+ COMMON_INTERCEPT_FUNCTION(getttyent); \
+ COMMON_INTERCEPT_FUNCTION(getttynam);
+#else
+#define INIT_TTYENT
+#endif
+
+#if SANITIZER_INTERCEPT_TTYENTPATH
INTERCEPTOR(int, setttyentpath, char *path) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, setttyentpath, path);
if (path)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
return REAL(setttyentpath)(path);
}
-#define INIT_TTYENT \
- COMMON_INTERCEPT_FUNCTION(getttyent); \
- COMMON_INTERCEPT_FUNCTION(getttynam); \
- COMMON_INTERCEPT_FUNCTION(setttyentpath)
+#define INIT_TTYENTPATH COMMON_INTERCEPT_FUNCTION(setttyentpath);
#else
-#define INIT_TTYENT
+#define INIT_TTYENTPATH
#endif
#if SANITIZER_INTERCEPT_PROTOENT
static void write_protoent(void *ctx, struct __sanitizer_protoent *p) {
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, sizeof(*p));
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->p_name, REAL(strlen)(p->p_name) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->p_name, internal_strlen(p->p_name) + 1);
SIZE_T pp_size = 1; // One handles the trailing \0
for (char **pp = p->p_aliases; *pp; ++pp, ++pp_size)
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *pp, REAL(strlen)(*pp) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *pp, internal_strlen(*pp) + 1);
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->p_aliases,
pp_size * sizeof(char **));
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, getprotobyname, name);
if (name)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
struct __sanitizer_protoent *p = REAL(getprotobyname)(name);
if (p)
write_protoent(ctx, p);
COMMON_INTERCEPTOR_ENTER(ctx, getprotobyname_r, name, result_buf, buf,
buflen, result);
if (name)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
int res = REAL(getprotobyname_r)(name, result_buf, buf, buflen, result);
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof *result);
if (n) {
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n, sizeof(*n));
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_name, REAL(strlen)(n->n_name) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_name, internal_strlen(n->n_name) + 1);
SIZE_T nn_size = 1; // One handles the trailing \0
for (char **nn = n->n_aliases; *nn; ++nn, ++nn_size)
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *nn, REAL(strlen)(*nn) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *nn, internal_strlen(*nn) + 1);
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_aliases,
nn_size * sizeof(char **));
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, getnetbyname, name);
if (name)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
struct __sanitizer_netent *n = REAL(getnetbyname)(name);
if (n) {
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n, sizeof(*n));
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_name, REAL(strlen)(n->n_name) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_name, internal_strlen(n->n_name) + 1);
SIZE_T nn_size = 1; // One handles the trailing \0
for (char **nn = n->n_aliases; *nn; ++nn, ++nn_size)
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *nn, REAL(strlen)(*nn) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *nn, internal_strlen(*nn) + 1);
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_aliases,
nn_size * sizeof(char **));
if (n) {
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n, sizeof(*n));
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_name, REAL(strlen)(n->n_name) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_name, internal_strlen(n->n_name) + 1);
SIZE_T nn_size = 1; // One handles the trailing \0
for (char **nn = n->n_aliases; *nn; ++nn, ++nn_size)
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *nn, REAL(strlen)(*nn) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *nn, internal_strlen(*nn) + 1);
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_aliases,
nn_size * sizeof(char **));
unpoison_file(stream);
}
-INTERCEPTOR(void, setbuffer, __sanitizer_FILE *stream, char *buf, int mode) {
+INTERCEPTOR(void, setbuffer, __sanitizer_FILE *stream, char *buf, SIZE_T size) {
void *ctx;
- COMMON_INTERCEPTOR_ENTER(ctx, setbuffer, stream, buf, mode);
- REAL(setbuffer)(stream, buf, mode);
+ COMMON_INTERCEPTOR_ENTER(ctx, setbuffer, stream, buf, size);
+ REAL(setbuffer)(stream, buf, size);
if (buf) {
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, __sanitizer_bufsiz);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, size);
}
if (stream)
unpoison_file(stream);
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, regcomp, preg, pattern, cflags);
if (pattern)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, pattern, REAL(strlen)(pattern) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, pattern, internal_strlen(pattern) + 1);
int res = REAL(regcomp)(preg, pattern, cflags);
- if (!res)
+ if (preg)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, preg, struct_regex_sz);
return res;
}
if (preg)
COMMON_INTERCEPTOR_READ_RANGE(ctx, preg, struct_regex_sz);
if (string)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, string, REAL(strlen)(string) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, string, internal_strlen(string) + 1);
int res = REAL(regexec)(preg, string, nmatch, pmatch, eflags);
if (!res && pmatch)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pmatch, nmatch * struct_regmatch_sz);
COMMON_INTERCEPTOR_READ_RANGE(ctx, preg, struct_regex_sz);
SIZE_T res = REAL(regerror)(errcode, preg, errbuf, errbuf_size);
if (errbuf)
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, errbuf, REAL(strlen)(errbuf) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, errbuf, internal_strlen(errbuf) + 1);
return res;
}
INTERCEPTOR(void, regfree, const void *preg) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, regnsub, buf, bufsiz, sub, rm, str);
if (sub)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, sub, REAL(strlen)(sub) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, sub, internal_strlen(sub) + 1);
// The implementation demands and hardcodes 10 elements
if (rm)
COMMON_INTERCEPTOR_READ_RANGE(ctx, rm, 10 * struct_regmatch_sz);
if (str)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, str, REAL(strlen)(str) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, str, internal_strlen(str) + 1);
SSIZE_T res = REAL(regnsub)(buf, bufsiz, sub, rm, str);
if (res > 0 && buf)
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, REAL(strlen)(buf) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, internal_strlen(buf) + 1);
return res;
}
INTERCEPTOR(SSIZE_T, regasub, char **buf, const char *sub,
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, regasub, buf, sub, rm, sstr);
if (sub)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, sub, REAL(strlen)(sub) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, sub, internal_strlen(sub) + 1);
// Hardcode 10 elements as this is hardcoded size
if (rm)
COMMON_INTERCEPTOR_READ_RANGE(ctx, rm, 10 * struct_regmatch_sz);
if (sstr)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, sstr, REAL(strlen)(sstr) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, sstr, internal_strlen(sstr) + 1);
SSIZE_T res = REAL(regasub)(buf, sub, rm, sstr);
if (res > 0 && buf) {
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, sizeof(char *));
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *buf, REAL(strlen)(*buf) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *buf, internal_strlen(*buf) + 1);
}
return res;
}
COMMON_INTERCEPTOR_READ_RANGE(ctx, pa, sizeof(char **));
if (!*pa)
break;
- COMMON_INTERCEPTOR_READ_RANGE(ctx, *pa, REAL(strlen)(*pa) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, *pa, internal_strlen(*pa) + 1);
}
}
// TODO(kamil): handle compar callback
COMMON_INTERCEPTOR_ENTER(ctx, sysctlbyname, sname, oldp, oldlenp, newp,
newlen);
if (sname)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, sname, REAL(strlen)(sname) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, sname, internal_strlen(sname) + 1);
if (oldlenp)
COMMON_INTERCEPTOR_READ_RANGE(ctx, oldlenp, sizeof(*oldlenp));
if (newp && newlen)
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, sysctlnametomib, sname, name, namelenp);
if (sname)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, sname, REAL(strlen)(sname) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, sname, internal_strlen(sname) + 1);
if (namelenp)
COMMON_INTERCEPTOR_READ_RANGE(ctx, namelenp, sizeof(*namelenp));
int res = REAL(sysctlnametomib)(sname, name, namelenp);
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, asysctlbyname, sname, len);
if (sname)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, sname, REAL(strlen)(sname) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, sname, internal_strlen(sname) + 1);
void *res = REAL(asysctlbyname)(sname, len);
if (res && len) {
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, len, sizeof(*len));
COMMON_INTERCEPTOR_ENTER(ctx, sysctlgetmibinfo, sname, name, namelenp, cname,
csz, rnode, v);
if (sname)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, sname, REAL(strlen)(sname) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, sname, internal_strlen(sname) + 1);
if (namelenp)
COMMON_INTERCEPTOR_READ_RANGE(ctx, namelenp, sizeof(*namelenp));
if (csz)
COMMON_INTERCEPTOR_ENTER(ctx, nl_langinfo, item);
char *ret = REAL(nl_langinfo)(item);
if (ret)
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, REAL(strlen)(ret) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, internal_strlen(ret) + 1);
return ret;
}
#define INIT_NL_LANGINFO COMMON_INTERCEPT_FUNCTION(nl_langinfo)
COMMON_INTERCEPTOR_READ_RANGE(ctx, ml, sizeof(*ml));
if (ml->ml_filename)
COMMON_INTERCEPTOR_READ_RANGE(ctx, ml->ml_filename,
- REAL(strlen)(ml->ml_filename) + 1);
+ internal_strlen(ml->ml_filename) + 1);
if (ml->ml_props)
COMMON_INTERCEPTOR_READ_RANGE(ctx, ml->ml_props, ml->ml_propslen);
}
} else if (operation == modctl_unload) {
if (argp) {
const char *name = (const char *)argp;
- COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
}
ret = REAL(modctl)(operation, argp);
} else if (operation == modctl_stat) {
if (errstr) {
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, errstr, sizeof(const char *));
if (*errstr)
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *errstr, REAL(strlen)(*errstr) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *errstr, internal_strlen(*errstr) + 1);
}
return ret;
}
COMMON_INTERCEPTOR_READ_RANGE(ctx, delim, sizeof(delim[0]) * 3);
char *ret = REAL(fparseln)(stream, len, lineno, delim, flags);
if (ret) {
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, REAL(strlen)(ret) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, internal_strlen(ret) + 1);
if (len)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, len, sizeof(*len));
if (lineno)
INTERCEPTOR(int, statvfs1, const char *path, void *buf, int flags) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, statvfs1, path, buf, flags);
- if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+ if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
int res = REAL(statvfs1)(path, buf, flags);
if (!res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statvfs_sz);
return res;
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, SHA1File, filename, buf);
if (filename)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, internal_strlen(filename) + 1);
char *ret = REAL(SHA1File)(filename, buf);
if (ret)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA1_return_length);
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, SHA1FileChunk, filename, buf, offset, length);
if (filename)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, internal_strlen(filename) + 1);
char *ret = REAL(SHA1FileChunk)(filename, buf, offset, length);
if (ret)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA1_return_length);
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, MD4File, filename, buf);
if (filename)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, internal_strlen(filename) + 1);
char *ret = REAL(MD4File)(filename, buf);
if (ret)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD4_return_length);
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, RMD160File, filename, buf);
if (filename)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, internal_strlen(filename) + 1);
char *ret = REAL(RMD160File)(filename, buf);
if (ret)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, RMD160_return_length);
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, RMD160FileChunk, filename, buf, offset, length);
if (filename)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, internal_strlen(filename) + 1);
char *ret = REAL(RMD160FileChunk)(filename, buf, offset, length);
if (ret)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, RMD160_return_length);
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, MD5File, filename, buf);
if (filename)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, internal_strlen(filename) + 1);
char *ret = REAL(MD5File)(filename, buf);
if (ret)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD5_return_length);
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, MD2File, filename, buf);
if (filename)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, internal_strlen(filename) + 1);
char *ret = REAL(MD2File)(filename, buf);
if (ret)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD2_return_length);
void *ctx; \
COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_File, filename, buf); \
if (filename) \
- COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1);\
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, internal_strlen(filename) + 1);\
char *ret = REAL(SHA##LEN##_File)(filename, buf); \
if (ret) \
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA##LEN##_return_length); \
COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_FileChunk, filename, buf, offset, \
length); \
if (filename) \
- COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1);\
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, internal_strlen(filename) + 1);\
char *ret = REAL(SHA##LEN##_FileChunk)(filename, buf, offset, length); \
if (ret) \
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA##LEN##_return_length); \
return ret; \
}
-SHA2_INTERCEPTORS(224, u32);
-SHA2_INTERCEPTORS(256, u32);
-SHA2_INTERCEPTORS(384, u64);
-SHA2_INTERCEPTORS(512, u64);
+SHA2_INTERCEPTORS(224, u32)
+SHA2_INTERCEPTORS(256, u32)
+SHA2_INTERCEPTORS(384, u64)
+SHA2_INTERCEPTORS(512, u64)
#define INIT_SHA2_INTECEPTORS(LEN) \
COMMON_INTERCEPT_FUNCTION(SHA##LEN##_Init); \
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, strvis, dst, src, flag);
if (src)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, src, internal_strlen(src) + 1);
int len = REAL(strvis)(dst, src, flag);
if (dst)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, len + 1);
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, stravis, dst, src, flag);
if (src)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, src, internal_strlen(src) + 1);
int len = REAL(stravis)(dst, src, flag);
if (dst) {
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, sizeof(char *));
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, strnvis, dst, dlen, src, flag);
if (src)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, src, internal_strlen(src) + 1);
int len = REAL(strnvis)(dst, dlen, src, flag);
// The interface will be valid even if there is no space for NULL char
if (dst && len > 0)
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, svis, dst, c, flag, nextc, extra);
if (extra)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, internal_strlen(extra) + 1);
char *end = REAL(svis)(dst, c, flag, nextc, extra);
if (dst && end)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, end - dst + 1);
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, snvis, dst, dlen, c, flag, nextc, extra);
if (extra)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, internal_strlen(extra) + 1);
char *end = REAL(snvis)(dst, dlen, c, flag, nextc, extra);
if (dst && end)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst,
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, strsvis, dst, src, flag, extra);
if (src)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, src, internal_strlen(src) + 1);
if (extra)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, internal_strlen(extra) + 1);
int len = REAL(strsvis)(dst, src, flag, extra);
if (dst)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, len + 1);
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, strsnvis, dst, dlen, src, flag, extra);
if (src)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, src, internal_strlen(src) + 1);
if (extra)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, internal_strlen(extra) + 1);
int len = REAL(strsnvis)(dst, dlen, src, flag, extra);
// The interface will be valid even if there is no space for NULL char
if (dst && len >= 0)
if (src)
COMMON_INTERCEPTOR_READ_RANGE(ctx, src, len);
if (extra)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, internal_strlen(extra) + 1);
int ret = REAL(strsvisx)(dst, src, len, flag, extra);
if (dst)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1);
if (src)
COMMON_INTERCEPTOR_READ_RANGE(ctx, src, len);
if (extra)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, internal_strlen(extra) + 1);
int ret = REAL(strsnvisx)(dst, dlen, src, len, flag, extra);
if (dst && ret >= 0)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1);
if (src)
COMMON_INTERCEPTOR_READ_RANGE(ctx, src, len);
if (extra)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, internal_strlen(extra) + 1);
// FIXME: only need to be checked when "flag | VIS_NOLOCALE" doesn't hold
// according to the implementation
if (cerr_ptr)
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, strunvis, dst, src);
if (src)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, src, internal_strlen(src) + 1);
int ret = REAL(strunvis)(dst, src);
if (ret != -1)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1);
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, strnunvis, dst, dlen, src);
if (src)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, src, internal_strlen(src) + 1);
int ret = REAL(strnunvis)(dst, dlen, src);
if (ret != -1)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1);
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, strunvisx, dst, src, flag);
if (src)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, src, internal_strlen(src) + 1);
int ret = REAL(strunvisx)(dst, src, flag);
if (ret != -1)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1);
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, strnunvisx, dst, dlen, src, flag);
if (src)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, src, internal_strlen(src) + 1);
int ret = REAL(strnunvisx)(dst, dlen, src, flag);
if (ret != -1)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1);
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, cdbr_open, path, flags);
if (path)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
struct __sanitizer_cdbr *cdbr = REAL(cdbr_open)(path, flags);
if (cdbr)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cdbr, sizeof(*cdbr));
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, getfsspec, spec);
if (spec)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, spec, REAL(strlen)(spec) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, spec, internal_strlen(spec) + 1);
void *ret = REAL(getfsspec)(spec);
if (ret)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, struct_fstab_sz);
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, getfsfile, file);
if (file)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, file, REAL(strlen)(file) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, file, internal_strlen(file) + 1);
void *ret = REAL(getfsfile)(file);
if (ret)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, struct_fstab_sz);
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, popen, command, type);
if (command)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, command, REAL(strlen)(command) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, command, internal_strlen(command) + 1);
if (type)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, type, REAL(strlen)(type) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, type, internal_strlen(type) + 1);
__sanitizer_FILE *res = REAL(popen)(command, type);
COMMON_INTERCEPTOR_FILE_OPEN(ctx, res, nullptr);
if (res) unpoison_file(res);
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, popenve, path, argv, envp, type);
if (path)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
if (argv) {
for (char *const *pa = argv; ; ++pa) {
COMMON_INTERCEPTOR_READ_RANGE(ctx, pa, sizeof(char **));
if (!*pa)
break;
- COMMON_INTERCEPTOR_READ_RANGE(ctx, *pa, REAL(strlen)(*pa) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, *pa, internal_strlen(*pa) + 1);
}
}
if (envp) {
COMMON_INTERCEPTOR_READ_RANGE(ctx, pa, sizeof(char **));
if (!*pa)
break;
- COMMON_INTERCEPTOR_READ_RANGE(ctx, *pa, REAL(strlen)(*pa) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, *pa, internal_strlen(*pa) + 1);
}
}
if (type)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, type, REAL(strlen)(type) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, type, internal_strlen(type) + 1);
__sanitizer_FILE *res = REAL(popenve)(path, argv, envp, type);
COMMON_INTERCEPTOR_FILE_OPEN(ctx, res, nullptr);
if (res) unpoison_file(res);
COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
char *name = REAL(fdevname)(fd);
if (name) {
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, REAL(strlen)(name) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, internal_strlen(name) + 1);
if (fd > 0)
COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
}
COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
char *name = REAL(fdevname_r)(fd, buf, len);
if (name && buf && len > 0) {
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, REAL(strlen)(buf) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, internal_strlen(buf) + 1);
if (fd > 0)
COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
}
COMMON_INTERCEPTOR_ENTER(ctx, getusershell);
char *res = REAL(getusershell)();
if (res)
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
return res;
}
if (sl)
COMMON_INTERCEPTOR_READ_RANGE(ctx, sl, __sanitizer::struct_StringList_sz);
if (item)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, item, REAL(strlen)(item) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, item, internal_strlen(item) + 1);
int res = REAL(sl_add)(sl, item);
if (!res)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, sl, __sanitizer::struct_StringList_sz);
if (sl)
COMMON_INTERCEPTOR_READ_RANGE(ctx, sl, __sanitizer::struct_StringList_sz);
if (item)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, item, REAL(strlen)(item) + 1);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, item, internal_strlen(item) + 1);
char *res = REAL(sl_find)(sl, item);
if (res)
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
return res;
}
#define INIT_GETENTROPY
#endif
-#if SANITIZER_INTERCEPT_QSORT
+#if SANITIZER_INTERCEPT_QSORT_R
+typedef int (*qsort_r_compar_f)(const void *, const void *, void *);
+struct qsort_r_compar_params {
+ SIZE_T size;
+ qsort_r_compar_f compar;
+ void *arg;
+};
+static int wrapped_qsort_r_compar(const void *a, const void *b, void *arg) {
+ qsort_r_compar_params *params = (qsort_r_compar_params *)arg;
+ COMMON_INTERCEPTOR_UNPOISON_PARAM(3);
+ COMMON_INTERCEPTOR_INITIALIZE_RANGE(a, params->size);
+ COMMON_INTERCEPTOR_INITIALIZE_RANGE(b, params->size);
+ return params->compar(a, b, params->arg);
+}
+
+INTERCEPTOR(void, qsort_r, void *base, SIZE_T nmemb, SIZE_T size,
+ qsort_r_compar_f compar, void *arg) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, qsort_r, base, nmemb, size, compar, arg);
+ // Run the comparator over all array elements to detect any memory issues.
+ if (nmemb > 1) {
+ for (SIZE_T i = 0; i < nmemb - 1; ++i) {
+ void *p = (void *)((char *)base + i * size);
+ void *q = (void *)((char *)base + (i + 1) * size);
+ COMMON_INTERCEPTOR_UNPOISON_PARAM(3);
+ compar(p, q, arg);
+ }
+ }
+ qsort_r_compar_params params = {size, compar, arg};
+ REAL(qsort_r)(base, nmemb, size, wrapped_qsort_r_compar, ¶ms);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, base, nmemb * size);
+}
+# define INIT_QSORT_R COMMON_INTERCEPT_FUNCTION(qsort_r)
+#else
+# define INIT_QSORT_R
+#endif
+
+#if SANITIZER_INTERCEPT_QSORT && SANITIZER_INTERCEPT_QSORT_R
+INTERCEPTOR(void, qsort, void *base, SIZE_T nmemb, SIZE_T size,
+ qsort_r_compar_f compar) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, qsort, base, nmemb, size, compar);
+ WRAP(qsort_r)(base, nmemb, size, compar, nullptr);
+}
+# define INIT_QSORT COMMON_INTERCEPT_FUNCTION(qsort)
+#elif SANITIZER_INTERCEPT_QSORT && !SANITIZER_INTERCEPT_QSORT_R
// Glibc qsort uses a temporary buffer allocated either on stack or on heap.
// Poisoned memory from there may get copied into the comparator arguments,
// where it needs to be dealt with. But even that is not enough - the results of
typedef int (*qsort_compar_f)(const void *, const void *);
static THREADLOCAL qsort_compar_f qsort_compar;
static THREADLOCAL SIZE_T qsort_size;
-int wrapped_qsort_compar(const void *a, const void *b) {
+static int wrapped_qsort_compar(const void *a, const void *b) {
COMMON_INTERCEPTOR_UNPOISON_PARAM(2);
COMMON_INTERCEPTOR_INITIALIZE_RANGE(a, qsort_size);
COMMON_INTERCEPTOR_INITIALIZE_RANGE(b, qsort_size);
}
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, base, nmemb * size);
}
-#define INIT_QSORT COMMON_INTERCEPT_FUNCTION(qsort)
+# define INIT_QSORT COMMON_INTERCEPT_FUNCTION(qsort)
#else
-#define INIT_QSORT
+# define INIT_QSORT
#endif
-#if SANITIZER_INTERCEPT_QSORT_R
-typedef int (*qsort_r_compar_f)(const void *, const void *, void *);
-static THREADLOCAL qsort_r_compar_f qsort_r_compar;
-static THREADLOCAL SIZE_T qsort_r_size;
-int wrapped_qsort_r_compar(const void *a, const void *b, void *arg) {
- COMMON_INTERCEPTOR_UNPOISON_PARAM(3);
- COMMON_INTERCEPTOR_INITIALIZE_RANGE(a, qsort_r_size);
- COMMON_INTERCEPTOR_INITIALIZE_RANGE(b, qsort_r_size);
- return qsort_r_compar(a, b, arg);
+#if SANITIZER_INTERCEPT_BSEARCH
+typedef int (*bsearch_compar_f)(const void *, const void *);
+struct bsearch_compar_params {
+ const void *key;
+ bsearch_compar_f compar;
+};
+
+static int wrapped_bsearch_compar(const void *key, const void *b) {
+ const bsearch_compar_params *params = (const bsearch_compar_params *)key;
+ COMMON_INTERCEPTOR_UNPOISON_PARAM(2);
+ return params->compar(params->key, b);
}
-INTERCEPTOR(void, qsort_r, void *base, SIZE_T nmemb, SIZE_T size,
- qsort_r_compar_f compar, void *arg) {
+INTERCEPTOR(void *, bsearch, const void *key, const void *base, SIZE_T nmemb,
+ SIZE_T size, bsearch_compar_f compar) {
void *ctx;
- COMMON_INTERCEPTOR_ENTER(ctx, qsort_r, base, nmemb, size, compar, arg);
- // Run the comparator over all array elements to detect any memory issues.
- if (nmemb > 1) {
- for (SIZE_T i = 0; i < nmemb - 1; ++i) {
- void *p = (void *)((char *)base + i * size);
- void *q = (void *)((char *)base + (i + 1) * size);
- COMMON_INTERCEPTOR_UNPOISON_PARAM(3);
- compar(p, q, arg);
- }
- }
- qsort_r_compar_f old_compar = qsort_r_compar;
- SIZE_T old_size = qsort_r_size;
- // Handle qsort_r() implementations that recurse using an
- // interposable function call:
- bool already_wrapped = compar == wrapped_qsort_r_compar;
- if (already_wrapped) {
- // This case should only happen if the qsort() implementation calls itself
- // using a preemptible function call (e.g. the FreeBSD libc version).
- // Check that the size and comparator arguments are as expected.
- CHECK_NE(compar, qsort_r_compar);
- CHECK_EQ(qsort_r_size, size);
- } else {
- qsort_r_compar = compar;
- qsort_r_size = size;
- }
- REAL(qsort_r)(base, nmemb, size, wrapped_qsort_r_compar, arg);
- if (!already_wrapped) {
- qsort_r_compar = old_compar;
- qsort_r_size = old_size;
- }
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, base, nmemb * size);
+ COMMON_INTERCEPTOR_ENTER(ctx, bsearch, key, base, nmemb, size, compar);
+ bsearch_compar_params params = {key, compar};
+ return REAL(bsearch)(¶ms, base, nmemb, size, wrapped_bsearch_compar);
}
-#define INIT_QSORT_R COMMON_INTERCEPT_FUNCTION(qsort_r)
+# define INIT_BSEARCH COMMON_INTERCEPT_FUNCTION(bsearch)
#else
-#define INIT_QSORT_R
+# define INIT_BSEARCH
#endif
#if SANITIZER_INTERCEPT_SIGALTSTACK
#define INIT_SIGALTSTACK
#endif
+#if SANITIZER_INTERCEPT_PROCCTL
+INTERCEPTOR(int, procctl, int idtype, u64 id, int cmd, uptr data) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, procctl, idtype, id, cmd, data);
+ static const int PROC_REAP_ACQUIRE = 2;
+ static const int PROC_REAP_RELEASE = 3;
+ static const int PROC_REAP_STATUS = 4;
+ static const int PROC_REAP_GETPIDS = 5;
+ static const int PROC_REAP_KILL = 6;
+ if (cmd < PROC_REAP_ACQUIRE || cmd > PROC_REAP_KILL) {
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, (void *)data, sizeof(int));
+ } else {
+ // reap_acquire/reap_release bears no arguments.
+ if (cmd > PROC_REAP_RELEASE) {
+ unsigned int reapsz;
+ switch (cmd) {
+ case PROC_REAP_STATUS:
+ reapsz = struct_procctl_reaper_status_sz;
+ break;
+ case PROC_REAP_GETPIDS:
+ reapsz = struct_procctl_reaper_pids_sz;
+ break;
+ case PROC_REAP_KILL:
+ reapsz = struct_procctl_reaper_kill_sz;
+ break;
+ }
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, (void *)data, reapsz);
+ }
+ }
+ return REAL(procctl)(idtype, id, cmd, data);
+}
+#define INIT_PROCCTL COMMON_INTERCEPT_FUNCTION(procctl)
+#else
+#define INIT_PROCCTL
+#endif
+
#if SANITIZER_INTERCEPT_UNAME
INTERCEPTOR(int, uname, struct utsname *utsname) {
#if SANITIZER_LINUX
#define INIT___XUNAME
#endif
+#if SANITIZER_INTERCEPT_HEXDUMP
+INTERCEPTOR(void, hexdump, const void *ptr, int length, const char *header, int flags) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, hexdump, ptr, length, header, flags);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, length);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, header, internal_strlen(header) + 1);
+ REAL(hexdump)(ptr, length, header, flags);
+}
+
+#define INIT_HEXDUMP COMMON_INTERCEPT_FUNCTION(hexdump);
+#else
+#define INIT_HEXDUMP
+#endif
+
#include "sanitizer_common_interceptors_netbsd_compat.inc"
static void InitializeCommonInterceptors() {
INIT_TIME;
INIT_GLOB;
INIT_GLOB64;
+ INIT___B64_TO;
+ INIT_DN_COMP_EXPAND;
+ INIT_POSIX_SPAWN;
INIT_WAIT;
INIT_WAIT4;
INIT_INET;
INIT_PTHREAD_SIGMASK;
INIT_BACKTRACE;
INIT__EXIT;
- INIT_PTHREAD_MUTEX_LOCK;
- INIT_PTHREAD_MUTEX_UNLOCK;
- INIT___PTHREAD_MUTEX_LOCK;
- INIT___PTHREAD_MUTEX_UNLOCK;
- INIT___LIBC_MUTEX_LOCK;
- INIT___LIBC_MUTEX_UNLOCK;
INIT___LIBC_THR_SETCANCELSTATE;
INIT_GETMNTENT;
INIT_GETMNTENT_R;
INIT_PTHREAD_ATTR_GET_SCHED;
INIT_PTHREAD_ATTR_GETINHERITSCHED;
INIT_PTHREAD_ATTR_GETAFFINITY_NP;
+ INIT_PTHREAD_GETAFFINITY_NP;
INIT_PTHREAD_MUTEXATTR_GETPSHARED;
INIT_PTHREAD_MUTEXATTR_GETTYPE;
INIT_PTHREAD_MUTEXATTR_GETPROTOCOL;
INIT_RECV_RECVFROM;
INIT_SEND_SENDTO;
INIT_STAT;
+ INIT_STAT64;
INIT_EVENTFD_READ_WRITE;
INIT_LSTAT;
+ INIT_LSTAT64;
INIT___XSTAT;
INIT___XSTAT64;
INIT___LXSTAT;
INIT_GETENTROPY;
INIT_QSORT;
INIT_QSORT_R;
+ INIT_BSEARCH;
INIT_SIGALTSTACK;
+ INIT_PROCCTL
INIT_UNAME;
INIT___XUNAME;
+ INIT_HEXDUMP;
INIT___PRINTF_CHK;
}
continue;
int size = scanf_get_value_size(&dir);
if (size == FSS_INVALID) {
- Report("%s: WARNING: unexpected format specifier in scanf interceptor: ",
- SanitizerToolName, "%.*s\n", dir.end - dir.begin, dir.begin);
+ Report("%s: WARNING: unexpected format specifier in scanf interceptor: %.*s\n",
+ SanitizerToolName, static_cast<int>(dir.end - dir.begin), dir.begin);
break;
}
void *argp = va_arg(aq, void *);
break; \
default: \
Report("WARNING: unexpected floating-point arg size" \
- " in printf interceptor: %d\n", size); \
+ " in printf interceptor: %zu\n", static_cast<uptr>(size)); \
return; \
} \
} else { \
break; \
default: \
Report("WARNING: unexpected arg size" \
- " in printf interceptor: %d\n", size); \
+ " in printf interceptor: %zu\n", static_cast<uptr>(size)); \
return; \
} \
} \
Report(
"%s: WARNING: unexpected format specifier in printf "
"interceptor: %.*s (reported once per process)\n",
- SanitizerToolName, dir.end - dir.begin, dir.begin);
+ SanitizerToolName, static_cast<int>(dir.end - dir.begin), dir.begin);
break;
}
if (dir.convSpecifier == 'n') {
// _(SOUND_MIXER_WRITE_MUTE, WRITE, sizeof(int)); // same as ...WRITE_ENHANCE
_(BLKFLSBUF, NONE, 0);
_(BLKGETSIZE, WRITE, sizeof(uptr));
- _(BLKRAGET, WRITE, sizeof(int));
+ _(BLKRAGET, WRITE, sizeof(uptr));
_(BLKRASET, NONE, 0);
_(BLKROGET, WRITE, sizeof(int));
_(BLKROSET, READ, sizeof(int));
_(BLKRRPART, NONE, 0);
+ _(BLKFRASET, NONE, 0);
+ _(BLKFRAGET, WRITE, sizeof(uptr));
+ _(BLKSECTSET, READ, sizeof(short));
+ _(BLKSECTGET, WRITE, sizeof(short));
+ _(BLKSSZGET, WRITE, sizeof(int));
+ _(BLKBSZGET, WRITE, sizeof(int));
+ _(BLKBSZSET, READ, sizeof(uptr));
+ _(BLKGETSIZE64, WRITE, sizeof(u64));
_(CDROMEJECT, NONE, 0);
_(CDROMEJECT_SW, NONE, 0);
_(CDROMMULTISESSION, WRITE, struct_cdrom_multisession_sz);
INTERCEPTOR(int, statvfs, char *path, void *buf) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, statvfs, path, buf);
- if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+ if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
// FIXME: under ASan the call below may write to freed memory and corrupt
// its metadata. See
// https://github.com/google/sanitizers/issues/321.
INTERCEPTOR(int, statvfs1, const char *path, void *buf, int flags) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, statvfs1, path, buf, flags);
- if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+ if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
int res = REAL(statvfs1)(path, buf, flags);
if (!res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statvfs90_sz);
return res;
.globl ASM_WRAPPER_NAME(vfork)
ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(vfork))
ASM_WRAPPER_NAME(vfork):
+ _CET_ENDBR
// Store return address in the spill area and tear down the stack frame.
sub $12, %esp
call COMMON_INTERCEPTOR_SPILL_AREA
--- /dev/null
+#if defined(__loongarch_lp64) && defined(__linux__)
+
+#include "sanitizer_common/sanitizer_asm.h"
+
+ASM_HIDDEN(COMMON_INTERCEPTOR_SPILL_AREA)
+ASM_HIDDEN(_ZN14__interception10real_vforkE)
+
+.text
+.globl ASM_WRAPPER_NAME(vfork)
+ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(vfork))
+ASM_WRAPPER_NAME(vfork):
+ // Save ra in the off-stack spill area.
+ // allocate space on stack
+ addi.d $sp, $sp, -16
+ // store $ra value
+ st.d $ra, $sp, 8
+ bl COMMON_INTERCEPTOR_SPILL_AREA
+ // restore previous values from stack
+ ld.d $ra, $sp, 8
+ // adjust stack
+ addi.d $sp, $sp, 16
+ // store $ra by $a0
+ st.d $ra, $a0, 0
+
+ // Call real vfork. This may return twice. User code that runs between the first and the second return
+ // may clobber the stack frame of the interceptor; that's why it does not have a frame.
+ la.local $a0, _ZN14__interception10real_vforkE
+ ld.d $a0, $a0, 0
+ jirl $ra, $a0, 0
+
+ // adjust stack
+ addi.d $sp, $sp, -16
+ // store $a0 by adjusted stack
+ st.d $a0, $sp, 8
+ // jump to exit label if $a0 is 0
+ beqz $a0, .L_exit
+
+ // $a0 != 0 => parent process. Clear stack shadow.
+ // put old $sp to $a0
+ addi.d $a0, $sp, 16
+ bl %plt(COMMON_INTERCEPTOR_HANDLE_VFORK)
+
+.L_exit:
+ // Restore $ra
+ bl COMMON_INTERCEPTOR_SPILL_AREA
+ ld.d $ra, $a0, 0
+ // load value by stack
+ ld.d $a0, $sp, 8
+ // adjust stack
+ addi.d $sp, $sp, 16
+ jr $ra
+ASM_SIZE(vfork)
+
+.weak vfork
+.set vfork, ASM_WRAPPER_NAME(vfork)
+
+#endif
.globl ASM_WRAPPER_NAME(vfork)
ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(vfork))
ASM_WRAPPER_NAME(vfork):
+ _CET_ENDBR
// Store return address in the spill area and tear down the stack frame.
push %rcx
call COMMON_INTERCEPTOR_SPILL_AREA
//===----------------------------------------------------------------------===//
INTERFACE_FUNCTION(__sanitizer_acquire_crash_state)
INTERFACE_FUNCTION(__sanitizer_annotate_contiguous_container)
+INTERFACE_FUNCTION(__sanitizer_annotate_double_ended_contiguous_container)
INTERFACE_FUNCTION(__sanitizer_contiguous_container_find_bad_address)
+INTERFACE_FUNCTION(
+ __sanitizer_double_ended_contiguous_container_find_bad_address)
INTERFACE_FUNCTION(__sanitizer_set_death_callback)
INTERFACE_FUNCTION(__sanitizer_set_report_path)
INTERFACE_FUNCTION(__sanitizer_set_report_fd)
INTERFACE_FUNCTION(__sanitizer_get_report_path)
INTERFACE_FUNCTION(__sanitizer_verify_contiguous_container)
+INTERFACE_FUNCTION(__sanitizer_verify_double_ended_contiguous_container)
INTERFACE_WEAK_FUNCTION(__sanitizer_on_print)
INTERFACE_WEAK_FUNCTION(__sanitizer_report_error_summary)
INTERFACE_WEAK_FUNCTION(__sanitizer_sandbox_on_notify)
INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_data)
INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_demangle)
INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_flush)
+INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_set_demangle)
+INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_set_inline_frames)
// run-time libraries.
//===----------------------------------------------------------------------===//
+#include "sanitizer_allocator.h"
#include "sanitizer_allocator_interface.h"
#include "sanitizer_common.h"
#include "sanitizer_flags.h"
+#include "sanitizer_interface_internal.h"
#include "sanitizer_procmaps.h"
-
+#include "sanitizer_stackdepot.h"
namespace __sanitizer {
-static void (*SoftRssLimitExceededCallback)(bool exceeded);
-void SetSoftRssLimitExceededCallback(void (*Callback)(bool exceeded)) {
- CHECK_EQ(SoftRssLimitExceededCallback, nullptr);
- SoftRssLimitExceededCallback = Callback;
-}
-
#if (SANITIZER_LINUX || SANITIZER_NETBSD) && !SANITIZER_GO
// Weak default implementation for when sanitizer_stackdepot is not linked in.
-SANITIZER_WEAK_ATTRIBUTE StackDepotStats *StackDepotGetStats() {
- return nullptr;
-}
+SANITIZER_WEAK_ATTRIBUTE StackDepotStats StackDepotGetStats() { return {}; }
void *BackgroundThread(void *arg) {
+ VPrintf(1, "%s: Started BackgroundThread\n", SanitizerToolName);
const uptr hard_rss_limit_mb = common_flags()->hard_rss_limit_mb;
const uptr soft_rss_limit_mb = common_flags()->soft_rss_limit_mb;
const bool heap_profile = common_flags()->heap_profile;
prev_reported_rss = current_rss_mb;
}
// If stack depot has grown 10% since last time, print it too.
- StackDepotStats *stack_depot_stats = StackDepotGetStats();
- if (stack_depot_stats) {
- if (prev_reported_stack_depot_size * 11 / 10 <
- stack_depot_stats->allocated) {
- Printf("%s: StackDepot: %zd ids; %zdM allocated\n",
- SanitizerToolName,
- stack_depot_stats->n_uniq_ids,
- stack_depot_stats->allocated >> 20);
- prev_reported_stack_depot_size = stack_depot_stats->allocated;
- }
+ StackDepotStats stack_depot_stats = StackDepotGetStats();
+ if (prev_reported_stack_depot_size * 11 / 10 <
+ stack_depot_stats.allocated) {
+ Printf("%s: StackDepot: %zd ids; %zdM allocated\n", SanitizerToolName,
+ stack_depot_stats.n_uniq_ids, stack_depot_stats.allocated >> 20);
+ prev_reported_stack_depot_size = stack_depot_stats.allocated;
}
}
// Check RSS against the limit.
reached_soft_rss_limit = true;
Report("%s: soft rss limit exhausted (%zdMb vs %zdMb)\n",
SanitizerToolName, soft_rss_limit_mb, current_rss_mb);
- if (SoftRssLimitExceededCallback)
- SoftRssLimitExceededCallback(true);
+ SetRssLimitExceeded(true);
} else if (soft_rss_limit_mb >= current_rss_mb &&
reached_soft_rss_limit) {
reached_soft_rss_limit = false;
- if (SoftRssLimitExceededCallback)
- SoftRssLimitExceededCallback(false);
+ SetRssLimitExceeded(false);
}
}
if (heap_profile &&
}
}
}
+
+void MaybeStartBackgroudThread() {
+ // Need to implement/test on other platforms.
+ // Start the background thread if one of the rss limits is given.
+ if (!common_flags()->hard_rss_limit_mb &&
+ !common_flags()->soft_rss_limit_mb &&
+ !common_flags()->heap_profile) return;
+ if (!&real_pthread_create) {
+ VPrintf(1, "%s: real_pthread_create undefined\n", SanitizerToolName);
+ return; // Can't spawn the thread anyway.
+ }
+
+ static bool started = false;
+ if (!started) {
+ started = true;
+ internal_start_thread(BackgroundThread, nullptr);
+ }
+}
+
+# if !SANITIZER_START_BACKGROUND_THREAD_IN_ASAN_INTERNAL
+# ifdef __clang__
+# pragma clang diagnostic push
+// We avoid global-constructors to be sure that globals are ready when
+// sanitizers need them. This can happend before global constructors executed.
+// Here we don't mind if thread is started on later stages.
+# pragma clang diagnostic ignored "-Wglobal-constructors"
+# endif
+static struct BackgroudThreadStarted {
+ BackgroudThreadStarted() { MaybeStartBackgroudThread(); }
+} background_thread_strarter UNUSED;
+# ifdef __clang__
+# pragma clang diagnostic pop
+# endif
+# endif
+#else
+void MaybeStartBackgroudThread() {}
#endif
void WriteToSyslog(const char *msg) {
WriteOneLineToSyslog(p);
}
-void MaybeStartBackgroudThread() {
-#if (SANITIZER_LINUX || SANITIZER_NETBSD) && \
- !SANITIZER_GO // Need to implement/test on other platforms.
- // Start the background thread if one of the rss limits is given.
- if (!common_flags()->hard_rss_limit_mb &&
- !common_flags()->soft_rss_limit_mb &&
- !common_flags()->heap_profile) return;
- if (!&real_pthread_create) return; // Can't spawn the thread anyway.
- internal_start_thread(BackgroundThread, nullptr);
-#endif
-}
-
static void (*sandboxing_callback)();
void SetSandboxingCallback(void (*f)()) {
sandboxing_callback = f;
#endif // !SANITIZER_FUCHSIA
+#if !SANITIZER_WINDOWS && !SANITIZER_GO
+// Weak default implementation for when sanitizer_stackdepot is not linked in.
+SANITIZER_WEAK_ATTRIBUTE void StackDepotStopBackgroundThread() {}
+static void StopStackDepotBackgroundThread() {
+ StackDepotStopBackgroundThread();
+}
+#else
+// SANITIZER_WEAK_ATTRIBUTE is unsupported.
+static void StopStackDepotBackgroundThread() {}
+#endif
+
} // namespace __sanitizer
SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_sandbox_on_notify,
__sanitizer_sandbox_arguments *args) {
+ __sanitizer::StopStackDepotBackgroundThread();
__sanitizer::PlatformPrepareForSandboxing(args);
if (__sanitizer::sandboxing_callback)
__sanitizer::sandboxing_callback();
#endif
void WriteToSyslog(const char *buffer) {}
void Abort() { internal__exit(1); }
+bool CreateDir(const char *pathname) { return false; }
#endif // !SANITIZER_WINDOWS
-#if !SANITIZER_WINDOWS && !SANITIZER_MAC
+#if !SANITIZER_WINDOWS && !SANITIZER_APPLE
void ListOfModules::init() {}
void InitializePlatformCommonFlags(CommonFlags *cf) {}
#endif
#include "sanitizer_platform.h"
#if SANITIZER_LINUX
-#include "sanitizer_libc.h"
+# include "sanitizer_libc.h"
-#define PRE_SYSCALL(name) \
- SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_syscall_pre_impl_##name
-#define PRE_READ(p, s) COMMON_SYSCALL_PRE_READ_RANGE(p, s)
-#define PRE_WRITE(p, s) COMMON_SYSCALL_PRE_WRITE_RANGE(p, s)
+# define PRE_SYSCALL(name) \
+ SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_syscall_pre_impl_##name
+# define PRE_READ(p, s) COMMON_SYSCALL_PRE_READ_RANGE(p, s)
+# define PRE_WRITE(p, s) COMMON_SYSCALL_PRE_WRITE_RANGE(p, s)
-#define POST_SYSCALL(name) \
- SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_syscall_post_impl_##name
-#define POST_READ(p, s) COMMON_SYSCALL_POST_READ_RANGE(p, s)
-#define POST_WRITE(p, s) COMMON_SYSCALL_POST_WRITE_RANGE(p, s)
+# define POST_SYSCALL(name) \
+ SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_syscall_post_impl_##name
+# define POST_READ(p, s) COMMON_SYSCALL_POST_READ_RANGE(p, s)
+# define POST_WRITE(p, s) COMMON_SYSCALL_POST_WRITE_RANGE(p, s)
-#ifndef COMMON_SYSCALL_ACQUIRE
-# define COMMON_SYSCALL_ACQUIRE(addr) ((void)(addr))
-#endif
+# ifndef COMMON_SYSCALL_ACQUIRE
+# define COMMON_SYSCALL_ACQUIRE(addr) ((void)(addr))
+# endif
-#ifndef COMMON_SYSCALL_RELEASE
-# define COMMON_SYSCALL_RELEASE(addr) ((void)(addr))
-#endif
+# ifndef COMMON_SYSCALL_RELEASE
+# define COMMON_SYSCALL_RELEASE(addr) ((void)(addr))
+# endif
-#ifndef COMMON_SYSCALL_FD_CLOSE
-# define COMMON_SYSCALL_FD_CLOSE(fd) ((void)(fd))
-#endif
+# ifndef COMMON_SYSCALL_FD_CLOSE
+# define COMMON_SYSCALL_FD_CLOSE(fd) ((void)(fd))
+# endif
-#ifndef COMMON_SYSCALL_FD_ACQUIRE
-# define COMMON_SYSCALL_FD_ACQUIRE(fd) ((void)(fd))
-#endif
+# ifndef COMMON_SYSCALL_FD_ACQUIRE
+# define COMMON_SYSCALL_FD_ACQUIRE(fd) ((void)(fd))
+# endif
-#ifndef COMMON_SYSCALL_FD_RELEASE
-# define COMMON_SYSCALL_FD_RELEASE(fd) ((void)(fd))
-#endif
+# ifndef COMMON_SYSCALL_FD_RELEASE
+# define COMMON_SYSCALL_FD_RELEASE(fd) ((void)(fd))
+# endif
-#ifndef COMMON_SYSCALL_PRE_FORK
-# define COMMON_SYSCALL_PRE_FORK() {}
-#endif
+# ifndef COMMON_SYSCALL_PRE_FORK
+# define COMMON_SYSCALL_PRE_FORK() \
+ {}
+# endif
-#ifndef COMMON_SYSCALL_POST_FORK
-# define COMMON_SYSCALL_POST_FORK(res) {}
-#endif
+# ifndef COMMON_SYSCALL_POST_FORK
+# define COMMON_SYSCALL_POST_FORK(res) \
+ {}
+# endif
// FIXME: do some kind of PRE_READ for all syscall arguments (int(s) and such).
// Declare it "void" to catch sizeof(kernel_sigset_t).
typedef void kernel_sigset_t;
-static void kernel_write_iovec(const __sanitizer_iovec *iovec,
- SIZE_T iovlen, SIZE_T maxlen) {
+static void kernel_write_iovec(const __sanitizer_iovec *iovec, SIZE_T iovlen,
+ SIZE_T maxlen) {
for (SIZE_T i = 0; i < iovlen && maxlen; ++i) {
SSIZE_T sz = Min(iovec[i].iov_len, maxlen);
POST_WRITE(iovec[i].iov_base, sz);
// This functions uses POST_READ, because it needs to run after syscall to know
// the real read range.
-static void kernel_read_iovec(const __sanitizer_iovec *iovec,
- SIZE_T iovlen, SIZE_T maxlen) {
+static void kernel_read_iovec(const __sanitizer_iovec *iovec, SIZE_T iovlen,
+ SIZE_T maxlen) {
POST_READ(iovec, sizeof(*iovec) * iovlen);
for (SIZE_T i = 0; i < iovlen && maxlen; ++i) {
SSIZE_T sz = Min(iovec[i].iov_len, maxlen);
PRE_READ(msg, sizeof(*msg));
}
-POST_SYSCALL(recvmsg)(long res, long sockfd, sanitizer_kernel_msghdr *msg,
- long flags) {
+POST_SYSCALL(recvmsg)
+(long res, long sockfd, sanitizer_kernel_msghdr *msg, long flags) {
if (res >= 0) {
if (msg) {
for (unsigned long i = 0; i < msg->msg_iovlen; ++i) {
}
}
-PRE_SYSCALL(recvmmsg)(long fd, sanitizer_kernel_mmsghdr *msg, long vlen,
- long flags, void *timeout) {
+PRE_SYSCALL(recvmmsg)
+(long fd, sanitizer_kernel_mmsghdr *msg, long vlen, long flags, void *timeout) {
PRE_READ(msg, vlen * sizeof(*msg));
}
-POST_SYSCALL(recvmmsg)(long res, long fd, sanitizer_kernel_mmsghdr *msg,
- long vlen, long flags, void *timeout) {
+POST_SYSCALL(recvmmsg)
+(long res, long fd, sanitizer_kernel_mmsghdr *msg, long vlen, long flags,
+ void *timeout) {
if (res >= 0) {
if (msg) {
for (unsigned long i = 0; i < msg->msg_hdr.msg_iovlen; ++i) {
POST_WRITE(msg->msg_hdr.msg_control, msg->msg_hdr.msg_controllen);
POST_WRITE(&msg->msg_len, sizeof(msg->msg_len));
}
- if (timeout) POST_WRITE(timeout, struct_timespec_sz);
+ if (timeout)
+ POST_WRITE(timeout, struct_timespec_sz);
}
}
POST_SYSCALL(time)(long res, void *tloc) {
if (res >= 0) {
- if (tloc) POST_WRITE(tloc, sizeof(long));
+ if (tloc)
+ POST_WRITE(tloc, sizeof(long));
}
}
POST_SYSCALL(stime)(long res, void *tptr) {
if (res >= 0) {
- if (tptr) POST_WRITE(tptr, sizeof(long));
+ if (tptr)
+ POST_WRITE(tptr, sizeof(long));
}
}
POST_SYSCALL(gettimeofday)(long res, void *tv, void *tz) {
if (res >= 0) {
- if (tv) POST_WRITE(tv, timeval_sz);
- if (tz) POST_WRITE(tz, struct_timezone_sz);
+ if (tv)
+ POST_WRITE(tv, timeval_sz);
+ if (tz)
+ POST_WRITE(tz, struct_timezone_sz);
}
}
POST_SYSCALL(settimeofday)(long res, void *tv, void *tz) {
if (res >= 0) {
- if (tv) POST_WRITE(tv, timeval_sz);
- if (tz) POST_WRITE(tz, struct_timezone_sz);
+ if (tv)
+ POST_WRITE(tv, timeval_sz);
+ if (tz)
+ POST_WRITE(tz, struct_timezone_sz);
}
}
-#if !SANITIZER_ANDROID
+# if !SANITIZER_ANDROID
PRE_SYSCALL(adjtimex)(void *txc_p) {}
POST_SYSCALL(adjtimex)(long res, void *txc_p) {
if (res >= 0) {
- if (txc_p) POST_WRITE(txc_p, struct_timex_sz);
+ if (txc_p)
+ POST_WRITE(txc_p, struct_timex_sz);
}
}
-#endif
+# endif
PRE_SYSCALL(times)(void *tbuf) {}
POST_SYSCALL(times)(long res, void *tbuf) {
if (res >= 0) {
- if (tbuf) POST_WRITE(tbuf, struct_tms_sz);
+ if (tbuf)
+ POST_WRITE(tbuf, struct_tms_sz);
}
}
POST_SYSCALL(nanosleep)(long res, void *rqtp, void *rmtp) {
if (res >= 0) {
- if (rqtp) POST_WRITE(rqtp, struct_timespec_sz);
- if (rmtp) POST_WRITE(rmtp, struct_timespec_sz);
+ if (rqtp)
+ POST_WRITE(rqtp, struct_timespec_sz);
+ if (rmtp)
+ POST_WRITE(rmtp, struct_timespec_sz);
}
}
POST_SYSCALL(getresuid)(long res, void *ruid, void *euid, void *suid) {
if (res >= 0) {
- if (ruid) POST_WRITE(ruid, sizeof(unsigned));
- if (euid) POST_WRITE(euid, sizeof(unsigned));
- if (suid) POST_WRITE(suid, sizeof(unsigned));
+ if (ruid)
+ POST_WRITE(ruid, sizeof(unsigned));
+ if (euid)
+ POST_WRITE(euid, sizeof(unsigned));
+ if (suid)
+ POST_WRITE(suid, sizeof(unsigned));
}
}
POST_SYSCALL(getresgid)(long res, void *rgid, void *egid, void *sgid) {
if (res >= 0) {
- if (rgid) POST_WRITE(rgid, sizeof(unsigned));
- if (egid) POST_WRITE(egid, sizeof(unsigned));
- if (sgid) POST_WRITE(sgid, sizeof(unsigned));
+ if (rgid)
+ POST_WRITE(rgid, sizeof(unsigned));
+ if (egid)
+ POST_WRITE(egid, sizeof(unsigned));
+ if (sgid)
+ POST_WRITE(sgid, sizeof(unsigned));
}
}
PRE_SYSCALL(getgroups)(long gidsetsize, void *grouplist) {}
-POST_SYSCALL(getgroups)(long res, long gidsetsize,
- __sanitizer___kernel_gid_t *grouplist) {
+POST_SYSCALL(getgroups)
+(long res, long gidsetsize, __sanitizer___kernel_gid_t *grouplist) {
if (res >= 0) {
- if (grouplist) POST_WRITE(grouplist, res * sizeof(*grouplist));
+ if (grouplist)
+ POST_WRITE(grouplist, res * sizeof(*grouplist));
}
}
POST_SYSCALL(setsid)(long res) {}
PRE_SYSCALL(setgroups)(long gidsetsize, __sanitizer___kernel_gid_t *grouplist) {
- if (grouplist) POST_WRITE(grouplist, gidsetsize * sizeof(*grouplist));
+ if (grouplist)
+ POST_WRITE(grouplist, gidsetsize * sizeof(*grouplist));
}
-POST_SYSCALL(setgroups)(long res, long gidsetsize,
- __sanitizer___kernel_gid_t *grouplist) {}
+POST_SYSCALL(setgroups)
+(long res, long gidsetsize, __sanitizer___kernel_gid_t *grouplist) {}
PRE_SYSCALL(acct)(const void *name) {
if (name)
POST_SYSCALL(acct)(long res, const void *name) {}
PRE_SYSCALL(capget)(void *header, void *dataptr) {
- if (header) PRE_READ(header, __user_cap_header_struct_sz);
+ if (header)
+ PRE_READ(header, __user_cap_header_struct_sz);
}
POST_SYSCALL(capget)(long res, void *header, void *dataptr) {
if (res >= 0)
- if (dataptr) POST_WRITE(dataptr, __user_cap_data_struct_sz);
+ if (dataptr)
+ POST_WRITE(dataptr, __user_cap_data_struct_sz);
}
PRE_SYSCALL(capset)(void *header, const void *data) {
- if (header) PRE_READ(header, __user_cap_header_struct_sz);
- if (data) PRE_READ(data, __user_cap_data_struct_sz);
+ if (header)
+ PRE_READ(header, __user_cap_header_struct_sz);
+ if (data)
+ PRE_READ(data, __user_cap_data_struct_sz);
}
POST_SYSCALL(capset)(long res, void *header, const void *data) {}
POST_SYSCALL(sigpending)(long res, void *set) {
if (res >= 0) {
- if (set) POST_WRITE(set, old_sigset_t_sz);
+ if (set)
+ POST_WRITE(set, old_sigset_t_sz);
}
}
POST_SYSCALL(sigprocmask)(long res, long how, void *set, void *oset) {
if (res >= 0) {
- if (set) POST_WRITE(set, old_sigset_t_sz);
- if (oset) POST_WRITE(oset, old_sigset_t_sz);
+ if (set)
+ POST_WRITE(set, old_sigset_t_sz);
+ if (oset)
+ POST_WRITE(oset, old_sigset_t_sz);
}
}
POST_SYSCALL(getitimer)(long res, long which, void *value) {
if (res >= 0) {
- if (value) POST_WRITE(value, struct_itimerval_sz);
+ if (value)
+ POST_WRITE(value, struct_itimerval_sz);
}
}
POST_SYSCALL(setitimer)(long res, long which, void *value, void *ovalue) {
if (res >= 0) {
- if (value) POST_WRITE(value, struct_itimerval_sz);
- if (ovalue) POST_WRITE(ovalue, struct_itimerval_sz);
+ if (value)
+ POST_WRITE(value, struct_itimerval_sz);
+ if (ovalue)
+ POST_WRITE(ovalue, struct_itimerval_sz);
}
}
-PRE_SYSCALL(timer_create)(long which_clock, void *timer_event_spec,
- void *created_timer_id) {}
+PRE_SYSCALL(timer_create)
+(long which_clock, void *timer_event_spec, void *created_timer_id) {}
-POST_SYSCALL(timer_create)(long res, long which_clock, void *timer_event_spec,
- void *created_timer_id) {
+POST_SYSCALL(timer_create)
+(long res, long which_clock, void *timer_event_spec, void *created_timer_id) {
if (res >= 0) {
- if (timer_event_spec) POST_WRITE(timer_event_spec, struct_sigevent_sz);
- if (created_timer_id) POST_WRITE(created_timer_id, sizeof(long));
+ if (timer_event_spec)
+ POST_WRITE(timer_event_spec, struct_sigevent_sz);
+ if (created_timer_id)
+ POST_WRITE(created_timer_id, sizeof(long));
}
}
POST_SYSCALL(timer_gettime)(long res, long timer_id, void *setting) {
if (res >= 0) {
- if (setting) POST_WRITE(setting, struct_itimerspec_sz);
+ if (setting)
+ POST_WRITE(setting, struct_itimerspec_sz);
}
}
POST_SYSCALL(timer_getoverrun)(long res, long timer_id) {}
-PRE_SYSCALL(timer_settime)(long timer_id, long flags, const void *new_setting,
- void *old_setting) {
- if (new_setting) PRE_READ(new_setting, struct_itimerspec_sz);
+PRE_SYSCALL(timer_settime)
+(long timer_id, long flags, const void *new_setting, void *old_setting) {
+ if (new_setting)
+ PRE_READ(new_setting, struct_itimerspec_sz);
}
-POST_SYSCALL(timer_settime)(long res, long timer_id, long flags,
- const void *new_setting, void *old_setting) {
+POST_SYSCALL(timer_settime)
+(long res, long timer_id, long flags, const void *new_setting,
+ void *old_setting) {
if (res >= 0) {
- if (old_setting) POST_WRITE(old_setting, struct_itimerspec_sz);
+ if (old_setting)
+ POST_WRITE(old_setting, struct_itimerspec_sz);
}
}
POST_SYSCALL(timer_delete)(long res, long timer_id) {}
PRE_SYSCALL(clock_settime)(long which_clock, const void *tp) {
- if (tp) PRE_READ(tp, struct_timespec_sz);
+ if (tp)
+ PRE_READ(tp, struct_timespec_sz);
}
POST_SYSCALL(clock_settime)(long res, long which_clock, const void *tp) {}
POST_SYSCALL(clock_gettime)(long res, long which_clock, void *tp) {
if (res >= 0) {
- if (tp) POST_WRITE(tp, struct_timespec_sz);
+ if (tp)
+ POST_WRITE(tp, struct_timespec_sz);
}
}
-#if !SANITIZER_ANDROID
+# if !SANITIZER_ANDROID
PRE_SYSCALL(clock_adjtime)(long which_clock, void *tx) {}
POST_SYSCALL(clock_adjtime)(long res, long which_clock, void *tx) {
if (res >= 0) {
- if (tx) POST_WRITE(tx, struct_timex_sz);
+ if (tx)
+ POST_WRITE(tx, struct_timex_sz);
}
}
-#endif
+# endif
PRE_SYSCALL(clock_getres)(long which_clock, void *tp) {}
POST_SYSCALL(clock_getres)(long res, long which_clock, void *tp) {
if (res >= 0) {
- if (tp) POST_WRITE(tp, struct_timespec_sz);
+ if (tp)
+ POST_WRITE(tp, struct_timespec_sz);
}
}
-PRE_SYSCALL(clock_nanosleep)(long which_clock, long flags, const void *rqtp,
- void *rmtp) {
- if (rqtp) PRE_READ(rqtp, struct_timespec_sz);
+PRE_SYSCALL(clock_nanosleep)
+(long which_clock, long flags, const void *rqtp, void *rmtp) {
+ if (rqtp)
+ PRE_READ(rqtp, struct_timespec_sz);
}
-POST_SYSCALL(clock_nanosleep)(long res, long which_clock, long flags,
- const void *rqtp, void *rmtp) {
+POST_SYSCALL(clock_nanosleep)
+(long res, long which_clock, long flags, const void *rqtp, void *rmtp) {
if (res >= 0) {
- if (rmtp) POST_WRITE(rmtp, struct_timespec_sz);
+ if (rmtp)
+ POST_WRITE(rmtp, struct_timespec_sz);
}
}
POST_SYSCALL(sched_setscheduler)(long res, long pid, long policy, void *param) {
if (res >= 0) {
- if (param) POST_WRITE(param, struct_sched_param_sz);
+ if (param)
+ POST_WRITE(param, struct_sched_param_sz);
}
}
PRE_SYSCALL(sched_setparam)(long pid, void *param) {
- if (param) PRE_READ(param, struct_sched_param_sz);
+ if (param)
+ PRE_READ(param, struct_sched_param_sz);
}
POST_SYSCALL(sched_setparam)(long res, long pid, void *param) {}
POST_SYSCALL(sched_getparam)(long res, long pid, void *param) {
if (res >= 0) {
- if (param) POST_WRITE(param, struct_sched_param_sz);
+ if (param)
+ POST_WRITE(param, struct_sched_param_sz);
}
}
PRE_SYSCALL(sched_setaffinity)(long pid, long len, void *user_mask_ptr) {
- if (user_mask_ptr) PRE_READ(user_mask_ptr, len);
+ if (user_mask_ptr)
+ PRE_READ(user_mask_ptr, len);
}
-POST_SYSCALL(sched_setaffinity)(long res, long pid, long len,
- void *user_mask_ptr) {}
+POST_SYSCALL(sched_setaffinity)
+(long res, long pid, long len, void *user_mask_ptr) {}
PRE_SYSCALL(sched_getaffinity)(long pid, long len, void *user_mask_ptr) {}
-POST_SYSCALL(sched_getaffinity)(long res, long pid, long len,
- void *user_mask_ptr) {
+POST_SYSCALL(sched_getaffinity)
+(long res, long pid, long len, void *user_mask_ptr) {
if (res >= 0) {
- if (user_mask_ptr) POST_WRITE(user_mask_ptr, len);
+ if (user_mask_ptr)
+ POST_WRITE(user_mask_ptr, len);
}
}
POST_SYSCALL(sched_rr_get_interval)(long res, long pid, void *interval) {
if (res >= 0) {
- if (interval) POST_WRITE(interval, struct_timespec_sz);
+ if (interval)
+ POST_WRITE(interval, struct_timespec_sz);
}
}
POST_SYSCALL(restart_syscall)(long res) {}
-PRE_SYSCALL(kexec_load)(long entry, long nr_segments, void *segments,
- long flags) {}
+PRE_SYSCALL(kexec_load)
+(long entry, long nr_segments, void *segments, long flags) {}
-POST_SYSCALL(kexec_load)(long res, long entry, long nr_segments, void *segments,
- long flags) {
+POST_SYSCALL(kexec_load)
+(long res, long entry, long nr_segments, void *segments, long flags) {
if (res >= 0) {
- if (segments) POST_WRITE(segments, struct_kexec_segment_sz);
+ if (segments)
+ POST_WRITE(segments, struct_kexec_segment_sz);
}
}
PRE_SYSCALL(wait4)(long pid, void *stat_addr, long options, void *ru) {}
-POST_SYSCALL(wait4)(long res, long pid, void *stat_addr, long options,
- void *ru) {
+POST_SYSCALL(wait4)
+(long res, long pid, void *stat_addr, long options, void *ru) {
if (res >= 0) {
- if (stat_addr) POST_WRITE(stat_addr, sizeof(int));
- if (ru) POST_WRITE(ru, struct_rusage_sz);
+ if (stat_addr)
+ POST_WRITE(stat_addr, sizeof(int));
+ if (ru)
+ POST_WRITE(ru, struct_rusage_sz);
}
}
-PRE_SYSCALL(waitid)(long which, long pid, void *infop, long options, void *ru) {
-}
+PRE_SYSCALL(waitid)
+(long which, long pid, void *infop, long options, void *ru) {}
-POST_SYSCALL(waitid)(long res, long which, long pid, void *infop, long options,
- void *ru) {
+POST_SYSCALL(waitid)
+(long res, long which, long pid, void *infop, long options, void *ru) {
if (res >= 0) {
- if (infop) POST_WRITE(infop, siginfo_t_sz);
- if (ru) POST_WRITE(ru, struct_rusage_sz);
+ if (infop)
+ POST_WRITE(infop, siginfo_t_sz);
+ if (ru)
+ POST_WRITE(ru, struct_rusage_sz);
}
}
POST_SYSCALL(waitpid)(long res, long pid, void *stat_addr, long options) {
if (res >= 0) {
- if (stat_addr) POST_WRITE(stat_addr, sizeof(int));
+ if (stat_addr)
+ POST_WRITE(stat_addr, sizeof(int));
}
}
POST_SYSCALL(set_tid_address)(long res, void *tidptr) {
if (res >= 0) {
- if (tidptr) POST_WRITE(tidptr, sizeof(int));
+ if (tidptr)
+ POST_WRITE(tidptr, sizeof(int));
}
}
PRE_SYSCALL(rt_sigprocmask)(long how, void *set, void *oset, long sigsetsize) {}
-POST_SYSCALL(rt_sigprocmask)(long res, long how, kernel_sigset_t *set,
- kernel_sigset_t *oset, long sigsetsize) {
+POST_SYSCALL(rt_sigprocmask)
+(long res, long how, kernel_sigset_t *set, kernel_sigset_t *oset,
+ long sigsetsize) {
if (res >= 0) {
- if (set) POST_WRITE(set, sigsetsize);
- if (oset) POST_WRITE(oset, sigsetsize);
+ if (set)
+ POST_WRITE(set, sigsetsize);
+ if (oset)
+ POST_WRITE(oset, sigsetsize);
}
}
POST_SYSCALL(rt_sigpending)(long res, kernel_sigset_t *set, long sigsetsize) {
if (res >= 0) {
- if (set) POST_WRITE(set, sigsetsize);
+ if (set)
+ POST_WRITE(set, sigsetsize);
}
}
-PRE_SYSCALL(rt_sigtimedwait)(const kernel_sigset_t *uthese, void *uinfo,
- const void *uts, long sigsetsize) {
- if (uthese) PRE_READ(uthese, sigsetsize);
- if (uts) PRE_READ(uts, struct_timespec_sz);
+PRE_SYSCALL(rt_sigtimedwait)
+(const kernel_sigset_t *uthese, void *uinfo, const void *uts, long sigsetsize) {
+ if (uthese)
+ PRE_READ(uthese, sigsetsize);
+ if (uts)
+ PRE_READ(uts, struct_timespec_sz);
}
-POST_SYSCALL(rt_sigtimedwait)(long res, const void *uthese, void *uinfo,
- const void *uts, long sigsetsize) {
+POST_SYSCALL(rt_sigtimedwait)
+(long res, const void *uthese, void *uinfo, const void *uts, long sigsetsize) {
if (res >= 0) {
- if (uinfo) POST_WRITE(uinfo, siginfo_t_sz);
+ if (uinfo)
+ POST_WRITE(uinfo, siginfo_t_sz);
}
}
PRE_SYSCALL(rt_tgsigqueueinfo)(long tgid, long pid, long sig, void *uinfo) {}
-POST_SYSCALL(rt_tgsigqueueinfo)(long res, long tgid, long pid, long sig,
- void *uinfo) {
+POST_SYSCALL(rt_tgsigqueueinfo)
+(long res, long tgid, long pid, long sig, void *uinfo) {
if (res >= 0) {
- if (uinfo) POST_WRITE(uinfo, siginfo_t_sz);
+ if (uinfo)
+ POST_WRITE(uinfo, siginfo_t_sz);
}
}
POST_SYSCALL(rt_sigqueueinfo)(long res, long pid, long sig, void *uinfo) {
if (res >= 0) {
- if (uinfo) POST_WRITE(uinfo, siginfo_t_sz);
+ if (uinfo)
+ POST_WRITE(uinfo, siginfo_t_sz);
}
}
POST_SYSCALL(bdflush)(long res, long func, long data) {}
-PRE_SYSCALL(mount)(void *dev_name, void *dir_name, void *type, long flags,
- void *data) {}
+PRE_SYSCALL(mount)
+(void *dev_name, void *dir_name, void *type, long flags, void *data) {}
-POST_SYSCALL(mount)(long res, void *dev_name, void *dir_name, void *type,
- long flags, void *data) {
+POST_SYSCALL(mount)
+(long res, void *dev_name, void *dir_name, void *type, long flags, void *data) {
if (res >= 0) {
if (dev_name)
POST_WRITE(dev_name,
POST_SYSCALL(stat)(long res, const void *filename, void *statbuf) {
if (res >= 0) {
- if (statbuf) POST_WRITE(statbuf, struct___old_kernel_stat_sz);
+ if (statbuf)
+ POST_WRITE(statbuf, struct___old_kernel_stat_sz);
}
}
-#if !SANITIZER_ANDROID
+# if !SANITIZER_ANDROID
PRE_SYSCALL(statfs)(const void *path, void *buf) {
if (path)
PRE_READ(path, __sanitizer::internal_strlen((const char *)path) + 1);
POST_SYSCALL(statfs)(long res, const void *path, void *buf) {
if (res >= 0) {
- if (buf) POST_WRITE(buf, struct_statfs_sz);
+ if (buf)
+ POST_WRITE(buf, struct_statfs_sz);
}
}
-PRE_SYSCALL(statfs64)(const void *path, long sz, void *buf) {
- if (path)
- PRE_READ(path, __sanitizer::internal_strlen((const char *)path) + 1);
-}
+PRE_SYSCALL(fstatfs)(long fd, void *buf) {}
-POST_SYSCALL(statfs64)(long res, const void *path, long sz, void *buf) {
+POST_SYSCALL(fstatfs)(long res, long fd, void *buf) {
if (res >= 0) {
- if (buf) POST_WRITE(buf, struct_statfs64_sz);
+ if (buf)
+ POST_WRITE(buf, struct_statfs_sz);
}
}
+# endif // !SANITIZER_ANDROID
-PRE_SYSCALL(fstatfs)(long fd, void *buf) {}
+# if SANITIZER_GLIBC
+PRE_SYSCALL(statfs64)(const void *path, long sz, void *buf) {
+ if (path)
+ PRE_READ(path, __sanitizer::internal_strlen((const char *)path) + 1);
+}
-POST_SYSCALL(fstatfs)(long res, long fd, void *buf) {
+POST_SYSCALL(statfs64)(long res, const void *path, long sz, void *buf) {
if (res >= 0) {
- if (buf) POST_WRITE(buf, struct_statfs_sz);
+ if (buf)
+ POST_WRITE(buf, struct_statfs64_sz);
}
}
POST_SYSCALL(fstatfs64)(long res, long fd, long sz, void *buf) {
if (res >= 0) {
- if (buf) POST_WRITE(buf, struct_statfs64_sz);
+ if (buf)
+ POST_WRITE(buf, struct_statfs64_sz);
}
}
-#endif // !SANITIZER_ANDROID
+# endif // SANITIZER_GLIBC
PRE_SYSCALL(lstat)(const void *filename, void *statbuf) {
if (filename)
POST_SYSCALL(lstat)(long res, const void *filename, void *statbuf) {
if (res >= 0) {
- if (statbuf) POST_WRITE(statbuf, struct___old_kernel_stat_sz);
+ if (statbuf)
+ POST_WRITE(statbuf, struct___old_kernel_stat_sz);
}
}
POST_SYSCALL(fstat)(long res, long fd, void *statbuf) {
if (res >= 0) {
- if (statbuf) POST_WRITE(statbuf, struct___old_kernel_stat_sz);
+ if (statbuf)
+ POST_WRITE(statbuf, struct___old_kernel_stat_sz);
}
}
POST_SYSCALL(newstat)(long res, const void *filename, void *statbuf) {
if (res >= 0) {
- if (statbuf) POST_WRITE(statbuf, struct_kernel_stat_sz);
+ if (statbuf)
+ POST_WRITE(statbuf, struct_kernel_stat_sz);
}
}
POST_SYSCALL(newlstat)(long res, const void *filename, void *statbuf) {
if (res >= 0) {
- if (statbuf) POST_WRITE(statbuf, struct_kernel_stat_sz);
+ if (statbuf)
+ POST_WRITE(statbuf, struct_kernel_stat_sz);
}
}
POST_SYSCALL(newfstat)(long res, long fd, void *statbuf) {
if (res >= 0) {
- if (statbuf) POST_WRITE(statbuf, struct_kernel_stat_sz);
+ if (statbuf)
+ POST_WRITE(statbuf, struct_kernel_stat_sz);
}
}
-#if !SANITIZER_ANDROID
+# if SANITIZER_GLIBC
PRE_SYSCALL(ustat)(long dev, void *ubuf) {}
POST_SYSCALL(ustat)(long res, long dev, void *ubuf) {
if (res >= 0) {
- if (ubuf) POST_WRITE(ubuf, struct_ustat_sz);
+ if (ubuf)
+ POST_WRITE(ubuf, struct_ustat_sz);
}
}
-#endif // !SANITIZER_ANDROID
+# endif // SANITIZER_GLIBC
PRE_SYSCALL(stat64)(const void *filename, void *statbuf) {
if (filename)
POST_SYSCALL(stat64)(long res, const void *filename, void *statbuf) {
if (res >= 0) {
- if (statbuf) POST_WRITE(statbuf, struct_kernel_stat64_sz);
+ if (statbuf)
+ POST_WRITE(statbuf, struct_kernel_stat64_sz);
}
}
POST_SYSCALL(fstat64)(long res, long fd, void *statbuf) {
if (res >= 0) {
- if (statbuf) POST_WRITE(statbuf, struct_kernel_stat64_sz);
+ if (statbuf)
+ POST_WRITE(statbuf, struct_kernel_stat64_sz);
}
}
POST_SYSCALL(lstat64)(long res, const void *filename, void *statbuf) {
if (res >= 0) {
- if (statbuf) POST_WRITE(statbuf, struct_kernel_stat64_sz);
+ if (statbuf)
+ POST_WRITE(statbuf, struct_kernel_stat64_sz);
}
}
-PRE_SYSCALL(setxattr)(const void *path, const void *name, const void *value,
- long size, long flags) {
+PRE_SYSCALL(setxattr)
+(const void *path, const void *name, const void *value, long size, long flags) {
if (path)
PRE_READ(path, __sanitizer::internal_strlen((const char *)path) + 1);
if (name)
PRE_READ(name, __sanitizer::internal_strlen((const char *)name) + 1);
- if (value) PRE_READ(value, size);
+ if (value)
+ PRE_READ(value, size);
}
-POST_SYSCALL(setxattr)(long res, const void *path, const void *name,
- const void *value, long size, long flags) {}
+POST_SYSCALL(setxattr)
+(long res, const void *path, const void *name, const void *value, long size,
+ long flags) {}
-PRE_SYSCALL(lsetxattr)(const void *path, const void *name, const void *value,
- long size, long flags) {
+PRE_SYSCALL(lsetxattr)
+(const void *path, const void *name, const void *value, long size, long flags) {
if (path)
PRE_READ(path, __sanitizer::internal_strlen((const char *)path) + 1);
if (name)
PRE_READ(name, __sanitizer::internal_strlen((const char *)name) + 1);
- if (value) PRE_READ(value, size);
+ if (value)
+ PRE_READ(value, size);
}
-POST_SYSCALL(lsetxattr)(long res, const void *path, const void *name,
- const void *value, long size, long flags) {}
+POST_SYSCALL(lsetxattr)
+(long res, const void *path, const void *name, const void *value, long size,
+ long flags) {}
-PRE_SYSCALL(fsetxattr)(long fd, const void *name, const void *value, long size,
- long flags) {
+PRE_SYSCALL(fsetxattr)
+(long fd, const void *name, const void *value, long size, long flags) {
if (name)
PRE_READ(name, __sanitizer::internal_strlen((const char *)name) + 1);
- if (value) PRE_READ(value, size);
+ if (value)
+ PRE_READ(value, size);
}
-POST_SYSCALL(fsetxattr)(long res, long fd, const void *name, const void *value,
- long size, long flags) {}
+POST_SYSCALL(fsetxattr)
+(long res, long fd, const void *name, const void *value, long size,
+ long flags) {}
-PRE_SYSCALL(getxattr)(const void *path, const void *name, void *value,
- long size) {
+PRE_SYSCALL(getxattr)
+(const void *path, const void *name, void *value, long size) {
if (path)
PRE_READ(path, __sanitizer::internal_strlen((const char *)path) + 1);
if (name)
PRE_READ(name, __sanitizer::internal_strlen((const char *)name) + 1);
}
-POST_SYSCALL(getxattr)(long res, const void *path, const void *name,
- void *value, long size) {
+POST_SYSCALL(getxattr)
+(long res, const void *path, const void *name, void *value, long size) {
if (size && res > 0) {
- if (value) POST_WRITE(value, res);
+ if (value)
+ POST_WRITE(value, res);
}
}
-PRE_SYSCALL(lgetxattr)(const void *path, const void *name, void *value,
- long size) {
+PRE_SYSCALL(lgetxattr)
+(const void *path, const void *name, void *value, long size) {
if (path)
PRE_READ(path, __sanitizer::internal_strlen((const char *)path) + 1);
if (name)
PRE_READ(name, __sanitizer::internal_strlen((const char *)name) + 1);
}
-POST_SYSCALL(lgetxattr)(long res, const void *path, const void *name,
- void *value, long size) {
+POST_SYSCALL(lgetxattr)
+(long res, const void *path, const void *name, void *value, long size) {
if (size && res > 0) {
- if (value) POST_WRITE(value, res);
+ if (value)
+ POST_WRITE(value, res);
}
}
PRE_READ(name, __sanitizer::internal_strlen((const char *)name) + 1);
}
-POST_SYSCALL(fgetxattr)(long res, long fd, const void *name, void *value,
- long size) {
+POST_SYSCALL(fgetxattr)
+(long res, long fd, const void *name, void *value, long size) {
if (size && res > 0) {
- if (value) POST_WRITE(value, res);
+ if (value)
+ POST_WRITE(value, res);
}
}
POST_SYSCALL(listxattr)(long res, const void *path, void *list, long size) {
if (size && res > 0) {
- if (list) POST_WRITE(list, res);
+ if (list)
+ POST_WRITE(list, res);
}
}
POST_SYSCALL(llistxattr)(long res, const void *path, void *list, long size) {
if (size && res > 0) {
- if (list) POST_WRITE(list, res);
+ if (list)
+ POST_WRITE(list, res);
}
}
POST_SYSCALL(flistxattr)(long res, long fd, void *list, long size) {
if (size && res > 0) {
- if (list) POST_WRITE(list, res);
+ if (list)
+ POST_WRITE(list, res);
}
}
POST_SYSCALL(mprotect)(long res, long start, long len, long prot) {}
-PRE_SYSCALL(mremap)(long addr, long old_len, long new_len, long flags,
- long new_addr) {}
+PRE_SYSCALL(mremap)
+(long addr, long old_len, long new_len, long flags, long new_addr) {}
-POST_SYSCALL(mremap)(long res, long addr, long old_len, long new_len,
- long flags, long new_addr) {}
+POST_SYSCALL(mremap)
+(long res, long addr, long old_len, long new_len, long flags, long new_addr) {}
-PRE_SYSCALL(remap_file_pages)(long start, long size, long prot, long pgoff,
- long flags) {}
+PRE_SYSCALL(remap_file_pages)
+(long start, long size, long prot, long pgoff, long flags) {}
-POST_SYSCALL(remap_file_pages)(long res, long start, long size, long prot,
- long pgoff, long flags) {}
+POST_SYSCALL(remap_file_pages)
+(long res, long start, long size, long prot, long pgoff, long flags) {}
PRE_SYSCALL(msync)(long start, long len, long flags) {}
POST_SYSCALL(link)(long res, const void *oldname, const void *newname) {}
PRE_SYSCALL(symlink)(const void *old, const void *new_) {
- if (old) PRE_READ(old, __sanitizer::internal_strlen((const char *)old) + 1);
+ if (old)
+ PRE_READ(old, __sanitizer::internal_strlen((const char *)old) + 1);
if (new_)
PRE_READ(new_, __sanitizer::internal_strlen((const char *)new_) + 1);
}
POST_SYSCALL(pipe)(long res, void *fildes) {
if (res >= 0)
- if (fildes) POST_WRITE(fildes, sizeof(int) * 2);
+ if (fildes)
+ POST_WRITE(fildes, sizeof(int) * 2);
}
PRE_SYSCALL(pipe2)(void *fildes, long flags) {}
POST_SYSCALL(pipe2)(long res, void *fildes, long flags) {
if (res >= 0)
- if (fildes) POST_WRITE(fildes, sizeof(int) * 2);
+ if (fildes)
+ POST_WRITE(fildes, sizeof(int) * 2);
}
PRE_SYSCALL(dup)(long fildes) {}
POST_SYSCALL(flock)(long res, long fd, long cmd) {}
PRE_SYSCALL(io_setup)(long nr_reqs, void **ctx) {
- if (ctx) PRE_WRITE(ctx, sizeof(*ctx));
+ if (ctx)
+ PRE_WRITE(ctx, sizeof(*ctx));
}
POST_SYSCALL(io_setup)(long res, long nr_reqs, void **ctx) {
if (res >= 0) {
- if (ctx) POST_WRITE(ctx, sizeof(*ctx));
+ if (ctx)
+ POST_WRITE(ctx, sizeof(*ctx));
// (*ctx) is actually a pointer to a kernel mapped page, and there are
// people out there who are crazy enough to peek into that page's 32-byte
// header.
- if (*ctx) POST_WRITE(*ctx, 32);
+ if (*ctx)
+ POST_WRITE(*ctx, 32);
}
}
POST_SYSCALL(io_destroy)(long res, long ctx) {}
-PRE_SYSCALL(io_getevents)(long ctx_id, long min_nr, long nr,
- __sanitizer_io_event *ioevpp, void *timeout) {
- if (timeout) PRE_READ(timeout, struct_timespec_sz);
+PRE_SYSCALL(io_getevents)
+(long ctx_id, long min_nr, long nr, __sanitizer_io_event *ioevpp,
+ void *timeout) {
+ if (timeout)
+ PRE_READ(timeout, struct_timespec_sz);
}
-POST_SYSCALL(io_getevents)(long res, long ctx_id, long min_nr, long nr,
- __sanitizer_io_event *ioevpp, void *timeout) {
+POST_SYSCALL(io_getevents)
+(long res, long ctx_id, long min_nr, long nr, __sanitizer_io_event *ioevpp,
+ void *timeout) {
if (res >= 0) {
- if (ioevpp) POST_WRITE(ioevpp, res * sizeof(*ioevpp));
- if (timeout) POST_WRITE(timeout, struct_timespec_sz);
+ if (ioevpp)
+ POST_WRITE(ioevpp, res * sizeof(*ioevpp));
+ if (timeout)
+ POST_WRITE(timeout, struct_timespec_sz);
}
for (long i = 0; i < res; i++) {
// We synchronize io_submit -> io_getevents/io_cancel using the
// synchronize on 0. But there does not seem to be a better solution
// (except wrapping all operations in own context, which is unreliable).
// We can not reliably extract fildes in io_getevents.
- COMMON_SYSCALL_ACQUIRE((void*)ioevpp[i].data);
+ COMMON_SYSCALL_ACQUIRE((void *)ioevpp[i].data);
}
}
PRE_SYSCALL(io_submit)(long ctx_id, long nr, __sanitizer_iocb **iocbpp) {
for (long i = 0; i < nr; ++i) {
uptr op = iocbpp[i]->aio_lio_opcode;
- void *data = (void*)iocbpp[i]->aio_data;
- void *buf = (void*)iocbpp[i]->aio_buf;
+ void *data = (void *)iocbpp[i]->aio_data;
+ void *buf = (void *)iocbpp[i]->aio_buf;
uptr len = (uptr)iocbpp[i]->aio_nbytes;
if (op == iocb_cmd_pwrite && buf && len) {
PRE_READ(buf, len);
} else if (op == iocb_cmd_pread && buf && len) {
POST_WRITE(buf, len);
} else if (op == iocb_cmd_pwritev) {
- __sanitizer_iovec *iovec = (__sanitizer_iovec*)buf;
+ __sanitizer_iovec *iovec = (__sanitizer_iovec *)buf;
for (uptr v = 0; v < len; v++)
PRE_READ(iovec[v].iov_base, iovec[v].iov_len);
} else if (op == iocb_cmd_preadv) {
- __sanitizer_iovec *iovec = (__sanitizer_iovec*)buf;
+ __sanitizer_iovec *iovec = (__sanitizer_iovec *)buf;
for (uptr v = 0; v < len; v++)
POST_WRITE(iovec[v].iov_base, iovec[v].iov_len);
}
}
}
-POST_SYSCALL(io_submit)(long res, long ctx_id, long nr,
- __sanitizer_iocb **iocbpp) {}
+POST_SYSCALL(io_submit)
+(long res, long ctx_id, long nr, __sanitizer_iocb **iocbpp) {}
-PRE_SYSCALL(io_cancel)(long ctx_id, __sanitizer_iocb *iocb,
- __sanitizer_io_event *result) {
-}
+PRE_SYSCALL(io_cancel)
+(long ctx_id, __sanitizer_iocb *iocb, __sanitizer_io_event *result) {}
-POST_SYSCALL(io_cancel)(long res, long ctx_id, __sanitizer_iocb *iocb,
- __sanitizer_io_event *result) {
+POST_SYSCALL(io_cancel)
+(long res, long ctx_id, __sanitizer_iocb *iocb, __sanitizer_io_event *result) {
if (res == 0) {
if (result) {
// See comment in io_getevents.
- COMMON_SYSCALL_ACQUIRE((void*)result->data);
+ COMMON_SYSCALL_ACQUIRE((void *)result->data);
POST_WRITE(result, sizeof(*result));
}
if (iocb)
PRE_SYSCALL(sendfile)(long out_fd, long in_fd, void *offset, long count) {}
-POST_SYSCALL(sendfile)(long res, long out_fd, long in_fd,
- __sanitizer___kernel_off_t *offset, long count) {
+POST_SYSCALL(sendfile)
+(long res, long out_fd, long in_fd, __sanitizer___kernel_off_t *offset,
+ long count) {
if (res >= 0) {
- if (offset) POST_WRITE(offset, sizeof(*offset));
+ if (offset)
+ POST_WRITE(offset, sizeof(*offset));
}
}
PRE_SYSCALL(sendfile64)(long out_fd, long in_fd, void *offset, long count) {}
-POST_SYSCALL(sendfile64)(long res, long out_fd, long in_fd,
- __sanitizer___kernel_loff_t *offset, long count) {
+POST_SYSCALL(sendfile64)
+(long res, long out_fd, long in_fd, __sanitizer___kernel_loff_t *offset,
+ long count) {
if (res >= 0) {
- if (offset) POST_WRITE(offset, sizeof(*offset));
+ if (offset)
+ POST_WRITE(offset, sizeof(*offset));
}
}
POST_SYSCALL(open)(long res, const void *filename, long flags, long mode) {}
-PRE_SYSCALL(close)(long fd) {
- COMMON_SYSCALL_FD_CLOSE((int)fd);
-}
+PRE_SYSCALL(close)(long fd) { COMMON_SYSCALL_FD_CLOSE((int)fd); }
POST_SYSCALL(close)(long res, long fd) {}
POST_SYSCALL(fchown)(long res, long fd, long user, long group) {}
-#if SANITIZER_USES_UID16_SYSCALLS
+# if SANITIZER_USES_UID16_SYSCALLS
PRE_SYSCALL(chown16)(const void *filename, long user, long group) {
if (filename)
PRE_READ(filename,
PRE_SYSCALL(getresuid16)(void *ruid, void *euid, void *suid) {}
-POST_SYSCALL(getresuid16)(long res, __sanitizer___kernel_old_uid_t *ruid,
- __sanitizer___kernel_old_uid_t *euid,
- __sanitizer___kernel_old_uid_t *suid) {
+POST_SYSCALL(getresuid16)
+(long res, __sanitizer___kernel_old_uid_t *ruid,
+ __sanitizer___kernel_old_uid_t *euid, __sanitizer___kernel_old_uid_t *suid) {
if (res >= 0) {
- if (ruid) POST_WRITE(ruid, sizeof(*ruid));
- if (euid) POST_WRITE(euid, sizeof(*euid));
- if (suid) POST_WRITE(suid, sizeof(*suid));
+ if (ruid)
+ POST_WRITE(ruid, sizeof(*ruid));
+ if (euid)
+ POST_WRITE(euid, sizeof(*euid));
+ if (suid)
+ POST_WRITE(suid, sizeof(*suid));
}
}
PRE_SYSCALL(getresgid16)(void *rgid, void *egid, void *sgid) {}
-POST_SYSCALL(getresgid16)(long res, __sanitizer___kernel_old_gid_t *rgid,
- __sanitizer___kernel_old_gid_t *egid,
- __sanitizer___kernel_old_gid_t *sgid) {
+POST_SYSCALL(getresgid16)
+(long res, __sanitizer___kernel_old_gid_t *rgid,
+ __sanitizer___kernel_old_gid_t *egid, __sanitizer___kernel_old_gid_t *sgid) {
if (res >= 0) {
- if (rgid) POST_WRITE(rgid, sizeof(*rgid));
- if (egid) POST_WRITE(egid, sizeof(*egid));
- if (sgid) POST_WRITE(sgid, sizeof(*sgid));
+ if (rgid)
+ POST_WRITE(rgid, sizeof(*rgid));
+ if (egid)
+ POST_WRITE(egid, sizeof(*egid));
+ if (sgid)
+ POST_WRITE(sgid, sizeof(*sgid));
}
}
POST_SYSCALL(setfsgid16)(long res, long gid) {}
-PRE_SYSCALL(getgroups16)(long gidsetsize,
- __sanitizer___kernel_old_gid_t *grouplist) {}
+PRE_SYSCALL(getgroups16)
+(long gidsetsize, __sanitizer___kernel_old_gid_t *grouplist) {}
-POST_SYSCALL(getgroups16)(long res, long gidsetsize,
- __sanitizer___kernel_old_gid_t *grouplist) {
+POST_SYSCALL(getgroups16)
+(long res, long gidsetsize, __sanitizer___kernel_old_gid_t *grouplist) {
if (res >= 0) {
- if (grouplist) POST_WRITE(grouplist, res * sizeof(*grouplist));
+ if (grouplist)
+ POST_WRITE(grouplist, res * sizeof(*grouplist));
}
}
-PRE_SYSCALL(setgroups16)(long gidsetsize,
- __sanitizer___kernel_old_gid_t *grouplist) {
- if (grouplist) POST_WRITE(grouplist, gidsetsize * sizeof(*grouplist));
+PRE_SYSCALL(setgroups16)
+(long gidsetsize, __sanitizer___kernel_old_gid_t *grouplist) {
+ if (grouplist)
+ POST_WRITE(grouplist, gidsetsize * sizeof(*grouplist));
}
-POST_SYSCALL(setgroups16)(long res, long gidsetsize,
- __sanitizer___kernel_old_gid_t *grouplist) {}
+POST_SYSCALL(setgroups16)
+(long res, long gidsetsize, __sanitizer___kernel_old_gid_t *grouplist) {}
PRE_SYSCALL(getuid16)() {}
PRE_SYSCALL(getegid16)() {}
POST_SYSCALL(getegid16)(long res) {}
-#endif // SANITIZER_USES_UID16_SYSCALLS
+# endif // SANITIZER_USES_UID16_SYSCALLS
PRE_SYSCALL(utime)(void *filename, void *times) {}
if (filename)
POST_WRITE(filename,
__sanitizer::internal_strlen((const char *)filename) + 1);
- if (times) POST_WRITE(times, struct_utimbuf_sz);
+ if (times)
+ POST_WRITE(times, struct_utimbuf_sz);
}
}
if (filename)
POST_WRITE(filename,
__sanitizer::internal_strlen((const char *)filename) + 1);
- if (utimes) POST_WRITE(utimes, timeval_sz);
+ if (utimes)
+ POST_WRITE(utimes, timeval_sz);
}
}
POST_SYSCALL(lseek)(long res, long fd, long offset, long origin) {}
-PRE_SYSCALL(llseek)(long fd, long offset_high, long offset_low, void *result,
- long origin) {}
+PRE_SYSCALL(llseek)
+(long fd, long offset_high, long offset_low, void *result, long origin) {}
-POST_SYSCALL(llseek)(long res, long fd, long offset_high, long offset_low,
- void *result, long origin) {
+POST_SYSCALL(llseek)
+(long res, long fd, long offset_high, long offset_low, void *result,
+ long origin) {
if (res >= 0) {
- if (result) POST_WRITE(result, sizeof(long long));
+ if (result)
+ POST_WRITE(result, sizeof(long long));
}
}
PRE_SYSCALL(readv)(long fd, const __sanitizer_iovec *vec, long vlen) {}
-POST_SYSCALL(readv)(long res, long fd, const __sanitizer_iovec *vec,
- long vlen) {
+POST_SYSCALL(readv)
+(long res, long fd, const __sanitizer_iovec *vec, long vlen) {
if (res >= 0) {
- if (vec) kernel_write_iovec(vec, vlen, res);
+ if (vec)
+ kernel_write_iovec(vec, vlen, res);
}
}
PRE_SYSCALL(write)(long fd, const void *buf, long count) {
- if (buf) PRE_READ(buf, count);
+ if (buf)
+ PRE_READ(buf, count);
}
POST_SYSCALL(write)(long res, long fd, const void *buf, long count) {}
PRE_SYSCALL(writev)(long fd, const __sanitizer_iovec *vec, long vlen) {}
-POST_SYSCALL(writev)(long res, long fd, const __sanitizer_iovec *vec,
- long vlen) {
+POST_SYSCALL(writev)
+(long res, long fd, const __sanitizer_iovec *vec, long vlen) {
if (res >= 0) {
- if (vec) kernel_read_iovec(vec, vlen, res);
+ if (vec)
+ kernel_read_iovec(vec, vlen, res);
}
}
-#ifdef _LP64
+# ifdef _LP64
PRE_SYSCALL(pread64)(long fd, void *buf, long count, long pos) {}
POST_SYSCALL(pread64)(long res, long fd, void *buf, long count, long pos) {
if (res >= 0) {
- if (buf) POST_WRITE(buf, res);
+ if (buf)
+ POST_WRITE(buf, res);
}
}
PRE_SYSCALL(pwrite64)(long fd, const void *buf, long count, long pos) {
- if (buf) PRE_READ(buf, count);
+ if (buf)
+ PRE_READ(buf, count);
}
-POST_SYSCALL(pwrite64)(long res, long fd, const void *buf, long count,
- long pos) {}
-#else
+POST_SYSCALL(pwrite64)
+(long res, long fd, const void *buf, long count, long pos) {}
+# else
PRE_SYSCALL(pread64)(long fd, void *buf, long count, long pos0, long pos1) {}
-POST_SYSCALL(pread64)(long res, long fd, void *buf, long count, long pos0,
- long pos1) {
+POST_SYSCALL(pread64)
+(long res, long fd, void *buf, long count, long pos0, long pos1) {
if (res >= 0) {
- if (buf) POST_WRITE(buf, res);
+ if (buf)
+ POST_WRITE(buf, res);
}
}
-PRE_SYSCALL(pwrite64)(long fd, const void *buf, long count, long pos0,
- long pos1) {
- if (buf) PRE_READ(buf, count);
+PRE_SYSCALL(pwrite64)
+(long fd, const void *buf, long count, long pos0, long pos1) {
+ if (buf)
+ PRE_READ(buf, count);
}
-POST_SYSCALL(pwrite64)(long res, long fd, const void *buf, long count,
- long pos0, long pos1) {}
-#endif
+POST_SYSCALL(pwrite64)
+(long res, long fd, const void *buf, long count, long pos0, long pos1) {}
+# endif
-PRE_SYSCALL(preadv)(long fd, const __sanitizer_iovec *vec, long vlen,
- long pos_l, long pos_h) {}
+PRE_SYSCALL(preadv)
+(long fd, const __sanitizer_iovec *vec, long vlen, long pos_l, long pos_h) {}
-POST_SYSCALL(preadv)(long res, long fd, const __sanitizer_iovec *vec, long vlen,
- long pos_l, long pos_h) {
+POST_SYSCALL(preadv)
+(long res, long fd, const __sanitizer_iovec *vec, long vlen, long pos_l,
+ long pos_h) {
if (res >= 0) {
- if (vec) kernel_write_iovec(vec, vlen, res);
+ if (vec)
+ kernel_write_iovec(vec, vlen, res);
}
}
-PRE_SYSCALL(pwritev)(long fd, const __sanitizer_iovec *vec, long vlen,
- long pos_l, long pos_h) {}
+PRE_SYSCALL(pwritev)
+(long fd, const __sanitizer_iovec *vec, long vlen, long pos_l, long pos_h) {}
-POST_SYSCALL(pwritev)(long res, long fd, const __sanitizer_iovec *vec,
- long vlen, long pos_l, long pos_h) {
+POST_SYSCALL(pwritev)
+(long res, long fd, const __sanitizer_iovec *vec, long vlen, long pos_l,
+ long pos_h) {
if (res >= 0) {
- if (vec) kernel_read_iovec(vec, vlen, res);
+ if (vec)
+ kernel_read_iovec(vec, vlen, res);
}
}
PRE_READ(special, __sanitizer::internal_strlen((const char *)special) + 1);
}
-POST_SYSCALL(quotactl)(long res, long cmd, const void *special, long id,
- void *addr) {}
+POST_SYSCALL(quotactl)
+(long res, long cmd, const void *special, long id, void *addr) {}
PRE_SYSCALL(getdents)(long fd, void *dirent, long count) {}
POST_SYSCALL(getdents)(long res, long fd, void *dirent, long count) {
if (res >= 0) {
- if (dirent) POST_WRITE(dirent, res);
+ if (dirent)
+ POST_WRITE(dirent, res);
}
}
POST_SYSCALL(getdents64)(long res, long fd, void *dirent, long count) {
if (res >= 0) {
- if (dirent) POST_WRITE(dirent, res);
+ if (dirent)
+ POST_WRITE(dirent, res);
}
}
-PRE_SYSCALL(setsockopt)(long fd, long level, long optname, void *optval,
- long optlen) {}
+PRE_SYSCALL(setsockopt)
+(long fd, long level, long optname, void *optval, long optlen) {}
-POST_SYSCALL(setsockopt)(long res, long fd, long level, long optname,
- void *optval, long optlen) {
+POST_SYSCALL(setsockopt)
+(long res, long fd, long level, long optname, void *optval, long optlen) {
if (res >= 0) {
if (optval)
POST_WRITE(optval,
}
}
-PRE_SYSCALL(getsockopt)(long fd, long level, long optname, void *optval,
- void *optlen) {}
+PRE_SYSCALL(getsockopt)
+(long fd, long level, long optname, void *optval, void *optlen) {}
-POST_SYSCALL(getsockopt)(long res, long fd, long level, long optname,
- void *optval, void *optlen) {
+POST_SYSCALL(getsockopt)
+(long res, long fd, long level, long optname, void *optval, void *optlen) {
if (res >= 0) {
if (optval)
POST_WRITE(optval,
__sanitizer::internal_strlen((const char *)optval) + 1);
- if (optlen) POST_WRITE(optlen, sizeof(int));
+ if (optlen)
+ POST_WRITE(optlen, sizeof(int));
}
}
PRE_SYSCALL(bind)(long arg0, sanitizer_kernel_sockaddr *arg1, long arg2) {}
-POST_SYSCALL(bind)(long res, long arg0, sanitizer_kernel_sockaddr *arg1,
- long arg2) {
+POST_SYSCALL(bind)
+(long res, long arg0, sanitizer_kernel_sockaddr *arg1, long arg2) {
if (res >= 0) {
- if (arg1) POST_WRITE(arg1, sizeof(*arg1));
+ if (arg1)
+ POST_WRITE(arg1, sizeof(*arg1));
}
}
PRE_SYSCALL(connect)(long arg0, sanitizer_kernel_sockaddr *arg1, long arg2) {}
-POST_SYSCALL(connect)(long res, long arg0, sanitizer_kernel_sockaddr *arg1,
- long arg2) {
+POST_SYSCALL(connect)
+(long res, long arg0, sanitizer_kernel_sockaddr *arg1, long arg2) {
if (res >= 0) {
- if (arg1) POST_WRITE(arg1, sizeof(*arg1));
+ if (arg1)
+ POST_WRITE(arg1, sizeof(*arg1));
}
}
PRE_SYSCALL(accept)(long arg0, sanitizer_kernel_sockaddr *arg1, void *arg2) {}
-POST_SYSCALL(accept)(long res, long arg0, sanitizer_kernel_sockaddr *arg1,
- void *arg2) {
+POST_SYSCALL(accept)
+(long res, long arg0, sanitizer_kernel_sockaddr *arg1, void *arg2) {
if (res >= 0) {
- if (arg1) POST_WRITE(arg1, sizeof(*arg1));
- if (arg2) POST_WRITE(arg2, sizeof(unsigned));
+ if (arg1)
+ POST_WRITE(arg1, sizeof(*arg1));
+ if (arg2)
+ POST_WRITE(arg2, sizeof(unsigned));
}
}
-PRE_SYSCALL(accept4)(long arg0, sanitizer_kernel_sockaddr *arg1, void *arg2,
- long arg3) {}
+PRE_SYSCALL(accept4)
+(long arg0, sanitizer_kernel_sockaddr *arg1, void *arg2, long arg3) {}
-POST_SYSCALL(accept4)(long res, long arg0, sanitizer_kernel_sockaddr *arg1,
- void *arg2, long arg3) {
+POST_SYSCALL(accept4)
+(long res, long arg0, sanitizer_kernel_sockaddr *arg1, void *arg2, long arg3) {
if (res >= 0) {
- if (arg1) POST_WRITE(arg1, sizeof(*arg1));
- if (arg2) POST_WRITE(arg2, sizeof(unsigned));
+ if (arg1)
+ POST_WRITE(arg1, sizeof(*arg1));
+ if (arg2)
+ POST_WRITE(arg2, sizeof(unsigned));
}
}
-PRE_SYSCALL(getsockname)(long arg0, sanitizer_kernel_sockaddr *arg1,
- void *arg2) {}
+PRE_SYSCALL(getsockname)
+(long arg0, sanitizer_kernel_sockaddr *arg1, void *arg2) {}
-POST_SYSCALL(getsockname)(long res, long arg0, sanitizer_kernel_sockaddr *arg1,
- void *arg2) {
+POST_SYSCALL(getsockname)
+(long res, long arg0, sanitizer_kernel_sockaddr *arg1, void *arg2) {
if (res >= 0) {
- if (arg1) POST_WRITE(arg1, sizeof(*arg1));
- if (arg2) POST_WRITE(arg2, sizeof(unsigned));
+ if (arg1)
+ POST_WRITE(arg1, sizeof(*arg1));
+ if (arg2)
+ POST_WRITE(arg2, sizeof(unsigned));
}
}
-PRE_SYSCALL(getpeername)(long arg0, sanitizer_kernel_sockaddr *arg1,
- void *arg2) {}
+PRE_SYSCALL(getpeername)
+(long arg0, sanitizer_kernel_sockaddr *arg1, void *arg2) {}
-POST_SYSCALL(getpeername)(long res, long arg0, sanitizer_kernel_sockaddr *arg1,
- void *arg2) {
+POST_SYSCALL(getpeername)
+(long res, long arg0, sanitizer_kernel_sockaddr *arg1, void *arg2) {
if (res >= 0) {
- if (arg1) POST_WRITE(arg1, sizeof(*arg1));
- if (arg2) POST_WRITE(arg2, sizeof(unsigned));
+ if (arg1)
+ POST_WRITE(arg1, sizeof(*arg1));
+ if (arg2)
+ POST_WRITE(arg2, sizeof(unsigned));
}
}
POST_SYSCALL(send)(long res, long arg0, void *arg1, long arg2, long arg3) {
if (res) {
- if (arg1) POST_READ(arg1, res);
+ if (arg1)
+ POST_READ(arg1, res);
}
}
-PRE_SYSCALL(sendto)(long arg0, void *arg1, long arg2, long arg3,
- sanitizer_kernel_sockaddr *arg4, long arg5) {}
+PRE_SYSCALL(sendto)
+(long arg0, void *arg1, long arg2, long arg3, sanitizer_kernel_sockaddr *arg4,
+ long arg5) {}
-POST_SYSCALL(sendto)(long res, long arg0, void *arg1, long arg2, long arg3,
- sanitizer_kernel_sockaddr *arg4, long arg5) {
+POST_SYSCALL(sendto)
+(long res, long arg0, void *arg1, long arg2, long arg3,
+ sanitizer_kernel_sockaddr *arg4, long arg5) {
if (res >= 0) {
- if (arg1) POST_READ(arg1, res);
- if (arg4) POST_WRITE(arg4, sizeof(*arg4));
+ if (arg1)
+ POST_READ(arg1, res);
+ if (arg4)
+ POST_WRITE(arg4, sizeof(*arg4));
}
}
POST_SYSCALL(recv)(long res, void *buf, long len, long flags) {
if (res >= 0) {
- if (buf) POST_WRITE(buf, res);
+ if (buf)
+ POST_WRITE(buf, res);
}
}
-PRE_SYSCALL(recvfrom)(long arg0, void *buf, long len, long flags,
- sanitizer_kernel_sockaddr *arg4, void *arg5) {}
+PRE_SYSCALL(recvfrom)
+(long arg0, void *buf, long len, long flags, sanitizer_kernel_sockaddr *arg4,
+ void *arg5) {}
-POST_SYSCALL(recvfrom)(long res, long arg0, void *buf, long len, long flags,
- sanitizer_kernel_sockaddr *arg4, void *arg5) {
+POST_SYSCALL(recvfrom)
+(long res, long arg0, void *buf, long len, long flags,
+ sanitizer_kernel_sockaddr *arg4, void *arg5) {
if (res >= 0) {
- if (buf) POST_WRITE(buf, res);
- if (arg4) POST_WRITE(arg4, sizeof(*arg4));
- if (arg5) POST_WRITE(arg5, sizeof(int));
+ if (buf)
+ POST_WRITE(buf, res);
+ if (arg4)
+ POST_WRITE(arg4, sizeof(*arg4));
+ if (arg5)
+ POST_WRITE(arg5, sizeof(int));
}
}
POST_SYSCALL(socketpair)(long res, long arg0, long arg1, long arg2, int *sv) {
if (res >= 0)
- if (sv) POST_WRITE(sv, sizeof(int) * 2);
+ if (sv)
+ POST_WRITE(sv, sizeof(int) * 2);
}
PRE_SYSCALL(socketcall)(long call, void *args) {}
POST_SYSCALL(socketcall)(long res, long call, void *args) {
if (res >= 0) {
- if (args) POST_WRITE(args, sizeof(long));
+ if (args)
+ POST_WRITE(args, sizeof(long));
}
}
PRE_SYSCALL(poll)(void *ufds, long nfds, long timeout) {}
-POST_SYSCALL(poll)(long res, __sanitizer_pollfd *ufds, long nfds,
- long timeout) {
+POST_SYSCALL(poll)
+(long res, __sanitizer_pollfd *ufds, long nfds, long timeout) {
if (res >= 0) {
- if (ufds) POST_WRITE(ufds, nfds * sizeof(*ufds));
+ if (ufds)
+ POST_WRITE(ufds, nfds * sizeof(*ufds));
}
}
-PRE_SYSCALL(select)(long n, __sanitizer___kernel_fd_set *inp,
- __sanitizer___kernel_fd_set *outp,
- __sanitizer___kernel_fd_set *exp, void *tvp) {}
+PRE_SYSCALL(select)
+(long n, __sanitizer___kernel_fd_set *inp, __sanitizer___kernel_fd_set *outp,
+ __sanitizer___kernel_fd_set *exp, void *tvp) {}
-POST_SYSCALL(select)(long res, long n, __sanitizer___kernel_fd_set *inp,
- __sanitizer___kernel_fd_set *outp,
- __sanitizer___kernel_fd_set *exp, void *tvp) {
+POST_SYSCALL(select)
+(long res, long n, __sanitizer___kernel_fd_set *inp,
+ __sanitizer___kernel_fd_set *outp, __sanitizer___kernel_fd_set *exp,
+ void *tvp) {
if (res >= 0) {
- if (inp) POST_WRITE(inp, sizeof(*inp));
- if (outp) POST_WRITE(outp, sizeof(*outp));
- if (exp) POST_WRITE(exp, sizeof(*exp));
- if (tvp) POST_WRITE(tvp, timeval_sz);
+ if (inp)
+ POST_WRITE(inp, sizeof(*inp));
+ if (outp)
+ POST_WRITE(outp, sizeof(*outp));
+ if (exp)
+ POST_WRITE(exp, sizeof(*exp));
+ if (tvp)
+ POST_WRITE(tvp, timeval_sz);
}
}
POST_SYSCALL(epoll_ctl)(long res, long epfd, long op, long fd, void *event) {
if (res >= 0) {
- if (event) POST_WRITE(event, struct_epoll_event_sz);
+ if (event)
+ POST_WRITE(event, struct_epoll_event_sz);
+ }
+}
+
+PRE_SYSCALL(epoll_wait)
+(long epfd, void *events, long maxevents, long timeout) {}
+
+POST_SYSCALL(epoll_wait)
+(long res, long epfd, void *events, long maxevents, long timeout) {
+ if (res >= 0) {
+ COMMON_SYSCALL_FD_ACQUIRE(epfd);
+ if (events)
+ POST_WRITE(events, res * struct_epoll_event_sz);
}
}
-PRE_SYSCALL(epoll_wait)(long epfd, void *events, long maxevents, long timeout) {
+PRE_SYSCALL(epoll_pwait)
+(long epfd, void *events, long maxevents, long timeout,
+ const kernel_sigset_t *sigmask, long sigsetsize) {
+ if (sigmask)
+ PRE_READ(sigmask, sigsetsize);
}
-POST_SYSCALL(epoll_wait)(long res, long epfd, void *events, long maxevents,
- long timeout) {
+POST_SYSCALL(epoll_pwait)
+(long res, long epfd, void *events, long maxevents, long timeout,
+ const void *sigmask, long sigsetsize) {
if (res >= 0) {
- if (events) POST_WRITE(events, struct_epoll_event_sz);
+ COMMON_SYSCALL_FD_ACQUIRE(epfd);
+ if (events)
+ POST_WRITE(events, res * struct_epoll_event_sz);
}
}
-PRE_SYSCALL(epoll_pwait)(long epfd, void *events, long maxevents, long timeout,
- const kernel_sigset_t *sigmask, long sigsetsize) {
- if (sigmask) PRE_READ(sigmask, sigsetsize);
+PRE_SYSCALL(epoll_pwait2)
+(long epfd, void *events, long maxevents,
+ const sanitizer_kernel_timespec *timeout, const kernel_sigset_t *sigmask,
+ long sigsetsize) {
+ if (timeout)
+ PRE_READ(timeout, sizeof(timeout));
+ if (sigmask)
+ PRE_READ(sigmask, sigsetsize);
}
-POST_SYSCALL(epoll_pwait)(long res, long epfd, void *events, long maxevents,
- long timeout, const void *sigmask, long sigsetsize) {
+POST_SYSCALL(epoll_pwait2)
+(long res, long epfd, void *events, long maxevents,
+ const sanitizer_kernel_timespec *timeout, const void *sigmask,
+ long sigsetsize) {
if (res >= 0) {
- if (events) POST_WRITE(events, struct_epoll_event_sz);
+ COMMON_SYSCALL_FD_ACQUIRE(epfd);
+ if (events)
+ POST_WRITE(events, res * struct_epoll_event_sz);
}
}
POST_SYSCALL(newuname)(long res, void *name) {
if (res >= 0) {
- if (name) POST_WRITE(name, struct_new_utsname_sz);
+ if (name)
+ POST_WRITE(name, struct_new_utsname_sz);
}
}
POST_SYSCALL(uname)(long res, void *arg0) {
if (res >= 0) {
- if (arg0) POST_WRITE(arg0, struct_old_utsname_sz);
+ if (arg0)
+ POST_WRITE(arg0, struct_old_utsname_sz);
}
}
POST_SYSCALL(olduname)(long res, void *arg0) {
if (res >= 0) {
- if (arg0) POST_WRITE(arg0, struct_oldold_utsname_sz);
+ if (arg0)
+ POST_WRITE(arg0, struct_oldold_utsname_sz);
}
}
POST_SYSCALL(getrlimit)(long res, long resource, void *rlim) {
if (res >= 0) {
- if (rlim) POST_WRITE(rlim, struct_rlimit_sz);
+ if (rlim)
+ POST_WRITE(rlim, struct_rlimit_sz);
}
}
POST_SYSCALL(old_getrlimit)(long res, long resource, void *rlim) {
if (res >= 0) {
- if (rlim) POST_WRITE(rlim, struct_rlimit_sz);
+ if (rlim)
+ POST_WRITE(rlim, struct_rlimit_sz);
}
}
POST_SYSCALL(setrlimit)(long res, long resource, void *rlim) {
if (res >= 0) {
- if (rlim) POST_WRITE(rlim, struct_rlimit_sz);
+ if (rlim)
+ POST_WRITE(rlim, struct_rlimit_sz);
}
}
-#if !SANITIZER_ANDROID
-PRE_SYSCALL(prlimit64)(long pid, long resource, const void *new_rlim,
- void *old_rlim) {
- if (new_rlim) PRE_READ(new_rlim, struct_rlimit64_sz);
+# if SANITIZER_GLIBC
+PRE_SYSCALL(prlimit64)
+(long pid, long resource, const void *new_rlim, void *old_rlim) {
+ if (new_rlim)
+ PRE_READ(new_rlim, struct_rlimit64_sz);
}
-POST_SYSCALL(prlimit64)(long res, long pid, long resource, const void *new_rlim,
- void *old_rlim) {
+POST_SYSCALL(prlimit64)
+(long res, long pid, long resource, const void *new_rlim, void *old_rlim) {
if (res >= 0) {
- if (old_rlim) POST_WRITE(old_rlim, struct_rlimit64_sz);
+ if (old_rlim)
+ POST_WRITE(old_rlim, struct_rlimit64_sz);
}
}
-#endif
+# endif
PRE_SYSCALL(getrusage)(long who, void *ru) {}
POST_SYSCALL(getrusage)(long res, long who, void *ru) {
if (res >= 0) {
- if (ru) POST_WRITE(ru, struct_rusage_sz);
+ if (ru)
+ POST_WRITE(ru, struct_rusage_sz);
}
}
POST_SYSCALL(msgget)(long res, long key, long msgflg) {}
PRE_SYSCALL(msgsnd)(long msqid, void *msgp, long msgsz, long msgflg) {
- if (msgp) PRE_READ(msgp, msgsz);
+ if (msgp)
+ PRE_READ(msgp, msgsz);
}
-POST_SYSCALL(msgsnd)(long res, long msqid, void *msgp, long msgsz,
- long msgflg) {}
+POST_SYSCALL(msgsnd)
+(long res, long msqid, void *msgp, long msgsz, long msgflg) {}
-PRE_SYSCALL(msgrcv)(long msqid, void *msgp, long msgsz, long msgtyp,
- long msgflg) {}
+PRE_SYSCALL(msgrcv)
+(long msqid, void *msgp, long msgsz, long msgtyp, long msgflg) {}
-POST_SYSCALL(msgrcv)(long res, long msqid, void *msgp, long msgsz, long msgtyp,
- long msgflg) {
+POST_SYSCALL(msgrcv)
+(long res, long msqid, void *msgp, long msgsz, long msgtyp, long msgflg) {
if (res >= 0) {
- if (msgp) POST_WRITE(msgp, res);
+ if (msgp)
+ POST_WRITE(msgp, res);
}
}
-#if !SANITIZER_ANDROID
+# if !SANITIZER_ANDROID
PRE_SYSCALL(msgctl)(long msqid, long cmd, void *buf) {}
POST_SYSCALL(msgctl)(long res, long msqid, long cmd, void *buf) {
if (res >= 0) {
- if (buf) POST_WRITE(buf, struct_msqid_ds_sz);
+ if (buf)
+ POST_WRITE(buf, struct_msqid_ds_sz);
}
}
-#endif
+# endif
PRE_SYSCALL(semget)(long key, long nsems, long semflg) {}
POST_SYSCALL(semctl)(long res, long semid, long semnum, long cmd, void *arg) {}
-PRE_SYSCALL(semtimedop)(long semid, void *sops, long nsops,
- const void *timeout) {
- if (timeout) PRE_READ(timeout, struct_timespec_sz);
+PRE_SYSCALL(semtimedop)
+(long semid, void *sops, long nsops, const void *timeout) {
+ if (timeout)
+ PRE_READ(timeout, struct_timespec_sz);
}
-POST_SYSCALL(semtimedop)(long res, long semid, void *sops, long nsops,
- const void *timeout) {}
+POST_SYSCALL(semtimedop)
+(long res, long semid, void *sops, long nsops, const void *timeout) {}
PRE_SYSCALL(shmat)(long shmid, void *shmaddr, long shmflg) {}
}
}
-PRE_SYSCALL(ipc)(long call, long first, long second, long third, void *ptr,
- long fifth) {}
+PRE_SYSCALL(ipc)
+(long call, long first, long second, long third, void *ptr, long fifth) {}
-POST_SYSCALL(ipc)(long res, long call, long first, long second, long third,
- void *ptr, long fifth) {}
+POST_SYSCALL(ipc)
+(long res, long call, long first, long second, long third, void *ptr,
+ long fifth) {}
-#if !SANITIZER_ANDROID
+# if !SANITIZER_ANDROID
PRE_SYSCALL(shmctl)(long shmid, long cmd, void *buf) {}
POST_SYSCALL(shmctl)(long res, long shmid, long cmd, void *buf) {
if (res >= 0) {
- if (buf) POST_WRITE(buf, sizeof(__sanitizer_shmid_ds));
+ if (buf)
+ POST_WRITE(buf, sizeof(__sanitizer_shmid_ds));
}
}
PRE_READ(name, __sanitizer::internal_strlen((const char *)name) + 1);
}
-POST_SYSCALL(mq_open)(long res, const void *name, long oflag, long mode,
- void *attr) {
+POST_SYSCALL(mq_open)
+(long res, const void *name, long oflag, long mode, void *attr) {
if (res >= 0) {
- if (attr) POST_WRITE(attr, struct_mq_attr_sz);
+ if (attr)
+ POST_WRITE(attr, struct_mq_attr_sz);
}
}
POST_SYSCALL(mq_unlink)(long res, const void *name) {}
-PRE_SYSCALL(mq_timedsend)(long mqdes, const void *msg_ptr, long msg_len,
- long msg_prio, const void *abs_timeout) {
- if (msg_ptr) PRE_READ(msg_ptr, msg_len);
- if (abs_timeout) PRE_READ(abs_timeout, struct_timespec_sz);
+PRE_SYSCALL(mq_timedsend)
+(long mqdes, const void *msg_ptr, long msg_len, long msg_prio,
+ const void *abs_timeout) {
+ if (msg_ptr)
+ PRE_READ(msg_ptr, msg_len);
+ if (abs_timeout)
+ PRE_READ(abs_timeout, struct_timespec_sz);
}
-POST_SYSCALL(mq_timedsend)(long res, long mqdes, const void *msg_ptr,
- long msg_len, long msg_prio,
- const void *abs_timeout) {}
+POST_SYSCALL(mq_timedsend)
+(long res, long mqdes, const void *msg_ptr, long msg_len, long msg_prio,
+ const void *abs_timeout) {}
-PRE_SYSCALL(mq_timedreceive)(long mqdes, void *msg_ptr, long msg_len,
- void *msg_prio, const void *abs_timeout) {
- if (abs_timeout) PRE_READ(abs_timeout, struct_timespec_sz);
+PRE_SYSCALL(mq_timedreceive)
+(long mqdes, void *msg_ptr, long msg_len, void *msg_prio,
+ const void *abs_timeout) {
+ if (abs_timeout)
+ PRE_READ(abs_timeout, struct_timespec_sz);
}
-POST_SYSCALL(mq_timedreceive)(long res, long mqdes, void *msg_ptr, long msg_len,
- int *msg_prio, const void *abs_timeout) {
+POST_SYSCALL(mq_timedreceive)
+(long res, long mqdes, void *msg_ptr, long msg_len, int *msg_prio,
+ const void *abs_timeout) {
if (res >= 0) {
- if (msg_ptr) POST_WRITE(msg_ptr, res);
- if (msg_prio) POST_WRITE(msg_prio, sizeof(*msg_prio));
+ if (msg_ptr)
+ POST_WRITE(msg_ptr, res);
+ if (msg_prio)
+ POST_WRITE(msg_prio, sizeof(*msg_prio));
}
}
PRE_SYSCALL(mq_notify)(long mqdes, const void *notification) {
- if (notification) PRE_READ(notification, struct_sigevent_sz);
+ if (notification)
+ PRE_READ(notification, struct_sigevent_sz);
}
POST_SYSCALL(mq_notify)(long res, long mqdes, const void *notification) {}
PRE_SYSCALL(mq_getsetattr)(long mqdes, const void *mqstat, void *omqstat) {
- if (mqstat) PRE_READ(mqstat, struct_mq_attr_sz);
+ if (mqstat)
+ PRE_READ(mqstat, struct_mq_attr_sz);
}
-POST_SYSCALL(mq_getsetattr)(long res, long mqdes, const void *mqstat,
- void *omqstat) {
+POST_SYSCALL(mq_getsetattr)
+(long res, long mqdes, const void *mqstat, void *omqstat) {
if (res >= 0) {
- if (omqstat) POST_WRITE(omqstat, struct_mq_attr_sz);
+ if (omqstat)
+ POST_WRITE(omqstat, struct_mq_attr_sz);
}
}
-#endif // SANITIZER_ANDROID
+# endif // SANITIZER_ANDROID
PRE_SYSCALL(pciconfig_iobase)(long which, long bus, long devfn) {}
POST_SYSCALL(pciconfig_iobase)(long res, long which, long bus, long devfn) {}
-PRE_SYSCALL(pciconfig_read)(long bus, long dfn, long off, long len, void *buf) {
-}
+PRE_SYSCALL(pciconfig_read)
+(long bus, long dfn, long off, long len, void *buf) {}
-POST_SYSCALL(pciconfig_read)(long res, long bus, long dfn, long off, long len,
- void *buf) {}
+POST_SYSCALL(pciconfig_read)
+(long res, long bus, long dfn, long off, long len, void *buf) {}
-PRE_SYSCALL(pciconfig_write)(long bus, long dfn, long off, long len,
- void *buf) {}
+PRE_SYSCALL(pciconfig_write)
+(long bus, long dfn, long off, long len, void *buf) {}
-POST_SYSCALL(pciconfig_write)(long res, long bus, long dfn, long off, long len,
- void *buf) {}
+POST_SYSCALL(pciconfig_write)
+(long res, long bus, long dfn, long off, long len, void *buf) {}
PRE_SYSCALL(swapon)(const void *specialfile, long swap_flags) {
if (specialfile)
PRE_SYSCALL(sysctl)(__sanitizer___sysctl_args *args) {
if (args) {
- if (args->name) PRE_READ(args->name, args->nlen * sizeof(*args->name));
- if (args->newval) PRE_READ(args->name, args->newlen);
+ if (args->name)
+ PRE_READ(args->name, args->nlen * sizeof(*args->name));
+ if (args->newval)
+ PRE_READ(args->name, args->newlen);
}
}
POST_SYSCALL(sysinfo)(long res, void *info) {
if (res >= 0) {
- if (info) POST_WRITE(info, struct_sysinfo_sz);
+ if (info)
+ POST_WRITE(info, struct_sysinfo_sz);
}
}
POST_SYSCALL(ni_syscall)(long res) {}
PRE_SYSCALL(ptrace)(long request, long pid, long addr, long data) {
-#if !SANITIZER_ANDROID && \
- (defined(__i386) || defined(__x86_64) || defined(__mips64) || \
- defined(__powerpc64__) || defined(__aarch64__) || defined(__s390__) || \
- SANITIZER_RISCV64)
+# if !SANITIZER_ANDROID && \
+ (defined(__i386) || defined(__x86_64) || defined(__mips64) || \
+ defined(__powerpc64__) || defined(__aarch64__) || defined(__s390__) || \
+ defined(__loongarch__) || SANITIZER_RISCV64)
if (data) {
if (request == ptrace_setregs) {
PRE_READ((void *)data, struct_user_regs_struct_sz);
PRE_READ(iov->iov_base, iov->iov_len);
}
}
-#endif
+# endif
}
POST_SYSCALL(ptrace)(long res, long request, long pid, long addr, long data) {
-#if !SANITIZER_ANDROID && \
- (defined(__i386) || defined(__x86_64) || defined(__mips64) || \
- defined(__powerpc64__) || defined(__aarch64__) || defined(__s390__) || \
- SANITIZER_RISCV64)
+# if !SANITIZER_ANDROID && \
+ (defined(__i386) || defined(__x86_64) || defined(__mips64) || \
+ defined(__powerpc64__) || defined(__aarch64__) || defined(__s390__) || \
+ defined(__loongarch__) || SANITIZER_RISCV64)
if (res >= 0 && data) {
// Note that this is different from the interceptor in
// sanitizer_common_interceptors.inc.
POST_WRITE((void *)data, sizeof(void *));
}
}
-#endif
+# endif
}
-PRE_SYSCALL(add_key)(const void *_type, const void *_description,
- const void *_payload, long plen, long destringid) {
+PRE_SYSCALL(add_key)
+(const void *_type, const void *_description, const void *_payload, long plen,
+ long destringid) {
if (_type)
PRE_READ(_type, __sanitizer::internal_strlen((const char *)_type) + 1);
if (_description)
__sanitizer::internal_strlen((const char *)_description) + 1);
}
-POST_SYSCALL(add_key)(long res, const void *_type, const void *_description,
- const void *_payload, long plen, long destringid) {}
+POST_SYSCALL(add_key)
+(long res, const void *_type, const void *_description, const void *_payload,
+ long plen, long destringid) {}
-PRE_SYSCALL(request_key)(const void *_type, const void *_description,
- const void *_callout_info, long destringid) {
+PRE_SYSCALL(request_key)
+(const void *_type, const void *_description, const void *_callout_info,
+ long destringid) {
if (_type)
PRE_READ(_type, __sanitizer::internal_strlen((const char *)_type) + 1);
if (_description)
__sanitizer::internal_strlen((const char *)_callout_info) + 1);
}
-POST_SYSCALL(request_key)(long res, const void *_type, const void *_description,
- const void *_callout_info, long destringid) {}
+POST_SYSCALL(request_key)
+(long res, const void *_type, const void *_description,
+ const void *_callout_info, long destringid) {}
PRE_SYSCALL(keyctl)(long cmd, long arg2, long arg3, long arg4, long arg5) {}
-POST_SYSCALL(keyctl)(long res, long cmd, long arg2, long arg3, long arg4,
- long arg5) {}
+POST_SYSCALL(keyctl)
+(long res, long cmd, long arg2, long arg3, long arg4, long arg5) {}
PRE_SYSCALL(ioprio_set)(long which, long who, long ioprio) {}
POST_SYSCALL(set_mempolicy)(long res, long mode, void *nmask, long maxnode) {
if (res >= 0) {
- if (nmask) POST_WRITE(nmask, sizeof(long));
+ if (nmask)
+ POST_WRITE(nmask, sizeof(long));
}
}
-PRE_SYSCALL(migrate_pages)(long pid, long maxnode, const void *from,
- const void *to) {
- if (from) PRE_READ(from, sizeof(long));
- if (to) PRE_READ(to, sizeof(long));
+PRE_SYSCALL(migrate_pages)
+(long pid, long maxnode, const void *from, const void *to) {
+ if (from)
+ PRE_READ(from, sizeof(long));
+ if (to)
+ PRE_READ(to, sizeof(long));
}
-POST_SYSCALL(migrate_pages)(long res, long pid, long maxnode, const void *from,
- const void *to) {}
+POST_SYSCALL(migrate_pages)
+(long res, long pid, long maxnode, const void *from, const void *to) {}
-PRE_SYSCALL(move_pages)(long pid, long nr_pages, const void **pages,
- const int *nodes, int *status, long flags) {
- if (pages) PRE_READ(pages, nr_pages * sizeof(*pages));
- if (nodes) PRE_READ(nodes, nr_pages * sizeof(*nodes));
+PRE_SYSCALL(move_pages)
+(long pid, long nr_pages, const void **pages, const int *nodes, int *status,
+ long flags) {
+ if (pages)
+ PRE_READ(pages, nr_pages * sizeof(*pages));
+ if (nodes)
+ PRE_READ(nodes, nr_pages * sizeof(*nodes));
}
-POST_SYSCALL(move_pages)(long res, long pid, long nr_pages, const void **pages,
- const int *nodes, int *status, long flags) {
+POST_SYSCALL(move_pages)
+(long res, long pid, long nr_pages, const void **pages, const int *nodes,
+ int *status, long flags) {
if (res >= 0) {
- if (status) POST_WRITE(status, nr_pages * sizeof(*status));
+ if (status)
+ POST_WRITE(status, nr_pages * sizeof(*status));
}
}
-PRE_SYSCALL(mbind)(long start, long len, long mode, void *nmask, long maxnode,
- long flags) {}
+PRE_SYSCALL(mbind)
+(long start, long len, long mode, void *nmask, long maxnode, long flags) {}
-POST_SYSCALL(mbind)(long res, long start, long len, long mode, void *nmask,
- long maxnode, long flags) {
+POST_SYSCALL(mbind)
+(long res, long start, long len, long mode, void *nmask, long maxnode,
+ long flags) {
if (res >= 0) {
- if (nmask) POST_WRITE(nmask, sizeof(long));
+ if (nmask)
+ POST_WRITE(nmask, sizeof(long));
}
}
-PRE_SYSCALL(get_mempolicy)(void *policy, void *nmask, long maxnode, long addr,
- long flags) {}
+PRE_SYSCALL(get_mempolicy)
+(void *policy, void *nmask, long maxnode, long addr, long flags) {}
-POST_SYSCALL(get_mempolicy)(long res, void *policy, void *nmask, long maxnode,
- long addr, long flags) {
+POST_SYSCALL(get_mempolicy)
+(long res, void *policy, void *nmask, long maxnode, long addr, long flags) {
if (res >= 0) {
- if (policy) POST_WRITE(policy, sizeof(int));
- if (nmask) POST_WRITE(nmask, sizeof(long));
+ if (policy)
+ POST_WRITE(policy, sizeof(int));
+ if (nmask)
+ POST_WRITE(nmask, sizeof(long));
}
}
PRE_READ(path, __sanitizer::internal_strlen((const char *)path) + 1);
}
-POST_SYSCALL(inotify_add_watch)(long res, long fd, const void *path,
- long mask) {}
+POST_SYSCALL(inotify_add_watch)
+(long res, long fd, const void *path, long mask) {}
PRE_SYSCALL(inotify_rm_watch)(long fd, long wd) {}
POST_SYSCALL(spu_run)(long res, long fd, unsigned *unpc, unsigned *ustatus) {
if (res >= 0) {
- if (unpc) POST_WRITE(unpc, sizeof(*unpc));
- if (ustatus) POST_WRITE(ustatus, sizeof(*ustatus));
+ if (unpc)
+ POST_WRITE(unpc, sizeof(*unpc));
+ if (ustatus)
+ POST_WRITE(ustatus, sizeof(*ustatus));
}
}
PRE_READ(name, __sanitizer::internal_strlen((const char *)name) + 1);
}
-POST_SYSCALL(spu_create)(long res, const void *name, long flags, long mode,
- long fd) {}
+POST_SYSCALL(spu_create)
+(long res, const void *name, long flags, long mode, long fd) {}
PRE_SYSCALL(mknodat)(long dfd, const void *filename, long mode, long dev) {
if (filename)
__sanitizer::internal_strlen((const char *)filename) + 1);
}
-POST_SYSCALL(mknodat)(long res, long dfd, const void *filename, long mode,
- long dev) {}
+POST_SYSCALL(mknodat)
+(long res, long dfd, const void *filename, long mode, long dev) {}
PRE_SYSCALL(mkdirat)(long dfd, const void *pathname, long mode) {
if (pathname)
PRE_READ(newname, __sanitizer::internal_strlen((const char *)newname) + 1);
}
-POST_SYSCALL(symlinkat)(long res, const void *oldname, long newdfd,
- const void *newname) {}
+POST_SYSCALL(symlinkat)
+(long res, const void *oldname, long newdfd, const void *newname) {}
-PRE_SYSCALL(linkat)(long olddfd, const void *oldname, long newdfd,
- const void *newname, long flags) {
+PRE_SYSCALL(linkat)
+(long olddfd, const void *oldname, long newdfd, const void *newname,
+ long flags) {
if (oldname)
PRE_READ(oldname, __sanitizer::internal_strlen((const char *)oldname) + 1);
if (newname)
PRE_READ(newname, __sanitizer::internal_strlen((const char *)newname) + 1);
}
-POST_SYSCALL(linkat)(long res, long olddfd, const void *oldname, long newdfd,
- const void *newname, long flags) {}
+POST_SYSCALL(linkat)
+(long res, long olddfd, const void *oldname, long newdfd, const void *newname,
+ long flags) {}
-PRE_SYSCALL(renameat)(long olddfd, const void *oldname, long newdfd,
- const void *newname) {
+PRE_SYSCALL(renameat)
+(long olddfd, const void *oldname, long newdfd, const void *newname) {
if (oldname)
PRE_READ(oldname, __sanitizer::internal_strlen((const char *)oldname) + 1);
if (newname)
PRE_READ(newname, __sanitizer::internal_strlen((const char *)newname) + 1);
}
-POST_SYSCALL(renameat)(long res, long olddfd, const void *oldname, long newdfd,
- const void *newname) {}
+POST_SYSCALL(renameat)
+(long res, long olddfd, const void *oldname, long newdfd, const void *newname) {
+}
PRE_SYSCALL(futimesat)(long dfd, const void *filename, void *utimes) {
if (filename)
__sanitizer::internal_strlen((const char *)filename) + 1);
}
-POST_SYSCALL(futimesat)(long res, long dfd, const void *filename,
- void *utimes) {
+POST_SYSCALL(futimesat)
+(long res, long dfd, const void *filename, void *utimes) {
if (res >= 0) {
- if (utimes) POST_WRITE(utimes, timeval_sz);
+ if (utimes)
+ POST_WRITE(utimes, timeval_sz);
}
}
POST_SYSCALL(fchmodat)(long res, long dfd, const void *filename, long mode) {}
-PRE_SYSCALL(fchownat)(long dfd, const void *filename, long user, long group,
- long flag) {
+PRE_SYSCALL(fchownat)
+(long dfd, const void *filename, long user, long group, long flag) {
if (filename)
PRE_READ(filename,
__sanitizer::internal_strlen((const char *)filename) + 1);
}
-POST_SYSCALL(fchownat)(long res, long dfd, const void *filename, long user,
- long group, long flag) {}
+POST_SYSCALL(fchownat)
+(long res, long dfd, const void *filename, long user, long group, long flag) {}
PRE_SYSCALL(openat)(long dfd, const void *filename, long flags, long mode) {
if (filename)
__sanitizer::internal_strlen((const char *)filename) + 1);
}
-POST_SYSCALL(openat)(long res, long dfd, const void *filename, long flags,
- long mode) {}
+POST_SYSCALL(openat)
+(long res, long dfd, const void *filename, long flags, long mode) {}
-PRE_SYSCALL(newfstatat)(long dfd, const void *filename, void *statbuf,
- long flag) {
+PRE_SYSCALL(newfstatat)
+(long dfd, const void *filename, void *statbuf, long flag) {
if (filename)
PRE_READ(filename,
__sanitizer::internal_strlen((const char *)filename) + 1);
}
-POST_SYSCALL(newfstatat)(long res, long dfd, const void *filename,
- void *statbuf, long flag) {
+POST_SYSCALL(newfstatat)
+(long res, long dfd, const void *filename, void *statbuf, long flag) {
if (res >= 0) {
- if (statbuf) POST_WRITE(statbuf, struct_kernel_stat_sz);
+ if (statbuf)
+ POST_WRITE(statbuf, struct_kernel_stat_sz);
}
}
-PRE_SYSCALL(fstatat64)(long dfd, const void *filename, void *statbuf,
- long flag) {
+PRE_SYSCALL(fstatat64)
+(long dfd, const void *filename, void *statbuf, long flag) {
if (filename)
PRE_READ(filename,
__sanitizer::internal_strlen((const char *)filename) + 1);
}
-POST_SYSCALL(fstatat64)(long res, long dfd, const void *filename, void *statbuf,
- long flag) {
+POST_SYSCALL(fstatat64)
+(long res, long dfd, const void *filename, void *statbuf, long flag) {
if (res >= 0) {
- if (statbuf) POST_WRITE(statbuf, struct_kernel_stat64_sz);
+ if (statbuf)
+ POST_WRITE(statbuf, struct_kernel_stat64_sz);
}
}
PRE_READ(path, __sanitizer::internal_strlen((const char *)path) + 1);
}
-POST_SYSCALL(readlinkat)(long res, long dfd, const void *path, void *buf,
- long bufsiz) {
+POST_SYSCALL(readlinkat)
+(long res, long dfd, const void *path, void *buf, long bufsiz) {
if (res >= 0) {
if (buf)
POST_WRITE(buf, __sanitizer::internal_strlen((const char *)buf) + 1);
}
}
-PRE_SYSCALL(utimensat)(long dfd, const void *filename, void *utimes,
- long flags) {
+PRE_SYSCALL(utimensat)
+(long dfd, const void *filename, void *utimes, long flags) {
if (filename)
PRE_READ(filename,
__sanitizer::internal_strlen((const char *)filename) + 1);
}
-POST_SYSCALL(utimensat)(long res, long dfd, const void *filename, void *utimes,
- long flags) {
+POST_SYSCALL(utimensat)
+(long res, long dfd, const void *filename, void *utimes, long flags) {
if (res >= 0) {
- if (utimes) POST_WRITE(utimes, struct_timespec_sz);
+ if (utimes)
+ POST_WRITE(utimes, struct_timespec_sz);
}
}
POST_SYSCALL(unshare)(long res, long unshare_flags) {}
-PRE_SYSCALL(splice)(long fd_in, void *off_in, long fd_out, void *off_out,
- long len, long flags) {}
+PRE_SYSCALL(splice)
+(long fd_in, void *off_in, long fd_out, void *off_out, long len, long flags) {}
-POST_SYSCALL(splice)(long res, long fd_in, void *off_in, long fd_out,
- void *off_out, long len, long flags) {
+POST_SYSCALL(splice)
+(long res, long fd_in, void *off_in, long fd_out, void *off_out, long len,
+ long flags) {
if (res >= 0) {
- if (off_in) POST_WRITE(off_in, sizeof(long long));
- if (off_out) POST_WRITE(off_out, sizeof(long long));
+ if (off_in)
+ POST_WRITE(off_in, sizeof(long long));
+ if (off_out)
+ POST_WRITE(off_out, sizeof(long long));
}
}
-PRE_SYSCALL(vmsplice)(long fd, const __sanitizer_iovec *iov, long nr_segs,
- long flags) {}
+PRE_SYSCALL(vmsplice)
+(long fd, const __sanitizer_iovec *iov, long nr_segs, long flags) {}
-POST_SYSCALL(vmsplice)(long res, long fd, const __sanitizer_iovec *iov,
- long nr_segs, long flags) {
+POST_SYSCALL(vmsplice)
+(long res, long fd, const __sanitizer_iovec *iov, long nr_segs, long flags) {
if (res >= 0) {
- if (iov) kernel_read_iovec(iov, nr_segs, res);
+ if (iov)
+ kernel_read_iovec(iov, nr_segs, res);
}
}
PRE_SYSCALL(get_robust_list)(long pid, void *head_ptr, void *len_ptr) {}
-POST_SYSCALL(get_robust_list)(long res, long pid, void *head_ptr,
- void *len_ptr) {}
+POST_SYSCALL(get_robust_list)
+(long res, long pid, void *head_ptr, void *len_ptr) {}
PRE_SYSCALL(set_robust_list)(void *head, long len) {}
POST_SYSCALL(getcpu)(long res, void *cpu, void *node, void *cache) {
if (res >= 0) {
- if (cpu) POST_WRITE(cpu, sizeof(unsigned));
- if (node) POST_WRITE(node, sizeof(unsigned));
+ if (cpu)
+ POST_WRITE(cpu, sizeof(unsigned));
+ if (node)
+ POST_WRITE(node, sizeof(unsigned));
// The third argument to this system call is nowadays unused.
}
}
PRE_SYSCALL(signalfd)(long ufd, void *user_mask, long sizemask) {}
-POST_SYSCALL(signalfd)(long res, long ufd, kernel_sigset_t *user_mask,
- long sizemask) {
+POST_SYSCALL(signalfd)
+(long res, long ufd, kernel_sigset_t *user_mask, long sizemask) {
if (res >= 0) {
- if (user_mask) POST_WRITE(user_mask, sizemask);
+ if (user_mask)
+ POST_WRITE(user_mask, sizemask);
}
}
PRE_SYSCALL(signalfd4)(long ufd, void *user_mask, long sizemask, long flags) {}
-POST_SYSCALL(signalfd4)(long res, long ufd, kernel_sigset_t *user_mask,
- long sizemask, long flags) {
+POST_SYSCALL(signalfd4)
+(long res, long ufd, kernel_sigset_t *user_mask, long sizemask, long flags) {
if (res >= 0) {
- if (user_mask) POST_WRITE(user_mask, sizemask);
+ if (user_mask)
+ POST_WRITE(user_mask, sizemask);
}
}
POST_SYSCALL(timerfd_create)(long res, long clockid, long flags) {}
-PRE_SYSCALL(timerfd_settime)(long ufd, long flags, const void *utmr,
- void *otmr) {
- if (utmr) PRE_READ(utmr, struct_itimerspec_sz);
+PRE_SYSCALL(timerfd_settime)
+(long ufd, long flags, const void *utmr, void *otmr) {
+ if (utmr)
+ PRE_READ(utmr, struct_itimerspec_sz);
}
-POST_SYSCALL(timerfd_settime)(long res, long ufd, long flags, const void *utmr,
- void *otmr) {
+POST_SYSCALL(timerfd_settime)
+(long res, long ufd, long flags, const void *utmr, void *otmr) {
if (res >= 0) {
- if (otmr) POST_WRITE(otmr, struct_itimerspec_sz);
+ if (otmr)
+ POST_WRITE(otmr, struct_itimerspec_sz);
}
}
POST_SYSCALL(timerfd_gettime)(long res, long ufd, void *otmr) {
if (res >= 0) {
- if (otmr) POST_WRITE(otmr, struct_itimerspec_sz);
+ if (otmr)
+ POST_WRITE(otmr, struct_itimerspec_sz);
}
}
// Missing definition of 'struct old_linux_dirent'.
}
-PRE_SYSCALL(pselect6)(long arg0, __sanitizer___kernel_fd_set *arg1,
- __sanitizer___kernel_fd_set *arg2,
- __sanitizer___kernel_fd_set *arg3, void *arg4,
- void *arg5) {}
+PRE_SYSCALL(pselect6)
+(long arg0, __sanitizer___kernel_fd_set *arg1,
+ __sanitizer___kernel_fd_set *arg2, __sanitizer___kernel_fd_set *arg3,
+ void *arg4, void *arg5) {}
-POST_SYSCALL(pselect6)(long res, long arg0, __sanitizer___kernel_fd_set *arg1,
- __sanitizer___kernel_fd_set *arg2,
- __sanitizer___kernel_fd_set *arg3, void *arg4,
- void *arg5) {
+POST_SYSCALL(pselect6)
+(long res, long arg0, __sanitizer___kernel_fd_set *arg1,
+ __sanitizer___kernel_fd_set *arg2, __sanitizer___kernel_fd_set *arg3,
+ void *arg4, void *arg5) {
if (res >= 0) {
- if (arg1) POST_WRITE(arg1, sizeof(*arg1));
- if (arg2) POST_WRITE(arg2, sizeof(*arg2));
- if (arg3) POST_WRITE(arg3, sizeof(*arg3));
- if (arg4) POST_WRITE(arg4, struct_timespec_sz);
+ if (arg1)
+ POST_WRITE(arg1, sizeof(*arg1));
+ if (arg2)
+ POST_WRITE(arg2, sizeof(*arg2));
+ if (arg3)
+ POST_WRITE(arg3, sizeof(*arg3));
+ if (arg4)
+ POST_WRITE(arg4, struct_timespec_sz);
}
}
-PRE_SYSCALL(ppoll)(__sanitizer_pollfd *arg0, long arg1, void *arg2,
- const kernel_sigset_t *arg3, long arg4) {
- if (arg3) PRE_READ(arg3, arg4);
+PRE_SYSCALL(ppoll)
+(__sanitizer_pollfd *arg0, long arg1, void *arg2, const kernel_sigset_t *arg3,
+ long arg4) {
+ if (arg3)
+ PRE_READ(arg3, arg4);
}
-POST_SYSCALL(ppoll)(long res, __sanitizer_pollfd *arg0, long arg1, void *arg2,
- const void *arg3, long arg4) {
+POST_SYSCALL(ppoll)
+(long res, __sanitizer_pollfd *arg0, long arg1, void *arg2, const void *arg3,
+ long arg4) {
if (res >= 0) {
- if (arg0) POST_WRITE(arg0, sizeof(*arg0));
- if (arg2) POST_WRITE(arg2, struct_timespec_sz);
+ if (arg0)
+ POST_WRITE(arg0, sizeof(*arg0));
+ if (arg2)
+ POST_WRITE(arg2, struct_timespec_sz);
}
}
POST_SYSCALL(syncfs)(long res, long fd) {}
-PRE_SYSCALL(perf_event_open)(__sanitizer_perf_event_attr *attr_uptr, long pid,
- long cpu, long group_fd, long flags) {
- if (attr_uptr) PRE_READ(attr_uptr, attr_uptr->size);
+PRE_SYSCALL(perf_event_open)
+(__sanitizer_perf_event_attr *attr_uptr, long pid, long cpu, long group_fd,
+ long flags) {
+ if (attr_uptr)
+ PRE_READ(attr_uptr, attr_uptr->size);
}
-POST_SYSCALL(perf_event_open)(long res, __sanitizer_perf_event_attr *attr_uptr,
- long pid, long cpu, long group_fd, long flags) {}
+POST_SYSCALL(perf_event_open)
+(long res, __sanitizer_perf_event_attr *attr_uptr, long pid, long cpu,
+ long group_fd, long flags) {}
-PRE_SYSCALL(mmap_pgoff)(long addr, long len, long prot, long flags, long fd,
- long pgoff) {}
+PRE_SYSCALL(mmap_pgoff)
+(long addr, long len, long prot, long flags, long fd, long pgoff) {}
-POST_SYSCALL(mmap_pgoff)(long res, long addr, long len, long prot, long flags,
- long fd, long pgoff) {}
+POST_SYSCALL(mmap_pgoff)
+(long res, long addr, long len, long prot, long flags, long fd, long pgoff) {}
PRE_SYSCALL(old_mmap)(void *arg) {}
POST_SYSCALL(old_mmap)(long res, void *arg) {}
-PRE_SYSCALL(name_to_handle_at)(long dfd, const void *name, void *handle,
- void *mnt_id, long flag) {}
+PRE_SYSCALL(name_to_handle_at)
+(long dfd, const void *name, void *handle, void *mnt_id, long flag) {}
-POST_SYSCALL(name_to_handle_at)(long res, long dfd, const void *name,
- void *handle, void *mnt_id, long flag) {}
+POST_SYSCALL(name_to_handle_at)
+(long res, long dfd, const void *name, void *handle, void *mnt_id, long flag) {}
PRE_SYSCALL(open_by_handle_at)(long mountdirfd, void *handle, long flags) {}
-POST_SYSCALL(open_by_handle_at)(long res, long mountdirfd, void *handle,
- long flags) {}
+POST_SYSCALL(open_by_handle_at)
+(long res, long mountdirfd, void *handle, long flags) {}
PRE_SYSCALL(setns)(long fd, long nstype) {}
POST_SYSCALL(setns)(long res, long fd, long nstype) {}
-PRE_SYSCALL(process_vm_readv)(long pid, const __sanitizer_iovec *lvec,
- long liovcnt, const void *rvec, long riovcnt,
- long flags) {}
+PRE_SYSCALL(process_vm_readv)
+(long pid, const __sanitizer_iovec *lvec, long liovcnt, const void *rvec,
+ long riovcnt, long flags) {}
-POST_SYSCALL(process_vm_readv)(long res, long pid,
- const __sanitizer_iovec *lvec, long liovcnt,
- const void *rvec, long riovcnt, long flags) {
+POST_SYSCALL(process_vm_readv)
+(long res, long pid, const __sanitizer_iovec *lvec, long liovcnt,
+ const void *rvec, long riovcnt, long flags) {
if (res >= 0) {
- if (lvec) kernel_write_iovec(lvec, liovcnt, res);
+ if (lvec)
+ kernel_write_iovec(lvec, liovcnt, res);
}
}
-PRE_SYSCALL(process_vm_writev)(long pid, const __sanitizer_iovec *lvec,
- long liovcnt, const void *rvec, long riovcnt,
- long flags) {}
+PRE_SYSCALL(process_vm_writev)
+(long pid, const __sanitizer_iovec *lvec, long liovcnt, const void *rvec,
+ long riovcnt, long flags) {}
-POST_SYSCALL(process_vm_writev)(long res, long pid,
- const __sanitizer_iovec *lvec, long liovcnt,
- const void *rvec, long riovcnt, long flags) {
+POST_SYSCALL(process_vm_writev)
+(long res, long pid, const __sanitizer_iovec *lvec, long liovcnt,
+ const void *rvec, long riovcnt, long flags) {
if (res >= 0) {
- if (lvec) kernel_read_iovec(lvec, liovcnt, res);
+ if (lvec)
+ kernel_read_iovec(lvec, liovcnt, res);
}
}
-PRE_SYSCALL(fork)() {
- COMMON_SYSCALL_PRE_FORK();
-}
+PRE_SYSCALL(fork)() { COMMON_SYSCALL_PRE_FORK(); }
-POST_SYSCALL(fork)(long res) {
- COMMON_SYSCALL_POST_FORK(res);
-}
+POST_SYSCALL(fork)(long res) { COMMON_SYSCALL_POST_FORK(res); }
-PRE_SYSCALL(vfork)() {
- COMMON_SYSCALL_PRE_FORK();
-}
+PRE_SYSCALL(vfork)() { COMMON_SYSCALL_PRE_FORK(); }
-POST_SYSCALL(vfork)(long res) {
- COMMON_SYSCALL_POST_FORK(res);
-}
+POST_SYSCALL(vfork)(long res) { COMMON_SYSCALL_POST_FORK(res); }
-PRE_SYSCALL(sigaction)(long signum, const __sanitizer_kernel_sigaction_t *act,
- __sanitizer_kernel_sigaction_t *oldact) {
+PRE_SYSCALL(sigaction)
+(long signum, const __sanitizer_kernel_sigaction_t *act,
+ __sanitizer_kernel_sigaction_t *oldact) {
if (act) {
PRE_READ(&act->sigaction, sizeof(act->sigaction));
PRE_READ(&act->sa_flags, sizeof(act->sa_flags));
}
}
-POST_SYSCALL(sigaction)(long res, long signum,
- const __sanitizer_kernel_sigaction_t *act,
- __sanitizer_kernel_sigaction_t *oldact) {
- if (res >= 0 && oldact) POST_WRITE(oldact, sizeof(*oldact));
+POST_SYSCALL(sigaction)
+(long res, long signum, const __sanitizer_kernel_sigaction_t *act,
+ __sanitizer_kernel_sigaction_t *oldact) {
+ if (res >= 0 && oldact)
+ POST_WRITE(oldact, sizeof(*oldact));
}
-PRE_SYSCALL(rt_sigaction)(long signum,
- const __sanitizer_kernel_sigaction_t *act,
- __sanitizer_kernel_sigaction_t *oldact, SIZE_T sz) {
+PRE_SYSCALL(rt_sigaction)
+(long signum, const __sanitizer_kernel_sigaction_t *act,
+ __sanitizer_kernel_sigaction_t *oldact, SIZE_T sz) {
if (act) {
PRE_READ(&act->sigaction, sizeof(act->sigaction));
PRE_READ(&act->sa_flags, sizeof(act->sa_flags));
}
}
-POST_SYSCALL(rt_sigaction)(long res, long signum,
- const __sanitizer_kernel_sigaction_t *act,
- __sanitizer_kernel_sigaction_t *oldact, SIZE_T sz) {
+POST_SYSCALL(rt_sigaction)
+(long res, long signum, const __sanitizer_kernel_sigaction_t *act,
+ __sanitizer_kernel_sigaction_t *oldact, SIZE_T sz) {
if (res >= 0 && oldact) {
SIZE_T oldact_sz = ((char *)&oldact->sa_mask) - ((char *)oldact) + sz;
POST_WRITE(oldact, oldact_sz);
}
} // extern "C"
-#undef PRE_SYSCALL
-#undef PRE_READ
-#undef PRE_WRITE
-#undef POST_SYSCALL
-#undef POST_READ
-#undef POST_WRITE
+# undef PRE_SYSCALL
+# undef PRE_READ
+# undef PRE_WRITE
+# undef POST_SYSCALL
+# undef POST_READ
+# undef POST_WRITE
#endif // SANITIZER_LINUX
#include "sanitizer_atomic.h"
#include "sanitizer_common.h"
+#include "sanitizer_interface_internal.h"
#include "sanitizer_internal_defs.h"
#include "sanitizer_symbolizer_fuchsia.h"
// This class relies on zero-initialization.
class TracePcGuardController final {
public:
+ constexpr TracePcGuardController() {}
+
// For each PC location being tracked, there is a u32 reserved in global
// data called the "guard". At startup, we assign each guard slot a
// unique index into the big results array. Later during runtime, the
}
void Dump() {
- BlockingMutexLock locked(&setup_lock_);
+ Lock locked(&setup_lock_);
if (array_) {
CHECK_NE(vmo_, ZX_HANDLE_INVALID);
// We can always spare the 32G of address space.
static constexpr size_t MappingSize = sizeof(uptr) << 32;
- BlockingMutex setup_lock_ = BlockingMutex(LINKER_INITIALIZED);
+ Mutex setup_lock_;
uptr *array_ = nullptr;
u32 next_index_ = 0;
zx_handle_t vmo_ = {};
size_t DataSize() const { return next_index_ * sizeof(uintptr_t); }
u32 Setup(u32 num_guards) {
- BlockingMutexLock locked(&setup_lock_);
+ Lock locked(&setup_lock_);
DCHECK(common_flags()->coverage);
if (next_index_ == 0) {
INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_pc_guard)
INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_pc_guard_init)
INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_pc_indir)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_load1)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_load2)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_load4)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_load8)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_load16)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_store1)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_store2)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_store4)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_store8)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_store16)
INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_switch)
INTERFACE_WEAK_FUNCTION(__sanitizer_cov_8bit_counters_init)
INTERFACE_WEAK_FUNCTION(__sanitizer_cov_bool_flag_init)
#include "sanitizer_platform.h"
#if !SANITIZER_FUCHSIA
-#include "sancov_flags.h"
-#include "sanitizer_allocator_internal.h"
-#include "sanitizer_atomic.h"
-#include "sanitizer_common.h"
-#include "sanitizer_file.h"
+# include "sancov_flags.h"
+# include "sanitizer_allocator_internal.h"
+# include "sanitizer_atomic.h"
+# include "sanitizer_common.h"
+# include "sanitizer_common/sanitizer_stacktrace.h"
+# include "sanitizer_file.h"
+# include "sanitizer_interface_internal.h"
using namespace __sanitizer;
const uptr pc = pcs[i];
if (!pc) continue;
- if (!__sanitizer_get_module_and_offset_for_pc(pc, nullptr, 0, &pcs[i])) {
- Printf("ERROR: unknown pc 0x%x (may happen if dlclose is used)\n", pc);
+ if (!GetModuleAndOffsetForPc(pc, nullptr, 0, &pcs[i])) {
+ Printf("ERROR: unknown pc 0x%zx (may happen if dlclose is used)\n", pc);
continue;
}
uptr module_base = pc - pcs[i];
last_base = module_base;
module_start_idx = i;
module_found = true;
- __sanitizer_get_module_and_offset_for_pc(pc, module_name, kMaxPathLength,
- &pcs[i]);
+ GetModuleAndOffsetForPc(pc, module_name, kMaxPathLength, &pcs[i]);
}
}
static TracePcGuardController pc_guard_controller;
+// A basic default implementation of callbacks for
+// -fsanitize-coverage=inline-8bit-counters,pc-table.
+// Use TOOL_OPTIONS (UBSAN_OPTIONS, etc) to dump the coverage data:
+// * cov_8bit_counters_out=PATH to dump the 8bit counters.
+// * cov_pcs_out=PATH to dump the pc table.
+//
+// Most users will still need to define their own callbacks for greater
+// flexibility.
+namespace SingletonCounterCoverage {
+
+static char *counters_beg, *counters_end;
+static const uptr *pcs_beg, *pcs_end;
+
+static void DumpCoverage() {
+ const char* file_path = common_flags()->cov_8bit_counters_out;
+ if (file_path && internal_strlen(file_path)) {
+ fd_t fd = OpenFile(file_path);
+ FileCloser file_closer(fd);
+ uptr size = counters_end - counters_beg;
+ WriteToFile(fd, counters_beg, size);
+ if (common_flags()->verbosity)
+ __sanitizer::Printf("cov_8bit_counters_out: written %zd bytes to %s\n",
+ size, file_path);
+ }
+ file_path = common_flags()->cov_pcs_out;
+ if (file_path && internal_strlen(file_path)) {
+ fd_t fd = OpenFile(file_path);
+ FileCloser file_closer(fd);
+ uptr size = (pcs_end - pcs_beg) * sizeof(uptr);
+ WriteToFile(fd, pcs_beg, size);
+ if (common_flags()->verbosity)
+ __sanitizer::Printf("cov_pcs_out: written %zd bytes to %s\n", size,
+ file_path);
+ }
+}
+
+static void Cov8bitCountersInit(char* beg, char* end) {
+ counters_beg = beg;
+ counters_end = end;
+ Atexit(DumpCoverage);
+}
+
+static void CovPcsInit(const uptr* beg, const uptr* end) {
+ pcs_beg = beg;
+ pcs_end = end;
+}
+
+} // namespace SingletonCounterCoverage
+
} // namespace
} // namespace __sancov
SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_pc_guard, u32* guard) {
if (!*guard) return;
- __sancov::pc_guard_controller.TracePcGuard(guard, GET_CALLER_PC() - 1);
+ __sancov::pc_guard_controller.TracePcGuard(
+ guard, StackTrace::GetPreviousInstructionPc(GET_CALLER_PC()));
}
SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_pc_guard_init,
SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_reset() {
__sancov::pc_guard_controller.Reset();
}
-// Default empty implementations (weak). Users should redefine them.
+// Default implementations (weak).
+// Either empty or very simple.
+// Most users should redefine them.
SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp, void) {}
SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp1, void) {}
SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp2, void) {}
SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_div8, void) {}
SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_gep, void) {}
SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_pc_indir, void) {}
-SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_8bit_counters_init, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_load1, void){}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_load2, void){}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_load4, void){}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_load8, void){}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_load16, void){}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_store1, void){}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_store2, void){}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_store4, void){}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_store8, void){}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_store16, void){}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_8bit_counters_init,
+ char* start, char* end) {
+ __sancov::SingletonCounterCoverage::Cov8bitCountersInit(start, end);
+}
SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_bool_flag_init, void) {}
-SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_pcs_init, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_pcs_init, const uptr* beg,
+ const uptr* end) {
+ __sancov::SingletonCounterCoverage::CovPcsInit(beg, end);
+}
} // extern "C"
// Weak definition for code instrumented with -fsanitize-coverage=stack-depth
// and later linked with code containing a strong definition.
}
// Returns true iff dtls is empty (no locks are currently held) and we can
- // add the node to the currently held locks w/o chanding the global state.
+ // add the node to the currently held locks w/o changing the global state.
// This operation is thread-safe as it only touches the dtls.
bool onFirstLock(DeadlockDetectorTLS<BV> *dtls, uptr node, u32 stk = 0) {
if (!dtls->empty()) return false;
--- /dev/null
+//===- sanitizer_dense_map.h - Dense probed hash table ----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This is fork of llvm/ADT/DenseMap.h class with the following changes:
+// * Use mmap to allocate.
+// * No iterators.
+// * Does not shrink.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_DENSE_MAP_H
+#define SANITIZER_DENSE_MAP_H
+
+#include "sanitizer_common.h"
+#include "sanitizer_dense_map_info.h"
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_type_traits.h"
+
+namespace __sanitizer {
+
+template <typename DerivedT, typename KeyT, typename ValueT, typename KeyInfoT,
+ typename BucketT>
+class DenseMapBase {
+ public:
+ using size_type = unsigned;
+ using key_type = KeyT;
+ using mapped_type = ValueT;
+ using value_type = BucketT;
+
+ WARN_UNUSED_RESULT bool empty() const { return getNumEntries() == 0; }
+ unsigned size() const { return getNumEntries(); }
+
+ /// Grow the densemap so that it can contain at least \p NumEntries items
+ /// before resizing again.
+ void reserve(size_type NumEntries) {
+ auto NumBuckets = getMinBucketToReserveForEntries(NumEntries);
+ if (NumBuckets > getNumBuckets())
+ grow(NumBuckets);
+ }
+
+ void clear() {
+ if (getNumEntries() == 0 && getNumTombstones() == 0)
+ return;
+
+ const KeyT EmptyKey = getEmptyKey(), TombstoneKey = getTombstoneKey();
+ if (__sanitizer::is_trivially_destructible<ValueT>::value) {
+ // Use a simpler loop when values don't need destruction.
+ for (BucketT *P = getBuckets(), *E = getBucketsEnd(); P != E; ++P)
+ P->getFirst() = EmptyKey;
+ } else {
+ unsigned NumEntries = getNumEntries();
+ for (BucketT *P = getBuckets(), *E = getBucketsEnd(); P != E; ++P) {
+ if (!KeyInfoT::isEqual(P->getFirst(), EmptyKey)) {
+ if (!KeyInfoT::isEqual(P->getFirst(), TombstoneKey)) {
+ P->getSecond().~ValueT();
+ --NumEntries;
+ }
+ P->getFirst() = EmptyKey;
+ }
+ }
+ CHECK_EQ(NumEntries, 0);
+ }
+ setNumEntries(0);
+ setNumTombstones(0);
+ }
+
+ /// Return 1 if the specified key is in the map, 0 otherwise.
+ size_type count(const KeyT &Key) const {
+ const BucketT *TheBucket;
+ return LookupBucketFor(Key, TheBucket) ? 1 : 0;
+ }
+
+ value_type *find(const KeyT &Key) {
+ BucketT *TheBucket;
+ if (LookupBucketFor(Key, TheBucket))
+ return TheBucket;
+ return nullptr;
+ }
+ const value_type *find(const KeyT &Key) const {
+ const BucketT *TheBucket;
+ if (LookupBucketFor(Key, TheBucket))
+ return TheBucket;
+ return nullptr;
+ }
+
+ /// Alternate version of find() which allows a different, and possibly
+ /// less expensive, key type.
+ /// The DenseMapInfo is responsible for supplying methods
+ /// getHashValue(LookupKeyT) and isEqual(LookupKeyT, KeyT) for each key
+ /// type used.
+ template <class LookupKeyT>
+ value_type *find_as(const LookupKeyT &Key) {
+ BucketT *TheBucket;
+ if (LookupBucketFor(Key, TheBucket))
+ return TheBucket;
+ return nullptr;
+ }
+ template <class LookupKeyT>
+ const value_type *find_as(const LookupKeyT &Key) const {
+ const BucketT *TheBucket;
+ if (LookupBucketFor(Key, TheBucket))
+ return TheBucket;
+ return nullptr;
+ }
+
+ /// lookup - Return the entry for the specified key, or a default
+ /// constructed value if no such entry exists.
+ ValueT lookup(const KeyT &Key) const {
+ const BucketT *TheBucket;
+ if (LookupBucketFor(Key, TheBucket))
+ return TheBucket->getSecond();
+ return ValueT();
+ }
+
+ // Inserts key,value pair into the map if the key isn't already in the map.
+ // If the key is already in the map, it returns false and doesn't update the
+ // value.
+ detail::DenseMapPair<value_type *, bool> insert(const value_type &KV) {
+ return try_emplace(KV.first, KV.second);
+ }
+
+ // Inserts key,value pair into the map if the key isn't already in the map.
+ // If the key is already in the map, it returns false and doesn't update the
+ // value.
+ detail::DenseMapPair<value_type *, bool> insert(value_type &&KV) {
+ return try_emplace(__sanitizer::move(KV.first),
+ __sanitizer::move(KV.second));
+ }
+
+ // Inserts key,value pair into the map if the key isn't already in the map.
+ // The value is constructed in-place if the key is not in the map, otherwise
+ // it is not moved.
+ template <typename... Ts>
+ detail::DenseMapPair<value_type *, bool> try_emplace(KeyT &&Key,
+ Ts &&...Args) {
+ BucketT *TheBucket;
+ if (LookupBucketFor(Key, TheBucket))
+ return {TheBucket, false}; // Already in map.
+
+ // Otherwise, insert the new element.
+ TheBucket = InsertIntoBucket(TheBucket, __sanitizer::move(Key),
+ __sanitizer::forward<Ts>(Args)...);
+ return {TheBucket, true};
+ }
+
+ // Inserts key,value pair into the map if the key isn't already in the map.
+ // The value is constructed in-place if the key is not in the map, otherwise
+ // it is not moved.
+ template <typename... Ts>
+ detail::DenseMapPair<value_type *, bool> try_emplace(const KeyT &Key,
+ Ts &&...Args) {
+ BucketT *TheBucket;
+ if (LookupBucketFor(Key, TheBucket))
+ return {TheBucket, false}; // Already in map.
+
+ // Otherwise, insert the new element.
+ TheBucket =
+ InsertIntoBucket(TheBucket, Key, __sanitizer::forward<Ts>(Args)...);
+ return {TheBucket, true};
+ }
+
+ /// Alternate version of insert() which allows a different, and possibly
+ /// less expensive, key type.
+ /// The DenseMapInfo is responsible for supplying methods
+ /// getHashValue(LookupKeyT) and isEqual(LookupKeyT, KeyT) for each key
+ /// type used.
+ template <typename LookupKeyT>
+ detail::DenseMapPair<value_type *, bool> insert_as(value_type &&KV,
+ const LookupKeyT &Val) {
+ BucketT *TheBucket;
+ if (LookupBucketFor(Val, TheBucket))
+ return {TheBucket, false}; // Already in map.
+
+ // Otherwise, insert the new element.
+ TheBucket =
+ InsertIntoBucketWithLookup(TheBucket, __sanitizer::move(KV.first),
+ __sanitizer::move(KV.second), Val);
+ return {TheBucket, true};
+ }
+
+ bool erase(const KeyT &Val) {
+ BucketT *TheBucket;
+ if (!LookupBucketFor(Val, TheBucket))
+ return false; // not in map.
+
+ TheBucket->getSecond().~ValueT();
+ TheBucket->getFirst() = getTombstoneKey();
+ decrementNumEntries();
+ incrementNumTombstones();
+ return true;
+ }
+
+ void erase(value_type *I) {
+ CHECK_NE(I, nullptr);
+ BucketT *TheBucket = &*I;
+ TheBucket->getSecond().~ValueT();
+ TheBucket->getFirst() = getTombstoneKey();
+ decrementNumEntries();
+ incrementNumTombstones();
+ }
+
+ value_type &FindAndConstruct(const KeyT &Key) {
+ BucketT *TheBucket;
+ if (LookupBucketFor(Key, TheBucket))
+ return *TheBucket;
+
+ return *InsertIntoBucket(TheBucket, Key);
+ }
+
+ ValueT &operator[](const KeyT &Key) { return FindAndConstruct(Key).second; }
+
+ value_type &FindAndConstruct(KeyT &&Key) {
+ BucketT *TheBucket;
+ if (LookupBucketFor(Key, TheBucket))
+ return *TheBucket;
+
+ return *InsertIntoBucket(TheBucket, __sanitizer::move(Key));
+ }
+
+ ValueT &operator[](KeyT &&Key) {
+ return FindAndConstruct(__sanitizer::move(Key)).second;
+ }
+
+ /// Iterate over active entries of the container.
+ ///
+ /// Function can return fast to stop the process.
+ template <class Fn>
+ void forEach(Fn fn) {
+ const KeyT EmptyKey = getEmptyKey(), TombstoneKey = getTombstoneKey();
+ for (auto *P = getBuckets(), *E = getBucketsEnd(); P != E; ++P) {
+ const KeyT K = P->getFirst();
+ if (!KeyInfoT::isEqual(K, EmptyKey) &&
+ !KeyInfoT::isEqual(K, TombstoneKey)) {
+ if (!fn(*P))
+ return;
+ }
+ }
+ }
+
+ template <class Fn>
+ void forEach(Fn fn) const {
+ const_cast<DenseMapBase *>(this)->forEach(
+ [&](const value_type &KV) { return fn(KV); });
+ }
+
+ protected:
+ DenseMapBase() = default;
+
+ void destroyAll() {
+ if (getNumBuckets() == 0) // Nothing to do.
+ return;
+
+ const KeyT EmptyKey = getEmptyKey(), TombstoneKey = getTombstoneKey();
+ for (BucketT *P = getBuckets(), *E = getBucketsEnd(); P != E; ++P) {
+ if (!KeyInfoT::isEqual(P->getFirst(), EmptyKey) &&
+ !KeyInfoT::isEqual(P->getFirst(), TombstoneKey))
+ P->getSecond().~ValueT();
+ P->getFirst().~KeyT();
+ }
+ }
+
+ void initEmpty() {
+ setNumEntries(0);
+ setNumTombstones(0);
+
+ CHECK_EQ((getNumBuckets() & (getNumBuckets() - 1)), 0);
+ const KeyT EmptyKey = getEmptyKey();
+ for (BucketT *B = getBuckets(), *E = getBucketsEnd(); B != E; ++B)
+ ::new (&B->getFirst()) KeyT(EmptyKey);
+ }
+
+ /// Returns the number of buckets to allocate to ensure that the DenseMap can
+ /// accommodate \p NumEntries without need to grow().
+ unsigned getMinBucketToReserveForEntries(unsigned NumEntries) {
+ // Ensure that "NumEntries * 4 < NumBuckets * 3"
+ if (NumEntries == 0)
+ return 0;
+ // +1 is required because of the strict equality.
+ // For example if NumEntries is 48, we need to return 401.
+ return RoundUpToPowerOfTwo((NumEntries * 4 / 3 + 1) + /* NextPowerOf2 */ 1);
+ }
+
+ void moveFromOldBuckets(BucketT *OldBucketsBegin, BucketT *OldBucketsEnd) {
+ initEmpty();
+
+ // Insert all the old elements.
+ const KeyT EmptyKey = getEmptyKey();
+ const KeyT TombstoneKey = getTombstoneKey();
+ for (BucketT *B = OldBucketsBegin, *E = OldBucketsEnd; B != E; ++B) {
+ if (!KeyInfoT::isEqual(B->getFirst(), EmptyKey) &&
+ !KeyInfoT::isEqual(B->getFirst(), TombstoneKey)) {
+ // Insert the key/value into the new table.
+ BucketT *DestBucket;
+ bool FoundVal = LookupBucketFor(B->getFirst(), DestBucket);
+ (void)FoundVal; // silence warning.
+ CHECK(!FoundVal);
+ DestBucket->getFirst() = __sanitizer::move(B->getFirst());
+ ::new (&DestBucket->getSecond())
+ ValueT(__sanitizer::move(B->getSecond()));
+ incrementNumEntries();
+
+ // Free the value.
+ B->getSecond().~ValueT();
+ }
+ B->getFirst().~KeyT();
+ }
+ }
+
+ template <typename OtherBaseT>
+ void copyFrom(
+ const DenseMapBase<OtherBaseT, KeyT, ValueT, KeyInfoT, BucketT> &other) {
+ CHECK_NE(&other, this);
+ CHECK_EQ(getNumBuckets(), other.getNumBuckets());
+
+ setNumEntries(other.getNumEntries());
+ setNumTombstones(other.getNumTombstones());
+
+ if (__sanitizer::is_trivially_copyable<KeyT>::value &&
+ __sanitizer::is_trivially_copyable<ValueT>::value)
+ internal_memcpy(reinterpret_cast<void *>(getBuckets()),
+ other.getBuckets(), getNumBuckets() * sizeof(BucketT));
+ else
+ for (uptr i = 0; i < getNumBuckets(); ++i) {
+ ::new (&getBuckets()[i].getFirst())
+ KeyT(other.getBuckets()[i].getFirst());
+ if (!KeyInfoT::isEqual(getBuckets()[i].getFirst(), getEmptyKey()) &&
+ !KeyInfoT::isEqual(getBuckets()[i].getFirst(), getTombstoneKey()))
+ ::new (&getBuckets()[i].getSecond())
+ ValueT(other.getBuckets()[i].getSecond());
+ }
+ }
+
+ static unsigned getHashValue(const KeyT &Val) {
+ return KeyInfoT::getHashValue(Val);
+ }
+
+ template <typename LookupKeyT>
+ static unsigned getHashValue(const LookupKeyT &Val) {
+ return KeyInfoT::getHashValue(Val);
+ }
+
+ static const KeyT getEmptyKey() { return KeyInfoT::getEmptyKey(); }
+
+ static const KeyT getTombstoneKey() { return KeyInfoT::getTombstoneKey(); }
+
+ private:
+ unsigned getNumEntries() const {
+ return static_cast<const DerivedT *>(this)->getNumEntries();
+ }
+
+ void setNumEntries(unsigned Num) {
+ static_cast<DerivedT *>(this)->setNumEntries(Num);
+ }
+
+ void incrementNumEntries() { setNumEntries(getNumEntries() + 1); }
+
+ void decrementNumEntries() { setNumEntries(getNumEntries() - 1); }
+
+ unsigned getNumTombstones() const {
+ return static_cast<const DerivedT *>(this)->getNumTombstones();
+ }
+
+ void setNumTombstones(unsigned Num) {
+ static_cast<DerivedT *>(this)->setNumTombstones(Num);
+ }
+
+ void incrementNumTombstones() { setNumTombstones(getNumTombstones() + 1); }
+
+ void decrementNumTombstones() { setNumTombstones(getNumTombstones() - 1); }
+
+ const BucketT *getBuckets() const {
+ return static_cast<const DerivedT *>(this)->getBuckets();
+ }
+
+ BucketT *getBuckets() { return static_cast<DerivedT *>(this)->getBuckets(); }
+
+ unsigned getNumBuckets() const {
+ return static_cast<const DerivedT *>(this)->getNumBuckets();
+ }
+
+ BucketT *getBucketsEnd() { return getBuckets() + getNumBuckets(); }
+
+ const BucketT *getBucketsEnd() const {
+ return getBuckets() + getNumBuckets();
+ }
+
+ void grow(unsigned AtLeast) { static_cast<DerivedT *>(this)->grow(AtLeast); }
+
+ template <typename KeyArg, typename... ValueArgs>
+ BucketT *InsertIntoBucket(BucketT *TheBucket, KeyArg &&Key,
+ ValueArgs &&...Values) {
+ TheBucket = InsertIntoBucketImpl(Key, Key, TheBucket);
+
+ TheBucket->getFirst() = __sanitizer::forward<KeyArg>(Key);
+ ::new (&TheBucket->getSecond())
+ ValueT(__sanitizer::forward<ValueArgs>(Values)...);
+ return TheBucket;
+ }
+
+ template <typename LookupKeyT>
+ BucketT *InsertIntoBucketWithLookup(BucketT *TheBucket, KeyT &&Key,
+ ValueT &&Value, LookupKeyT &Lookup) {
+ TheBucket = InsertIntoBucketImpl(Key, Lookup, TheBucket);
+
+ TheBucket->getFirst() = __sanitizer::move(Key);
+ ::new (&TheBucket->getSecond()) ValueT(__sanitizer::move(Value));
+ return TheBucket;
+ }
+
+ template <typename LookupKeyT>
+ BucketT *InsertIntoBucketImpl(const KeyT &Key, const LookupKeyT &Lookup,
+ BucketT *TheBucket) {
+ // If the load of the hash table is more than 3/4, or if fewer than 1/8 of
+ // the buckets are empty (meaning that many are filled with tombstones),
+ // grow the table.
+ //
+ // The later case is tricky. For example, if we had one empty bucket with
+ // tons of tombstones, failing lookups (e.g. for insertion) would have to
+ // probe almost the entire table until it found the empty bucket. If the
+ // table completely filled with tombstones, no lookup would ever succeed,
+ // causing infinite loops in lookup.
+ unsigned NewNumEntries = getNumEntries() + 1;
+ unsigned NumBuckets = getNumBuckets();
+ if (UNLIKELY(NewNumEntries * 4 >= NumBuckets * 3)) {
+ this->grow(NumBuckets * 2);
+ LookupBucketFor(Lookup, TheBucket);
+ NumBuckets = getNumBuckets();
+ } else if (UNLIKELY(NumBuckets - (NewNumEntries + getNumTombstones()) <=
+ NumBuckets / 8)) {
+ this->grow(NumBuckets);
+ LookupBucketFor(Lookup, TheBucket);
+ }
+ CHECK(TheBucket);
+
+ // Only update the state after we've grown our bucket space appropriately
+ // so that when growing buckets we have self-consistent entry count.
+ incrementNumEntries();
+
+ // If we are writing over a tombstone, remember this.
+ const KeyT EmptyKey = getEmptyKey();
+ if (!KeyInfoT::isEqual(TheBucket->getFirst(), EmptyKey))
+ decrementNumTombstones();
+
+ return TheBucket;
+ }
+
+ /// LookupBucketFor - Lookup the appropriate bucket for Val, returning it in
+ /// FoundBucket. If the bucket contains the key and a value, this returns
+ /// true, otherwise it returns a bucket with an empty marker or tombstone and
+ /// returns false.
+ template <typename LookupKeyT>
+ bool LookupBucketFor(const LookupKeyT &Val,
+ const BucketT *&FoundBucket) const {
+ const BucketT *BucketsPtr = getBuckets();
+ const unsigned NumBuckets = getNumBuckets();
+
+ if (NumBuckets == 0) {
+ FoundBucket = nullptr;
+ return false;
+ }
+
+ // FoundTombstone - Keep track of whether we find a tombstone while probing.
+ const BucketT *FoundTombstone = nullptr;
+ const KeyT EmptyKey = getEmptyKey();
+ const KeyT TombstoneKey = getTombstoneKey();
+ CHECK(!KeyInfoT::isEqual(Val, EmptyKey));
+ CHECK(!KeyInfoT::isEqual(Val, TombstoneKey));
+
+ unsigned BucketNo = getHashValue(Val) & (NumBuckets - 1);
+ unsigned ProbeAmt = 1;
+ while (true) {
+ const BucketT *ThisBucket = BucketsPtr + BucketNo;
+ // Found Val's bucket? If so, return it.
+ if (LIKELY(KeyInfoT::isEqual(Val, ThisBucket->getFirst()))) {
+ FoundBucket = ThisBucket;
+ return true;
+ }
+
+ // If we found an empty bucket, the key doesn't exist in the set.
+ // Insert it and return the default value.
+ if (LIKELY(KeyInfoT::isEqual(ThisBucket->getFirst(), EmptyKey))) {
+ // If we've already seen a tombstone while probing, fill it in instead
+ // of the empty bucket we eventually probed to.
+ FoundBucket = FoundTombstone ? FoundTombstone : ThisBucket;
+ return false;
+ }
+
+ // If this is a tombstone, remember it. If Val ends up not in the map, we
+ // prefer to return it than something that would require more probing.
+ if (KeyInfoT::isEqual(ThisBucket->getFirst(), TombstoneKey) &&
+ !FoundTombstone)
+ FoundTombstone = ThisBucket; // Remember the first tombstone found.
+
+ // Otherwise, it's a hash collision or a tombstone, continue quadratic
+ // probing.
+ BucketNo += ProbeAmt++;
+ BucketNo &= (NumBuckets - 1);
+ }
+ }
+
+ template <typename LookupKeyT>
+ bool LookupBucketFor(const LookupKeyT &Val, BucketT *&FoundBucket) {
+ const BucketT *ConstFoundBucket;
+ bool Result = const_cast<const DenseMapBase *>(this)->LookupBucketFor(
+ Val, ConstFoundBucket);
+ FoundBucket = const_cast<BucketT *>(ConstFoundBucket);
+ return Result;
+ }
+
+ public:
+ /// Return the approximate size (in bytes) of the actual map.
+ /// This is just the raw memory used by DenseMap.
+ /// If entries are pointers to objects, the size of the referenced objects
+ /// are not included.
+ uptr getMemorySize() const {
+ return RoundUpTo(getNumBuckets() * sizeof(BucketT), GetPageSizeCached());
+ }
+};
+
+/// Equality comparison for DenseMap.
+///
+/// Iterates over elements of LHS confirming that each (key, value) pair in LHS
+/// is also in RHS, and that no additional pairs are in RHS.
+/// Equivalent to N calls to RHS.find and N value comparisons. Amortized
+/// complexity is linear, worst case is O(N^2) (if every hash collides).
+template <typename DerivedT, typename KeyT, typename ValueT, typename KeyInfoT,
+ typename BucketT>
+bool operator==(
+ const DenseMapBase<DerivedT, KeyT, ValueT, KeyInfoT, BucketT> &LHS,
+ const DenseMapBase<DerivedT, KeyT, ValueT, KeyInfoT, BucketT> &RHS) {
+ if (LHS.size() != RHS.size())
+ return false;
+
+ bool R = true;
+ LHS.forEach(
+ [&](const typename DenseMapBase<DerivedT, KeyT, ValueT, KeyInfoT,
+ BucketT>::value_type &KV) -> bool {
+ const auto *I = RHS.find(KV.first);
+ if (!I || I->second != KV.second) {
+ R = false;
+ return false;
+ }
+ return true;
+ });
+
+ return R;
+}
+
+/// Inequality comparison for DenseMap.
+///
+/// Equivalent to !(LHS == RHS). See operator== for performance notes.
+template <typename DerivedT, typename KeyT, typename ValueT, typename KeyInfoT,
+ typename BucketT>
+bool operator!=(
+ const DenseMapBase<DerivedT, KeyT, ValueT, KeyInfoT, BucketT> &LHS,
+ const DenseMapBase<DerivedT, KeyT, ValueT, KeyInfoT, BucketT> &RHS) {
+ return !(LHS == RHS);
+}
+
+template <typename KeyT, typename ValueT,
+ typename KeyInfoT = DenseMapInfo<KeyT>,
+ typename BucketT = detail::DenseMapPair<KeyT, ValueT>>
+class DenseMap : public DenseMapBase<DenseMap<KeyT, ValueT, KeyInfoT, BucketT>,
+ KeyT, ValueT, KeyInfoT, BucketT> {
+ friend class DenseMapBase<DenseMap, KeyT, ValueT, KeyInfoT, BucketT>;
+
+ // Lift some types from the dependent base class into this class for
+ // simplicity of referring to them.
+ using BaseT = DenseMapBase<DenseMap, KeyT, ValueT, KeyInfoT, BucketT>;
+
+ BucketT *Buckets = nullptr;
+ unsigned NumEntries = 0;
+ unsigned NumTombstones = 0;
+ unsigned NumBuckets = 0;
+
+ public:
+ /// Create a DenseMap with an optional \p InitialReserve that guarantee that
+ /// this number of elements can be inserted in the map without grow()
+ explicit DenseMap(unsigned InitialReserve) { init(InitialReserve); }
+ constexpr DenseMap() = default;
+
+ DenseMap(const DenseMap &other) : BaseT() {
+ init(0);
+ copyFrom(other);
+ }
+
+ DenseMap(DenseMap &&other) : BaseT() {
+ init(0);
+ swap(other);
+ }
+
+ ~DenseMap() {
+ this->destroyAll();
+ deallocate_buffer(Buckets, sizeof(BucketT) * NumBuckets);
+ }
+
+ void swap(DenseMap &RHS) {
+ Swap(Buckets, RHS.Buckets);
+ Swap(NumEntries, RHS.NumEntries);
+ Swap(NumTombstones, RHS.NumTombstones);
+ Swap(NumBuckets, RHS.NumBuckets);
+ }
+
+ DenseMap &operator=(const DenseMap &other) {
+ if (&other != this)
+ copyFrom(other);
+ return *this;
+ }
+
+ DenseMap &operator=(DenseMap &&other) {
+ this->destroyAll();
+ deallocate_buffer(Buckets, sizeof(BucketT) * NumBuckets, alignof(BucketT));
+ init(0);
+ swap(other);
+ return *this;
+ }
+
+ void copyFrom(const DenseMap &other) {
+ this->destroyAll();
+ deallocate_buffer(Buckets, sizeof(BucketT) * NumBuckets);
+ if (allocateBuckets(other.NumBuckets)) {
+ this->BaseT::copyFrom(other);
+ } else {
+ NumEntries = 0;
+ NumTombstones = 0;
+ }
+ }
+
+ void init(unsigned InitNumEntries) {
+ auto InitBuckets = BaseT::getMinBucketToReserveForEntries(InitNumEntries);
+ if (allocateBuckets(InitBuckets)) {
+ this->BaseT::initEmpty();
+ } else {
+ NumEntries = 0;
+ NumTombstones = 0;
+ }
+ }
+
+ void grow(unsigned AtLeast) {
+ unsigned OldNumBuckets = NumBuckets;
+ BucketT *OldBuckets = Buckets;
+
+ allocateBuckets(RoundUpToPowerOfTwo(Max<unsigned>(64, AtLeast)));
+ CHECK(Buckets);
+ if (!OldBuckets) {
+ this->BaseT::initEmpty();
+ return;
+ }
+
+ this->moveFromOldBuckets(OldBuckets, OldBuckets + OldNumBuckets);
+
+ // Free the old table.
+ deallocate_buffer(OldBuckets, sizeof(BucketT) * OldNumBuckets);
+ }
+
+ private:
+ unsigned getNumEntries() const { return NumEntries; }
+
+ void setNumEntries(unsigned Num) { NumEntries = Num; }
+
+ unsigned getNumTombstones() const { return NumTombstones; }
+
+ void setNumTombstones(unsigned Num) { NumTombstones = Num; }
+
+ BucketT *getBuckets() const { return Buckets; }
+
+ unsigned getNumBuckets() const { return NumBuckets; }
+
+ bool allocateBuckets(unsigned Num) {
+ NumBuckets = Num;
+ if (NumBuckets == 0) {
+ Buckets = nullptr;
+ return false;
+ }
+
+ uptr Size = sizeof(BucketT) * NumBuckets;
+ if (Size * 2 <= GetPageSizeCached()) {
+ // We always allocate at least a page, so use entire space.
+ unsigned Log2 = MostSignificantSetBitIndex(GetPageSizeCached() / Size);
+ Size <<= Log2;
+ NumBuckets <<= Log2;
+ CHECK_EQ(Size, sizeof(BucketT) * NumBuckets);
+ CHECK_GT(Size * 2, GetPageSizeCached());
+ }
+ Buckets = static_cast<BucketT *>(allocate_buffer(Size));
+ return true;
+ }
+
+ static void *allocate_buffer(uptr Size) {
+ return MmapOrDie(RoundUpTo(Size, GetPageSizeCached()), "DenseMap");
+ }
+
+ static void deallocate_buffer(void *Ptr, uptr Size) {
+ UnmapOrDie(Ptr, RoundUpTo(Size, GetPageSizeCached()));
+ }
+};
+
+} // namespace __sanitizer
+
+#endif // SANITIZER_DENSE_MAP_H
--- /dev/null
+//===- sanitizer_dense_map_info.h - Type traits for DenseMap ----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_DENSE_MAP_INFO_H
+#define SANITIZER_DENSE_MAP_INFO_H
+
+#include "sanitizer_common.h"
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_type_traits.h"
+
+namespace __sanitizer {
+
+namespace detail {
+
+/// Simplistic combination of 32-bit hash values into 32-bit hash values.
+static constexpr unsigned combineHashValue(unsigned a, unsigned b) {
+ u64 key = (u64)a << 32 | (u64)b;
+ key += ~(key << 32);
+ key ^= (key >> 22);
+ key += ~(key << 13);
+ key ^= (key >> 8);
+ key += (key << 3);
+ key ^= (key >> 15);
+ key += ~(key << 27);
+ key ^= (key >> 31);
+ return (unsigned)key;
+}
+
+// We extend a pair to allow users to override the bucket type with their own
+// implementation without requiring two members.
+template <typename KeyT, typename ValueT>
+struct DenseMapPair {
+ KeyT first = {};
+ ValueT second = {};
+ constexpr DenseMapPair() = default;
+ constexpr DenseMapPair(const KeyT &f, const ValueT &s)
+ : first(f), second(s) {}
+
+ template <typename KeyT2, typename ValueT2>
+ constexpr DenseMapPair(KeyT2 &&f, ValueT2 &&s)
+ : first(__sanitizer::forward<KeyT2>(f)),
+ second(__sanitizer::forward<ValueT2>(s)) {}
+
+ constexpr DenseMapPair(const DenseMapPair &other) = default;
+ constexpr DenseMapPair &operator=(const DenseMapPair &other) = default;
+ constexpr DenseMapPair(DenseMapPair &&other) = default;
+ constexpr DenseMapPair &operator=(DenseMapPair &&other) = default;
+
+ KeyT &getFirst() { return first; }
+ const KeyT &getFirst() const { return first; }
+ ValueT &getSecond() { return second; }
+ const ValueT &getSecond() const { return second; }
+};
+
+} // end namespace detail
+
+template <typename T>
+struct DenseMapInfo {
+ // static T getEmptyKey();
+ // static T getTombstoneKey();
+ // static unsigned getHashValue(const T &Val);
+ // static bool isEqual(const T &LHS, const T &RHS);
+};
+
+// Provide DenseMapInfo for all pointers. Come up with sentinel pointer values
+// that are aligned to alignof(T) bytes, but try to avoid requiring T to be
+// complete. This allows clients to instantiate DenseMap<T*, ...> with forward
+// declared key types. Assume that no pointer key type requires more than 4096
+// bytes of alignment.
+template <typename T>
+struct DenseMapInfo<T *> {
+ // The following should hold, but it would require T to be complete:
+ // static_assert(alignof(T) <= (1 << Log2MaxAlign),
+ // "DenseMap does not support pointer keys requiring more than "
+ // "Log2MaxAlign bits of alignment");
+ static constexpr uptr Log2MaxAlign = 12;
+
+ static constexpr T *getEmptyKey() {
+ uptr Val = static_cast<uptr>(-1);
+ Val <<= Log2MaxAlign;
+ return reinterpret_cast<T *>(Val);
+ }
+
+ static constexpr T *getTombstoneKey() {
+ uptr Val = static_cast<uptr>(-2);
+ Val <<= Log2MaxAlign;
+ return reinterpret_cast<T *>(Val);
+ }
+
+ static constexpr unsigned getHashValue(const T *PtrVal) {
+ return (unsigned((uptr)PtrVal) >> 4) ^ (unsigned((uptr)PtrVal) >> 9);
+ }
+
+ static constexpr bool isEqual(const T *LHS, const T *RHS) {
+ return LHS == RHS;
+ }
+};
+
+// Provide DenseMapInfo for chars.
+template <>
+struct DenseMapInfo<char> {
+ static constexpr char getEmptyKey() { return ~0; }
+ static constexpr char getTombstoneKey() { return ~0 - 1; }
+ static constexpr unsigned getHashValue(const char &Val) { return Val * 37U; }
+
+ static constexpr bool isEqual(const char &LHS, const char &RHS) {
+ return LHS == RHS;
+ }
+};
+
+// Provide DenseMapInfo for unsigned chars.
+template <>
+struct DenseMapInfo<unsigned char> {
+ static constexpr unsigned char getEmptyKey() { return ~0; }
+ static constexpr unsigned char getTombstoneKey() { return ~0 - 1; }
+ static constexpr unsigned getHashValue(const unsigned char &Val) {
+ return Val * 37U;
+ }
+
+ static constexpr bool isEqual(const unsigned char &LHS,
+ const unsigned char &RHS) {
+ return LHS == RHS;
+ }
+};
+
+// Provide DenseMapInfo for unsigned shorts.
+template <>
+struct DenseMapInfo<unsigned short> {
+ static constexpr unsigned short getEmptyKey() { return 0xFFFF; }
+ static constexpr unsigned short getTombstoneKey() { return 0xFFFF - 1; }
+ static constexpr unsigned getHashValue(const unsigned short &Val) {
+ return Val * 37U;
+ }
+
+ static constexpr bool isEqual(const unsigned short &LHS,
+ const unsigned short &RHS) {
+ return LHS == RHS;
+ }
+};
+
+// Provide DenseMapInfo for unsigned ints.
+template <>
+struct DenseMapInfo<unsigned> {
+ static constexpr unsigned getEmptyKey() { return ~0U; }
+ static constexpr unsigned getTombstoneKey() { return ~0U - 1; }
+ static constexpr unsigned getHashValue(const unsigned &Val) {
+ return Val * 37U;
+ }
+
+ static constexpr bool isEqual(const unsigned &LHS, const unsigned &RHS) {
+ return LHS == RHS;
+ }
+};
+
+// Provide DenseMapInfo for unsigned longs.
+template <>
+struct DenseMapInfo<unsigned long> {
+ static constexpr unsigned long getEmptyKey() { return ~0UL; }
+ static constexpr unsigned long getTombstoneKey() { return ~0UL - 1L; }
+
+ static constexpr unsigned getHashValue(const unsigned long &Val) {
+ return (unsigned)(Val * 37UL);
+ }
+
+ static constexpr bool isEqual(const unsigned long &LHS,
+ const unsigned long &RHS) {
+ return LHS == RHS;
+ }
+};
+
+// Provide DenseMapInfo for unsigned long longs.
+template <>
+struct DenseMapInfo<unsigned long long> {
+ static constexpr unsigned long long getEmptyKey() { return ~0ULL; }
+ static constexpr unsigned long long getTombstoneKey() { return ~0ULL - 1ULL; }
+
+ static constexpr unsigned getHashValue(const unsigned long long &Val) {
+ return (unsigned)(Val * 37ULL);
+ }
+
+ static constexpr bool isEqual(const unsigned long long &LHS,
+ const unsigned long long &RHS) {
+ return LHS == RHS;
+ }
+};
+
+// Provide DenseMapInfo for shorts.
+template <>
+struct DenseMapInfo<short> {
+ static constexpr short getEmptyKey() { return 0x7FFF; }
+ static constexpr short getTombstoneKey() { return -0x7FFF - 1; }
+ static constexpr unsigned getHashValue(const short &Val) { return Val * 37U; }
+ static constexpr bool isEqual(const short &LHS, const short &RHS) {
+ return LHS == RHS;
+ }
+};
+
+// Provide DenseMapInfo for ints.
+template <>
+struct DenseMapInfo<int> {
+ static constexpr int getEmptyKey() { return 0x7fffffff; }
+ static constexpr int getTombstoneKey() { return -0x7fffffff - 1; }
+ static constexpr unsigned getHashValue(const int &Val) {
+ return (unsigned)(Val * 37U);
+ }
+
+ static constexpr bool isEqual(const int &LHS, const int &RHS) {
+ return LHS == RHS;
+ }
+};
+
+// Provide DenseMapInfo for longs.
+template <>
+struct DenseMapInfo<long> {
+ static constexpr long getEmptyKey() {
+ return (1UL << (sizeof(long) * 8 - 1)) - 1UL;
+ }
+
+ static constexpr long getTombstoneKey() { return getEmptyKey() - 1L; }
+
+ static constexpr unsigned getHashValue(const long &Val) {
+ return (unsigned)(Val * 37UL);
+ }
+
+ static constexpr bool isEqual(const long &LHS, const long &RHS) {
+ return LHS == RHS;
+ }
+};
+
+// Provide DenseMapInfo for long longs.
+template <>
+struct DenseMapInfo<long long> {
+ static constexpr long long getEmptyKey() { return 0x7fffffffffffffffLL; }
+ static constexpr long long getTombstoneKey() {
+ return -0x7fffffffffffffffLL - 1;
+ }
+
+ static constexpr unsigned getHashValue(const long long &Val) {
+ return (unsigned)(Val * 37ULL);
+ }
+
+ static constexpr bool isEqual(const long long &LHS, const long long &RHS) {
+ return LHS == RHS;
+ }
+};
+
+// Provide DenseMapInfo for all pairs whose members have info.
+template <typename T, typename U>
+struct DenseMapInfo<detail::DenseMapPair<T, U>> {
+ using Pair = detail::DenseMapPair<T, U>;
+ using FirstInfo = DenseMapInfo<T>;
+ using SecondInfo = DenseMapInfo<U>;
+
+ static constexpr Pair getEmptyKey() {
+ return detail::DenseMapPair<T, U>(FirstInfo::getEmptyKey(),
+ SecondInfo::getEmptyKey());
+ }
+
+ static constexpr Pair getTombstoneKey() {
+ return detail::DenseMapPair<T, U>(FirstInfo::getTombstoneKey(),
+ SecondInfo::getTombstoneKey());
+ }
+
+ static constexpr unsigned getHashValue(const Pair &PairVal) {
+ return detail::combineHashValue(FirstInfo::getHashValue(PairVal.first),
+ SecondInfo::getHashValue(PairVal.second));
+ }
+
+ static constexpr bool isEqual(const Pair &LHS, const Pair &RHS) {
+ return FirstInfo::isEqual(LHS.first, RHS.first) &&
+ SecondInfo::isEqual(LHS.second, RHS.second);
+ }
+};
+
+} // namespace __sanitizer
+
+#endif // SANITIZER_DENSE_MAP_INFO_H
#include "sanitizer_errno_codes.h"
#include "sanitizer_platform.h"
-#if SANITIZER_FREEBSD || SANITIZER_MAC
+#if SANITIZER_FREEBSD || SANITIZER_APPLE
# define __errno_location __error
#elif SANITIZER_ANDROID || SANITIZER_NETBSD
# define __errno_location __errno
#define errno_EBUSY 16
#define errno_EINVAL 22
#define errno_ENAMETOOLONG 36
+#define errno_ENOSYS 38
// Those might not present or their value differ on different platforms.
extern const int errno_EOWNERDEAD;
#include "sanitizer_common.h"
#include "sanitizer_file.h"
+# include "sanitizer_interface_internal.h"
namespace __sanitizer {
fd_pid = pid;
}
+static void RecursiveCreateParentDirs(char *path) {
+ if (path[0] == '\0')
+ return;
+ for (int i = 1; path[i] != '\0'; ++i) {
+ char save = path[i];
+ if (!IsPathSeparator(path[i]))
+ continue;
+ path[i] = '\0';
+ if (!DirExists(path) && !CreateDir(path)) {
+ const char *ErrorMsgPrefix = "ERROR: Can't create directory: ";
+ WriteToFile(kStderrFd, ErrorMsgPrefix, internal_strlen(ErrorMsgPrefix));
+ WriteToFile(kStderrFd, path, internal_strlen(path));
+ Die();
+ }
+ path[i] = save;
+ }
+}
+
void ReportFile::SetReportPath(const char *path) {
if (path) {
uptr len = internal_strlen(path);
fd = kStdoutFd;
} else {
internal_snprintf(path_prefix, kMaxPathLength, "%s", path);
+ RecursiveCreateParentDirs(path_prefix);
}
}
#ifndef SANITIZER_FILE_H
#define SANITIZER_FILE_H
-#include "sanitizer_interface_internal.h"
#include "sanitizer_internal_defs.h"
#include "sanitizer_libc.h"
#include "sanitizer_mutex.h"
// OS
const char *GetPwd();
bool FileExists(const char *filename);
+bool DirExists(const char *path);
char *FindPathToBinary(const char *name);
bool IsPathSeparator(const char c);
bool IsAbsolutePath(const char *path);
+// Returns true on success, false on failure.
+bool CreateDir(const char *pathname);
// Starts a subprocess and returs its pid.
// If *_fd parameters are not kInvalidFd their corresponding input/output
// streams will be redirect to the file. The files will always be closed
template <>
inline bool FlagHandler<uptr>::Format(char *buffer, uptr size) {
- uptr num_symbols_should_write = internal_snprintf(buffer, size, "%p", *t_);
+ uptr num_symbols_should_write = internal_snprintf(buffer, size, "0x%zx", *t_);
return num_symbols_should_write < size;
}
COMMON_FLAG(const char *, log_suffix, nullptr,
"String to append to log file name, e.g. \".txt\".")
COMMON_FLAG(
- bool, log_to_syslog, (bool)SANITIZER_ANDROID || (bool)SANITIZER_MAC,
+ bool, log_to_syslog, (bool)SANITIZER_ANDROID || (bool)SANITIZER_APPLE,
"Write all sanitizer output to syslog in addition to other means of "
"logging.")
COMMON_FLAG(
int, verbosity, 0,
"Verbosity level (0 - silent, 1 - a bit of output, 2+ - more output).")
-COMMON_FLAG(bool, strip_env, 1,
+COMMON_FLAG(bool, strip_env, true,
"Whether to remove the sanitizer from DYLD_INSERT_LIBRARIES to "
- "avoid passing it to children. Default is true.")
-COMMON_FLAG(bool, detect_leaks, !SANITIZER_MAC, "Enable memory leak detection.")
+ "avoid passing it to children on Apple platforms. Default is true.")
+COMMON_FLAG(bool, verify_interceptors, true,
+ "Verify that interceptors are working on Apple platforms. Default "
+ "is true.")
+COMMON_FLAG(bool, detect_leaks, !SANITIZER_APPLE, "Enable memory leak detection.")
COMMON_FLAG(
bool, leak_check_at_exit, true,
"Invoke leak checking in an atexit handler. Has no effect if "
COMMON_FLAG(const char *, coverage_dir, ".",
"Target directory for coverage dumps. Defaults to the current "
"directory.")
+COMMON_FLAG(const char *, cov_8bit_counters_out, "",
+ "If non-empty, write 8bit counters to this file. ")
+COMMON_FLAG(const char *, cov_pcs_out, "",
+ "If non-empty, write the coverage pc table to this file. ")
COMMON_FLAG(bool, full_address_space, false,
"Sanitize complete address space; "
"by default kernel area on 32-bit platforms will not be sanitized")
"in core file.")
COMMON_FLAG(bool, symbolize_inline_frames, true,
"Print inlined frames in stacktraces. Defaults to true.")
+COMMON_FLAG(bool, demangle, true, "Print demangled symbols.")
COMMON_FLAG(bool, symbolize_vs_style, false,
"Print file locations in Visual Studio style (e.g: "
" file(10,42): ...")
"Format string used to render stack frames. "
"See sanitizer_stacktrace_printer.h for the format description. "
"Use DEFAULT to get default format.")
+COMMON_FLAG(int, compress_stack_depot, 0,
+ "Compress stack depot to save memory.")
COMMON_FLAG(bool, no_huge_pages_for_shadow, true,
"If true, the shadow is not allowed to use huge pages. ")
COMMON_FLAG(bool, strict_string_checks, false,
COMMON_FLAG(int, exitcode, 1, "Override the program exit status if the tool "
"found an error")
COMMON_FLAG(
- bool, abort_on_error, (bool)SANITIZER_ANDROID || (bool)SANITIZER_MAC,
+ bool, abort_on_error, (bool)SANITIZER_ANDROID || (bool)SANITIZER_APPLE,
"If set, the tool calls abort() instead of _exit() after printing the "
"error report.")
COMMON_FLAG(bool, suppress_equal_pcs, true,
--- /dev/null
+//===-- sanitizer_flat_map.h ------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Part of the Sanitizer Allocator.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_FLAT_MAP_H
+#define SANITIZER_FLAT_MAP_H
+
+#include "sanitizer_atomic.h"
+#include "sanitizer_common.h"
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_local_address_space_view.h"
+#include "sanitizer_mutex.h"
+
+namespace __sanitizer {
+
+// Call these callbacks on mmap/munmap.
+struct NoOpMapUnmapCallback {
+ void OnMap(uptr p, uptr size) const {}
+ void OnUnmap(uptr p, uptr size) const {}
+};
+
+// Maps integers in rage [0, kSize) to values.
+template <typename T, u64 kSize,
+ typename AddressSpaceViewTy = LocalAddressSpaceView>
+class FlatMap {
+ public:
+ using AddressSpaceView = AddressSpaceViewTy;
+ void Init() { internal_memset(map_, 0, sizeof(map_)); }
+
+ constexpr uptr size() const { return kSize; }
+
+ bool contains(uptr idx) const {
+ CHECK_LT(idx, kSize);
+ return true;
+ }
+
+ T &operator[](uptr idx) {
+ DCHECK_LT(idx, kSize);
+ return map_[idx];
+ }
+
+ const T &operator[](uptr idx) const {
+ DCHECK_LT(idx, kSize);
+ return map_[idx];
+ }
+
+ private:
+ T map_[kSize];
+};
+
+// TwoLevelMap maps integers in range [0, kSize1*kSize2) to values.
+// It is implemented as a two-dimensional array: array of kSize1 pointers
+// to kSize2-byte arrays. The secondary arrays are mmaped on demand.
+// Each value is initially zero and can be set to something else only once.
+// Setting and getting values from multiple threads is safe w/o extra locking.
+template <typename T, u64 kSize1, u64 kSize2,
+ typename AddressSpaceViewTy = LocalAddressSpaceView,
+ class MapUnmapCallback = NoOpMapUnmapCallback>
+class TwoLevelMap {
+ static_assert(IsPowerOfTwo(kSize2), "Use a power of two for performance.");
+
+ public:
+ using AddressSpaceView = AddressSpaceViewTy;
+ void Init() {
+ mu_.Init();
+ internal_memset(map1_, 0, sizeof(map1_));
+ }
+
+ void TestOnlyUnmap() {
+ for (uptr i = 0; i < kSize1; i++) {
+ T *p = Get(i);
+ if (!p)
+ continue;
+ MapUnmapCallback().OnUnmap(reinterpret_cast<uptr>(p), MmapSize());
+ UnmapOrDie(p, kSize2);
+ }
+ Init();
+ }
+
+ uptr MemoryUsage() const {
+ uptr res = 0;
+ for (uptr i = 0; i < kSize1; i++) {
+ T *p = Get(i);
+ if (!p)
+ continue;
+ res += MmapSize();
+ }
+ return res;
+ }
+
+ constexpr uptr size() const { return kSize1 * kSize2; }
+ constexpr uptr size1() const { return kSize1; }
+ constexpr uptr size2() const { return kSize2; }
+
+ bool contains(uptr idx) const {
+ CHECK_LT(idx, kSize1 * kSize2);
+ return Get(idx / kSize2);
+ }
+
+ const T &operator[](uptr idx) const {
+ DCHECK_LT(idx, kSize1 * kSize2);
+ T *map2 = GetOrCreate(idx / kSize2);
+ return *AddressSpaceView::Load(&map2[idx % kSize2]);
+ }
+
+ T &operator[](uptr idx) {
+ DCHECK_LT(idx, kSize1 * kSize2);
+ T *map2 = GetOrCreate(idx / kSize2);
+ return *AddressSpaceView::LoadWritable(&map2[idx % kSize2]);
+ }
+
+ private:
+ constexpr uptr MmapSize() const {
+ return RoundUpTo(kSize2 * sizeof(T), GetPageSizeCached());
+ }
+
+ T *Get(uptr idx) const {
+ DCHECK_LT(idx, kSize1);
+ return reinterpret_cast<T *>(
+ atomic_load(&map1_[idx], memory_order_acquire));
+ }
+
+ T *GetOrCreate(uptr idx) const {
+ DCHECK_LT(idx, kSize1);
+ // This code needs to use memory_order_acquire/consume, but we use
+ // memory_order_relaxed for performance reasons (matters for arm64). We
+ // expect memory_order_relaxed to be effectively equivalent to
+ // memory_order_consume in this case for all relevant architectures: all
+ // dependent data is reachable only by dereferencing the resulting pointer.
+ // If relaxed load fails to see stored ptr, the code will fall back to
+ // Create() and reload the value again with locked mutex as a memory
+ // barrier.
+ T *res = reinterpret_cast<T *>(atomic_load_relaxed(&map1_[idx]));
+ if (LIKELY(res))
+ return res;
+ return Create(idx);
+ }
+
+ NOINLINE T *Create(uptr idx) const {
+ SpinMutexLock l(&mu_);
+ T *res = Get(idx);
+ if (!res) {
+ res = reinterpret_cast<T *>(MmapOrDie(MmapSize(), "TwoLevelMap"));
+ MapUnmapCallback().OnMap(reinterpret_cast<uptr>(res), kSize2);
+ atomic_store(&map1_[idx], reinterpret_cast<uptr>(res),
+ memory_order_release);
+ }
+ return res;
+ }
+
+ mutable StaticSpinMutex mu_;
+ mutable atomic_uintptr_t map1_[kSize1];
+};
+
+template <u64 kSize, typename AddressSpaceViewTy = LocalAddressSpaceView>
+using FlatByteMap = FlatMap<u8, kSize, AddressSpaceViewTy>;
+
+template <u64 kSize1, u64 kSize2,
+ typename AddressSpaceViewTy = LocalAddressSpaceView,
+ class MapUnmapCallback = NoOpMapUnmapCallback>
+using TwoLevelByteMap =
+ TwoLevelMap<u8, kSize1, kSize2, AddressSpaceViewTy, MapUnmapCallback>;
+} // namespace __sanitizer
+
+#endif
#include "sanitizer_fuchsia.h"
#if SANITIZER_FUCHSIA
-#include <pthread.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <zircon/errors.h>
-#include <zircon/process.h>
-#include <zircon/syscalls.h>
-#include <zircon/utc.h>
-
-#include "sanitizer_common.h"
-#include "sanitizer_libc.h"
-#include "sanitizer_mutex.h"
+# include <pthread.h>
+# include <stdlib.h>
+# include <unistd.h>
+# include <zircon/errors.h>
+# include <zircon/process.h>
+# include <zircon/syscalls.h>
+# include <zircon/utc.h>
+
+# include "sanitizer_common.h"
+# include "sanitizer_interface_internal.h"
+# include "sanitizer_libc.h"
+# include "sanitizer_mutex.h"
namespace __sanitizer {
void NORETURN internal__exit(int exitcode) { _zx_process_exit(exitcode); }
uptr internal_sched_yield() {
- zx_status_t status = _zx_nanosleep(0);
+ zx_status_t status = _zx_thread_legacy_yield(0u);
CHECK_EQ(status, ZX_OK);
return 0; // Why doesn't this return void?
}
}
void InitializePlatformEarly() {}
-void MaybeReexec() {}
void CheckASLR() {}
void CheckMPROTECT() {}
-void PlatformPrepareForSandboxing(__sanitizer_sandbox_arguments *args) {}
+void PlatformPrepareForSandboxing(void *args) {}
void DisableCoreDumperIfNecessary() {}
void InstallDeadlySignalHandlers(SignalHandlerType handler) {}
void SetAlternateSignalStack() {}
CHECK_EQ(status, ZX_OK);
}
-enum MutexState : int { MtxUnlocked = 0, MtxLocked = 1, MtxSleeping = 2 };
-
-BlockingMutex::BlockingMutex() {
- // NOTE! It's important that this use internal_memset, because plain
- // memset might be intercepted (e.g., actually be __asan_memset).
- // Defining this so the compiler initializes each field, e.g.:
- // BlockingMutex::BlockingMutex() : BlockingMutex(LINKER_INITIALIZED) {}
- // might result in the compiler generating a call to memset, which would
- // have the same problem.
- internal_memset(this, 0, sizeof(*this));
-}
-
-void BlockingMutex::Lock() {
- CHECK_EQ(owner_, 0);
- atomic_uint32_t *m = reinterpret_cast<atomic_uint32_t *>(&opaque_storage_);
- if (atomic_exchange(m, MtxLocked, memory_order_acquire) == MtxUnlocked)
- return;
- while (atomic_exchange(m, MtxSleeping, memory_order_acquire) != MtxUnlocked) {
- zx_status_t status =
- _zx_futex_wait(reinterpret_cast<zx_futex_t *>(m), MtxSleeping,
- ZX_HANDLE_INVALID, ZX_TIME_INFINITE);
- if (status != ZX_ERR_BAD_STATE) // Normal race.
- CHECK_EQ(status, ZX_OK);
- }
-}
-
-void BlockingMutex::Unlock() {
- atomic_uint32_t *m = reinterpret_cast<atomic_uint32_t *>(&opaque_storage_);
- u32 v = atomic_exchange(m, MtxUnlocked, memory_order_release);
- CHECK_NE(v, MtxUnlocked);
- if (v == MtxSleeping) {
- zx_status_t status = _zx_futex_wake(reinterpret_cast<zx_futex_t *>(m), 1);
- CHECK_EQ(status, ZX_OK);
- }
-}
-
-void BlockingMutex::CheckLocked() const {
- auto m = reinterpret_cast<atomic_uint32_t const *>(&opaque_storage_);
- CHECK_NE(MtxUnlocked, atomic_load(m, memory_order_relaxed));
-}
-
uptr GetPageSize() { return _zx_system_get_page_size(); }
uptr GetMmapGranularity() { return _zx_system_get_page_size(); }
uptr GetMaxVirtualAddress() { return GetMaxUserVirtualAddress(); }
+bool ErrorIsOOM(error_t err) { return err == ZX_ERR_NO_MEMORY; }
+
static void *DoAnonymousMmapOrDie(uptr size, const char *mem_type,
bool raw_report, bool die_for_nomem) {
size = RoundUpTo(size, GetPageSize());
UNIMPLEMENTED();
}
+bool MprotectNoAccess(uptr addr, uptr size) {
+ return _zx_vmar_protect(_zx_vmar_root_self(), 0, addr, size) == ZX_OK;
+}
+
+bool MprotectReadOnly(uptr addr, uptr size) {
+ return _zx_vmar_protect(_zx_vmar_root_self(), ZX_VM_PERM_READ, addr, size) ==
+ ZX_OK;
+}
+
void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment,
const char *mem_type) {
CHECK_GE(size, GetPageSize());
}
// FIXME implement on this platform.
-void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size) {}
+void GetMemoryProfile(fill_profile_f cb, uptr *stats) {}
bool ReadFileToBuffer(const char *file_name, char **buff, uptr *buff_size,
uptr *read_len, uptr max_len, error_t *errno_p) {
- zx_handle_t vmo;
- zx_status_t status = __sanitizer_get_configuration(file_name, &vmo);
- if (status == ZX_OK) {
- uint64_t vmo_size;
- status = _zx_vmo_get_size(vmo, &vmo_size);
- if (status == ZX_OK) {
- if (vmo_size < max_len)
- max_len = vmo_size;
- size_t map_size = RoundUpTo(max_len, GetPageSize());
- uintptr_t addr;
- status = _zx_vmar_map(_zx_vmar_root_self(), ZX_VM_PERM_READ, 0, vmo, 0,
- map_size, &addr);
- if (status == ZX_OK) {
- *buff = reinterpret_cast<char *>(addr);
- *buff_size = map_size;
- *read_len = max_len;
- }
- }
- _zx_handle_close(vmo);
- }
- if (status != ZX_OK && errno_p)
- *errno_p = status;
- return status == ZX_OK;
+ *errno_p = ZX_ERR_NOT_SUPPORTED;
+ return false;
}
void RawWrite(const char *buffer) {
uptr GetRSS() { UNIMPLEMENTED(); }
+void *internal_start_thread(void *(*func)(void *arg), void *arg) { return 0; }
+void internal_join_thread(void *th) {}
+
void InitializePlatformCommonFlags(CommonFlags *cf) {}
} // namespace __sanitizer
return x;
}
};
+
+class MurMur2Hash64Builder {
+ static const u64 m = 0xc6a4a7935bd1e995ull;
+ static const u64 seed = 0x9747b28c9747b28cull;
+ static const u64 r = 47;
+ u64 h;
+
+ public:
+ explicit MurMur2Hash64Builder(u64 init = 0) { h = seed ^ (init * m); }
+ void add(u64 k) {
+ k *= m;
+ k ^= k >> r;
+ k *= m;
+ h ^= k;
+ h *= m;
+ }
+ u64 get() {
+ u64 x = h;
+ x ^= x >> r;
+ x *= m;
+ x ^= x >> r;
+ return x;
+ }
+};
} //namespace __sanitizer
#endif // SANITIZER_HASH_H
_(URIO_SEND_COMMAND, READWRITE, struct_urio_command_sz);
_(URIO_RECV_COMMAND, READWRITE, struct_urio_command_sz);
#undef _
-} // NOLINT
+}
static bool ioctl_initialized = false;
#include "sanitizer_internal_defs.h"
extern "C" {
-
// Tell the tools to write their reports to "path.<pid>" instead of stderr.
- // The special values are "stdout" and "stderr".
- SANITIZER_INTERFACE_ATTRIBUTE
- void __sanitizer_set_report_path(const char *path);
- // Tell the tools to write their reports to the provided file descriptor
- // (casted to void *).
- SANITIZER_INTERFACE_ATTRIBUTE
- void __sanitizer_set_report_fd(void *fd);
- // Get the current full report file path, if a path was specified by
- // an earlier call to __sanitizer_set_report_path. Returns null otherwise.
- SANITIZER_INTERFACE_ATTRIBUTE
- const char *__sanitizer_get_report_path();
+// Tell the tools to write their reports to "path.<pid>" instead of stderr.
+// The special values are "stdout" and "stderr".
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_set_report_path(const char *path);
+// Tell the tools to write their reports to the provided file descriptor
+// (casted to void *).
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_set_report_fd(void *fd);
+// Get the current full report file path, if a path was specified by
+// an earlier call to __sanitizer_set_report_path. Returns null otherwise.
+SANITIZER_INTERFACE_ATTRIBUTE
+const char *__sanitizer_get_report_path();
- typedef struct {
- int coverage_sandboxed;
- __sanitizer::sptr coverage_fd;
- unsigned int coverage_max_block_size;
- } __sanitizer_sandbox_arguments;
+typedef struct {
+ int coverage_sandboxed;
+ __sanitizer::sptr coverage_fd;
+ unsigned int coverage_max_block_size;
+} __sanitizer_sandbox_arguments;
- // Notify the tools that the sandbox is going to be turned on.
- SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
- __sanitizer_sandbox_on_notify(__sanitizer_sandbox_arguments *args);
+// Notify the tools that the sandbox is going to be turned on.
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+__sanitizer_sandbox_on_notify(__sanitizer_sandbox_arguments *args);
- // This function is called by the tool when it has just finished reporting
- // an error. 'error_summary' is a one-line string that summarizes
- // the error message. This function can be overridden by the client.
- SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
- void __sanitizer_report_error_summary(const char *error_summary);
+// This function is called by the tool when it has just finished reporting
+// an error. 'error_summary' is a one-line string that summarizes
+// the error message. This function can be overridden by the client.
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+__sanitizer_report_error_summary(const char *error_summary);
- SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump();
- SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_dump_coverage(
- const __sanitizer::uptr *pcs, const __sanitizer::uptr len);
- SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_dump_trace_pc_guard_coverage();
+SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump();
+SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_dump_coverage(
+ const __sanitizer::uptr *pcs, const __sanitizer::uptr len);
+SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_dump_trace_pc_guard_coverage();
- SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov(__sanitizer::u32 *guard);
+SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov(__sanitizer::u32 *guard);
- // Returns 1 on the first call, then returns 0 thereafter. Called by the tool
- // to ensure only one report is printed when multiple errors occur
- // simultaneously.
- SANITIZER_INTERFACE_ATTRIBUTE int __sanitizer_acquire_crash_state();
+// Returns 1 on the first call, then returns 0 thereafter. Called by the tool
+// to ensure only one report is printed when multiple errors occur
+// simultaneously.
+SANITIZER_INTERFACE_ATTRIBUTE int __sanitizer_acquire_crash_state();
- SANITIZER_INTERFACE_ATTRIBUTE
- void __sanitizer_annotate_contiguous_container(const void *beg,
- const void *end,
- const void *old_mid,
- const void *new_mid);
- SANITIZER_INTERFACE_ATTRIBUTE
- int __sanitizer_verify_contiguous_container(const void *beg, const void *mid,
- const void *end);
- SANITIZER_INTERFACE_ATTRIBUTE
- const void *__sanitizer_contiguous_container_find_bad_address(
- const void *beg, const void *mid, const void *end);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_annotate_contiguous_container(const void *beg, const void *end,
+ const void *old_mid,
+ const void *new_mid);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_annotate_double_ended_contiguous_container(
+ const void *storage_beg, const void *storage_end,
+ const void *old_container_beg, const void *old_container_end,
+ const void *new_container_beg, const void *new_container_end);
+SANITIZER_INTERFACE_ATTRIBUTE
+int __sanitizer_verify_contiguous_container(const void *beg, const void *mid,
+ const void *end);
+SANITIZER_INTERFACE_ATTRIBUTE
+int __sanitizer_verify_double_ended_contiguous_container(
+ const void *storage_beg, const void *container_beg,
+ const void *container_end, const void *storage_end);
+SANITIZER_INTERFACE_ATTRIBUTE
+const void *__sanitizer_contiguous_container_find_bad_address(const void *beg,
+ const void *mid,
+ const void *end);
+SANITIZER_INTERFACE_ATTRIBUTE
+const void *__sanitizer_double_ended_contiguous_container_find_bad_address(
+ const void *storage_beg, const void *container_beg,
+ const void *container_end, const void *storage_end);
- SANITIZER_INTERFACE_ATTRIBUTE
- int __sanitizer_get_module_and_offset_for_pc(
- __sanitizer::uptr pc, char *module_path,
- __sanitizer::uptr module_path_len, __sanitizer::uptr *pc_offset);
-
- SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
- void __sanitizer_cov_trace_cmp();
- SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
- void __sanitizer_cov_trace_cmp1();
- SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
- void __sanitizer_cov_trace_cmp2();
- SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
- void __sanitizer_cov_trace_cmp4();
- SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
- void __sanitizer_cov_trace_cmp8();
- SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
- void __sanitizer_cov_trace_const_cmp1();
- SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
- void __sanitizer_cov_trace_const_cmp2();
- SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
- void __sanitizer_cov_trace_const_cmp4();
- SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
- void __sanitizer_cov_trace_const_cmp8();
- SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
- void __sanitizer_cov_trace_switch();
- SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
- void __sanitizer_cov_trace_div4();
- SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
- void __sanitizer_cov_trace_div8();
- SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
- void __sanitizer_cov_trace_gep();
- SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
- void __sanitizer_cov_trace_pc_indir();
- SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
- void __sanitizer_cov_trace_pc_guard(__sanitizer::u32*);
- SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
- void __sanitizer_cov_trace_pc_guard_init(__sanitizer::u32*,
- __sanitizer::u32*);
- SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
- void __sanitizer_cov_8bit_counters_init();
- SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
- __sanitizer_cov_bool_flag_init();
- SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
- __sanitizer_cov_pcs_init();
-} // extern "C"
+SANITIZER_INTERFACE_ATTRIBUTE
+int __sanitizer_get_module_and_offset_for_pc(void *pc, char *module_path,
+ __sanitizer::uptr module_path_len,
+ void **pc_offset);
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+__sanitizer_cov_trace_cmp();
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+__sanitizer_cov_trace_cmp1();
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+__sanitizer_cov_trace_cmp2();
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+__sanitizer_cov_trace_cmp4();
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+__sanitizer_cov_trace_cmp8();
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+__sanitizer_cov_trace_const_cmp1();
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+__sanitizer_cov_trace_const_cmp2();
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+__sanitizer_cov_trace_const_cmp4();
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+__sanitizer_cov_trace_const_cmp8();
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+__sanitizer_cov_trace_switch();
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+__sanitizer_cov_trace_div4();
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+__sanitizer_cov_trace_div8();
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+__sanitizer_cov_trace_gep();
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+__sanitizer_cov_trace_pc_indir();
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+__sanitizer_cov_load1();
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+__sanitizer_cov_load2();
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+__sanitizer_cov_load4();
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+__sanitizer_cov_load8();
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+__sanitizer_cov_load16();
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+__sanitizer_cov_store1();
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+__sanitizer_cov_store2();
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+__sanitizer_cov_store4();
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+__sanitizer_cov_store8();
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+__sanitizer_cov_store16();
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+__sanitizer_cov_trace_pc_guard(__sanitizer::u32 *);
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+__sanitizer_cov_trace_pc_guard_init(__sanitizer::u32 *, __sanitizer::u32 *);
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+__sanitizer_cov_8bit_counters_init(char *, char *);
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+__sanitizer_cov_bool_flag_init();
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+__sanitizer_cov_pcs_init(const __sanitizer::uptr *, const __sanitizer::uptr *);
+} // extern "C"
#endif // SANITIZER_INTERFACE_INTERNAL_H
// Before Xcode 4.5, the Darwin linker doesn't reliably support undefined
// weak symbols. Mac OS X 10.9/Darwin 13 is the first release only supported
// by Xcode >= 4.5.
-#elif SANITIZER_MAC && \
+#elif SANITIZER_APPLE && \
__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1090 && !SANITIZER_GO
# define SANITIZER_SUPPORTS_WEAK_HOOKS 1
#else
# define __has_attribute(x) 0
#endif
+#if !defined(__has_cpp_attribute)
+# define __has_cpp_attribute(x) 0
+#endif
+
// For portability reasons we do not include stddef.h, stdint.h or any other
// system header, but we do need some basic types that are not defined
// in a portable way by the language itself.
typedef unsigned long long uptr;
typedef signed long long sptr;
#else
+# if (SANITIZER_WORDSIZE == 64) || SANITIZER_APPLE || SANITIZER_WINDOWS
typedef unsigned long uptr;
typedef signed long sptr;
+# else
+typedef unsigned int uptr;
+typedef signed int sptr;
+# endif
#endif // defined(_WIN64)
#if defined(__x86_64__)
// Since x32 uses ILP32 data model in 64-bit hardware mode, we must use
typedef int pid_t;
#endif
-#if SANITIZER_FREEBSD || SANITIZER_NETBSD || \
- SANITIZER_MAC || \
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_APPLE || \
(SANITIZER_SOLARIS && (defined(_LP64) || _FILE_OFFSET_BITS == 64)) || \
- (SANITIZER_LINUX && defined(__x86_64__))
+ (SANITIZER_LINUX && !SANITIZER_GLIBC && !SANITIZER_ANDROID) || \
+ (SANITIZER_LINUX && (defined(__x86_64__) || defined(__hexagon__)))
typedef u64 OFF_T;
#else
typedef uptr OFF_T;
#endif
typedef u64 OFF64_T;
-#if (SANITIZER_WORDSIZE == 64) || SANITIZER_MAC
+#if (SANITIZER_WORDSIZE == 64) || SANITIZER_APPLE
typedef uptr operator_new_size_type;
#else
# if defined(__s390__) && !defined(__s390x__)
# define NOEXCEPT throw()
#endif
+#if __has_cpp_attribute(clang::fallthrough)
+# define FALLTHROUGH [[clang::fallthrough]]
+#elif __has_cpp_attribute(fallthrough)
+# define FALLTHROUGH [[fallthrough]]
+#else
+# define FALLTHROUGH
+#endif
+
// Unaligned versions of basic types.
typedef ALIGNED(1) u16 uu16;
typedef ALIGNED(1) u32 uu32;
u64 v1, u64 v2);
// Check macro
-#define RAW_CHECK_MSG(expr, msg) do { \
- if (UNLIKELY(!(expr))) { \
- RawWrite(msg); \
- Die(); \
- } \
-} while (0)
+#define RAW_CHECK_MSG(expr, msg, ...) \
+ do { \
+ if (UNLIKELY(!(expr))) { \
+ const char* msgs[] = {msg, __VA_ARGS__}; \
+ for (const char* m : msgs) RawWrite(m); \
+ Die(); \
+ } \
+ } while (0)
-#define RAW_CHECK(expr) RAW_CHECK_MSG(expr, #expr)
+#define RAW_CHECK(expr) RAW_CHECK_MSG(expr, #expr "\n", )
+#define RAW_CHECK_VA(expr, ...) RAW_CHECK_MSG(expr, #expr "\n", __VA_ARGS__)
#define CHECK_IMPL(c1, op, c2) \
do { \
enum LinkerInitialized { LINKER_INITIALIZED = 0 };
#if !defined(_MSC_VER) || defined(__clang__)
-#if SANITIZER_S390_31
-#define GET_CALLER_PC() \
- (__sanitizer::uptr) __builtin_extract_return_addr(__builtin_return_address(0))
-#else
-#define GET_CALLER_PC() (__sanitizer::uptr) __builtin_return_address(0)
-#endif
-#define GET_CURRENT_FRAME() (__sanitizer::uptr) __builtin_frame_address(0)
+# define GET_CALLER_PC() \
+ ((__sanitizer::uptr)__builtin_extract_return_addr( \
+ __builtin_return_address(0)))
+# define GET_CURRENT_FRAME() ((__sanitizer::uptr)__builtin_frame_address(0))
inline void Trap() {
__builtin_trap();
}
extern "C" void* _AddressOfReturnAddress(void);
# pragma intrinsic(_ReturnAddress)
# pragma intrinsic(_AddressOfReturnAddress)
-#define GET_CALLER_PC() (__sanitizer::uptr) _ReturnAddress()
+# define GET_CALLER_PC() ((__sanitizer::uptr)_ReturnAddress())
// CaptureStackBackTrace doesn't need to know BP on Windows.
-#define GET_CURRENT_FRAME() \
- (((__sanitizer::uptr)_AddressOfReturnAddress()) + sizeof(__sanitizer::uptr))
+# define GET_CURRENT_FRAME() \
+ (((__sanitizer::uptr)_AddressOfReturnAddress()) + sizeof(__sanitizer::uptr))
extern "C" void __ud2(void);
-# pragma intrinsic(__ud2)
+# pragma intrinsic(__ud2)
inline void Trap() {
__ud2();
}
(void)enable_fp; \
} while (0)
-constexpr u32 kInvalidTid = -1;
-constexpr u32 kMainTid = 0;
+// Internal thread identifier allocated by ThreadRegistry.
+typedef u32 Tid;
+constexpr Tid kInvalidTid = -1;
+constexpr Tid kMainTid = 0;
+
+// Stack depot stack identifier.
+typedef u32 StackID;
+const StackID kInvalidStackID = 0;
} // namespace __sanitizer
--- /dev/null
+//===-- sanitizer_leb128.h --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_LEB128_H
+#define SANITIZER_LEB128_H
+
+#include "sanitizer_common.h"
+#include "sanitizer_internal_defs.h"
+
+namespace __sanitizer {
+
+template <typename T, typename It>
+It EncodeSLEB128(T value, It begin, It end) {
+ bool more;
+ do {
+ u8 byte = value & 0x7f;
+ // NOTE: this assumes that this signed shift is an arithmetic right shift.
+ value >>= 7;
+ more = !((((value == 0) && ((byte & 0x40) == 0)) ||
+ ((value == -1) && ((byte & 0x40) != 0))));
+ if (more)
+ byte |= 0x80;
+ if (UNLIKELY(begin == end))
+ break;
+ *(begin++) = byte;
+ } while (more);
+ return begin;
+}
+
+template <typename T, typename It>
+It DecodeSLEB128(It begin, It end, T* v) {
+ T value = 0;
+ unsigned shift = 0;
+ u8 byte;
+ do {
+ if (UNLIKELY(begin == end))
+ return begin;
+ byte = *(begin++);
+ T slice = byte & 0x7f;
+ value |= slice << shift;
+ shift += 7;
+ } while (byte >= 128);
+ if (shift < 64 && (byte & 0x40))
+ value |= (-1ULL) << shift;
+ *v = value;
+ return begin;
+}
+
+template <typename T, typename It>
+It EncodeULEB128(T value, It begin, It end) {
+ do {
+ u8 byte = value & 0x7f;
+ value >>= 7;
+ if (value)
+ byte |= 0x80;
+ if (UNLIKELY(begin == end))
+ break;
+ *(begin++) = byte;
+ } while (value);
+ return begin;
+}
+
+template <typename T, typename It>
+It DecodeULEB128(It begin, It end, T* v) {
+ T value = 0;
+ unsigned shift = 0;
+ u8 byte;
+ do {
+ if (UNLIKELY(begin == end))
+ return begin;
+ byte = *(begin++);
+ T slice = byte & 0x7f;
+ value += slice << shift;
+ shift += 7;
+ } while (byte >= 128);
+ *v = value;
+ return begin;
+}
+
+} // namespace __sanitizer
+
+#endif // SANITIZER_LEB128_H
}
}
+uptr internal_wcslen(const wchar_t *s) {
+ uptr i = 0;
+ while (s[i]) i++;
+ return i;
+}
+
+uptr internal_wcsnlen(const wchar_t *s, uptr maxlen) {
+ uptr i = 0;
+ while (i < maxlen && s[i]) i++;
+ return i;
+}
+
bool mem_is_zero(const char *beg, uptr size) {
CHECK_LE(size, 1ULL << FIRST_32_SECOND_64(30, 40)); // Sanity check.
const char *end = beg + size;
char *internal_strstr(const char *haystack, const char *needle);
// Works only for base=10 and doesn't set errno.
s64 internal_simple_strtoll(const char *nptr, const char **endptr, int base);
-int internal_snprintf(char *buffer, uptr length, const char *format, ...);
+int internal_snprintf(char *buffer, uptr length, const char *format, ...)
+ FORMAT(3, 4);
+uptr internal_wcslen(const wchar_t *s);
+uptr internal_wcsnlen(const wchar_t *s, uptr maxlen);
// Return true if all bytes in [mem, mem+size) are zero.
// Optimized for the case when the result is true.
#include "sanitizer_platform.h"
-#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC || \
+#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_APPLE || \
SANITIZER_NETBSD
#include "sanitizer_libignore.h"
}
void LibIgnore::AddIgnoredLibrary(const char *name_templ) {
- BlockingMutexLock lock(&mutex_);
+ Lock lock(&mutex_);
if (count_ >= kMaxLibs) {
- Report("%s: too many ignored libraries (max: %d)\n", SanitizerToolName,
+ Report("%s: too many ignored libraries (max: %zu)\n", SanitizerToolName,
kMaxLibs);
Die();
}
}
void LibIgnore::OnLibraryLoaded(const char *name) {
- BlockingMutexLock lock(&mutex_);
+ Lock lock(&mutex_);
// Try to match suppressions with symlink target.
InternalMmapVector<char> buf(kMaxPathLength);
if (name && internal_readlink(name, buf.data(), buf.size() - 1) > 0 &&
continue;
if (IsPcInstrumented(range.beg) && IsPcInstrumented(range.end - 1))
continue;
- VReport(1, "Adding instrumented range %p-%p from library '%s'\n",
+ VReport(1, "Adding instrumented range 0x%zx-0x%zx from library '%s'\n",
range.beg, range.end, mod.full_name());
const uptr idx =
atomic_load(&instrumented_ranges_count_, memory_order_relaxed);
} // namespace __sanitizer
-#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC ||
+#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_APPLE ||
// SANITIZER_NETBSD
LibCodeRange instrumented_code_ranges_[kMaxInstrumentedRanges];
// Cold part:
- BlockingMutex mutex_;
+ Mutex mutex_;
uptr count_;
Lib libs_[kMaxLibs];
bool track_instrumented_libs_;
#include "sanitizer_linux.h"
#include "sanitizer_placement_new.h"
#include "sanitizer_procmaps.h"
+#include "sanitizer_solaris.h"
#if SANITIZER_NETBSD
#define _RTLD_SOURCE // for __lwp_gettcb_fast() / __lwp_getprivate_fast()
#endif
#if SANITIZER_SOLARIS
+#include <stddef.h>
#include <stdlib.h>
#include <thread.h>
#endif
g_use_dlpi_tls_data =
GetLibcVersion(&major, &minor, &patch) && major == 2 && minor >= 25;
-#if defined(__aarch64__) || defined(__x86_64__) || defined(__powerpc64__)
+#if defined(__aarch64__) || defined(__x86_64__) || defined(__powerpc64__) || \
+ defined(__loongarch__)
void *get_tls_static_info = dlsym(RTLD_NEXT, "_dl_get_tls_static_info");
size_t tls_align;
((void (*)(size_t *, size_t *))get_tls_static_info)(&g_tls_size, &tls_align);
// On glibc x86_64, ThreadDescriptorSize() needs to be precise due to the usage
// of g_tls_size. On other targets, ThreadDescriptorSize() is only used by lsan
// to get the pointer to thread-specific data keys in the thread control block.
-#if (SANITIZER_FREEBSD || SANITIZER_LINUX) && !SANITIZER_ANDROID
+#if (SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_SOLARIS) && \
+ !SANITIZER_ANDROID && !SANITIZER_GO
// sizeof(struct pthread) from glibc.
static atomic_uintptr_t thread_descriptor_size;
-uptr ThreadDescriptorSize() {
- uptr val = atomic_load_relaxed(&thread_descriptor_size);
- if (val)
- return val;
+static uptr ThreadDescriptorSizeFallback() {
+ uptr val = 0;
#if defined(__x86_64__) || defined(__i386__) || defined(__arm__)
int major;
int minor;
#elif defined(__mips__)
// TODO(sagarthakur): add more values as per different glibc versions.
val = FIRST_32_SECOND_64(1152, 1776);
+#elif SANITIZER_LOONGARCH64
+ val = 1856; // from glibc 2.36
#elif SANITIZER_RISCV64
int major;
int minor;
#elif defined(__powerpc64__)
val = 1776; // from glibc.ppc64le 2.20-8.fc21
#endif
+ return val;
+}
+
+uptr ThreadDescriptorSize() {
+ uptr val = atomic_load_relaxed(&thread_descriptor_size);
if (val)
- atomic_store_relaxed(&thread_descriptor_size, val);
+ return val;
+ // _thread_db_sizeof_pthread is a GLIBC_PRIVATE symbol that is exported in
+ // glibc 2.34 and later.
+ if (unsigned *psizeof = static_cast<unsigned *>(
+ dlsym(RTLD_DEFAULT, "_thread_db_sizeof_pthread")))
+ val = *psizeof;
+ if (!val)
+ val = ThreadDescriptorSizeFallback();
+ atomic_store_relaxed(&thread_descriptor_size, val);
return val;
}
-#if defined(__mips__) || defined(__powerpc64__) || SANITIZER_RISCV64
+#if defined(__mips__) || defined(__powerpc64__) || SANITIZER_RISCV64 || \
+ SANITIZER_LOONGARCH64
// TlsPreTcbSize includes size of struct pthread_descr and size of tcb
// head structure. It lies before the static tls blocks.
static uptr TlsPreTcbSize() {
const uptr kTcbHead = 88; // sizeof (tcbhead_t)
#elif SANITIZER_RISCV64
const uptr kTcbHead = 16; // sizeof (tcbhead_t)
+#elif SANITIZER_LOONGARCH64
+ const uptr kTcbHead = 16; // sizeof (tcbhead_t)
#endif
const uptr kTlsAlign = 16;
const uptr kTlsPreTcbSize =
}
#endif
-#if !SANITIZER_GO
namespace {
struct TlsBlock {
uptr begin, end, align;
extern "C" void *__tls_get_addr(size_t *);
#endif
+static size_t main_tls_modid;
+
static int CollectStaticTlsBlocks(struct dl_phdr_info *info, size_t size,
void *data) {
- if (!info->dlpi_tls_modid)
+ size_t tls_modid;
+#if SANITIZER_SOLARIS
+ // dlpi_tls_modid is only available since Solaris 11.4 SRU 10. Use
+ // dlinfo(RTLD_DI_LINKMAP) instead which works on all of Solaris 11.3,
+ // 11.4, and Illumos. The tlsmodid of the executable was changed to 1 in
+ // 11.4 to match other implementations.
+ if (size >= offsetof(dl_phdr_info_test, dlpi_tls_modid))
+ main_tls_modid = 1;
+ else
+ main_tls_modid = 0;
+ g_use_dlpi_tls_data = 0;
+ Rt_map *map;
+ dlinfo(RTLD_SELF, RTLD_DI_LINKMAP, &map);
+ tls_modid = map->rt_tlsmodid;
+#else
+ main_tls_modid = 1;
+ tls_modid = info->dlpi_tls_modid;
+#endif
+
+ if (tls_modid < main_tls_modid)
return 0;
- uptr begin = (uptr)info->dlpi_tls_data;
+ uptr begin;
+#if !SANITIZER_SOLARIS
+ begin = (uptr)info->dlpi_tls_data;
+#endif
if (!g_use_dlpi_tls_data) {
// Call __tls_get_addr as a fallback. This forces TLS allocation on glibc
// and FreeBSD.
#ifdef __s390__
begin = (uptr)__builtin_thread_pointer() +
- TlsGetOffset(info->dlpi_tls_modid, 0);
+ TlsGetOffset(tls_modid, 0);
#else
- size_t mod_and_off[2] = {info->dlpi_tls_modid, 0};
+ size_t mod_and_off[2] = {tls_modid, 0};
begin = (uptr)__tls_get_addr(mod_and_off);
#endif
}
if (info->dlpi_phdr[i].p_type == PT_TLS) {
static_cast<InternalMmapVector<TlsBlock> *>(data)->push_back(
TlsBlock{begin, begin + info->dlpi_phdr[i].p_memsz,
- info->dlpi_phdr[i].p_align, info->dlpi_tls_modid});
+ info->dlpi_phdr[i].p_align, tls_modid});
break;
}
return 0;
dl_iterate_phdr(CollectStaticTlsBlocks, &ranges);
uptr len = ranges.size();
Sort(ranges.begin(), len);
- // Find the range with tls_modid=1. For glibc, because libc.so uses PT_TLS,
- // this module is guaranteed to exist and is one of the initially loaded
- // modules.
+ // Find the range with tls_modid == main_tls_modid. For glibc, because
+ // libc.so uses PT_TLS, this module is guaranteed to exist and is one of
+ // the initially loaded modules.
uptr one = 0;
- while (one != len && ranges[one].tls_modid != 1) ++one;
+ while (one != len && ranges[one].tls_modid != main_tls_modid) ++one;
if (one == len) {
// This may happen with musl if no module uses PT_TLS.
*addr = 0;
return;
}
// Find the maximum consecutive ranges. We consider two modules consecutive if
- // the gap is smaller than the alignment. The dynamic loader places static TLS
- // blocks this way not to waste space.
+ // the gap is smaller than the alignment of the latter range. The dynamic
+ // loader places static TLS blocks this way not to waste space.
uptr l = one;
*align = ranges[l].align;
- while (l != 0 && ranges[l].begin < ranges[l - 1].end + ranges[l - 1].align)
+ while (l != 0 && ranges[l].begin < ranges[l - 1].end + ranges[l].align)
*align = Max(*align, ranges[--l].align);
uptr r = one + 1;
- while (r != len && ranges[r].begin < ranges[r - 1].end + ranges[r - 1].align)
+ while (r != len && ranges[r].begin < ranges[r - 1].end + ranges[r].align)
*align = Max(*align, ranges[r++].align);
*addr = ranges[l].begin;
*size = ranges[r - 1].end - ranges[l].begin;
}
-#endif // !SANITIZER_GO
#endif // (x86_64 || i386 || mips || ...) && (SANITIZER_FREEBSD ||
- // SANITIZER_LINUX) && !SANITIZER_ANDROID
+ // SANITIZER_LINUX) && !SANITIZER_ANDROID && !SANITIZER_GO
#if SANITIZER_NETBSD
static struct tls_tcb * ThreadSelfTlsTcb() {
#elif SANITIZER_GLIBC && defined(__x86_64__)
// For aarch64 and x86-64, use an O(1) approach which requires relatively
// precise ThreadDescriptorSize. g_tls_size was initialized in InitTlsSize.
+# if SANITIZER_X32
+ asm("mov %%fs:8,%0" : "=r"(*addr));
+# else
asm("mov %%fs:16,%0" : "=r"(*addr));
+# endif
*size = g_tls_size;
*addr -= *size;
*addr += ThreadDescriptorSize();
*addr = reinterpret_cast<uptr>(__builtin_thread_pointer()) -
ThreadDescriptorSize();
*size = g_tls_size + ThreadDescriptorSize();
+#elif SANITIZER_GLIBC && defined(__loongarch__)
+# ifdef __clang__
+ *addr = reinterpret_cast<uptr>(__builtin_thread_pointer()) -
+ ThreadDescriptorSize();
+# else
+ asm("or %0,$tp,$zero" : "=r"(*addr));
+ *addr -= ThreadDescriptorSize();
+# endif
+ *size = g_tls_size + ThreadDescriptorSize();
#elif SANITIZER_GLIBC && defined(__powerpc64__)
// Workaround for glibc<2.25(?). 2.27 is known to not need this.
uptr tp;
const uptr pre_tcb_size = TlsPreTcbSize();
*addr = tp - pre_tcb_size;
*size = g_tls_size + pre_tcb_size;
-#elif SANITIZER_FREEBSD || SANITIZER_LINUX
+#elif SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_SOLARIS
uptr align;
GetStaticTlsBoundary(addr, size, &align);
#if defined(__x86_64__) || defined(__i386__) || defined(__s390__) || \
*addr = (uptr)tcb->tcb_dtv[1];
}
}
-#elif SANITIZER_SOLARIS
- // FIXME
- *addr = 0;
- *size = 0;
#else
#error "Unknown OS"
#endif
bool writable = phdr->p_flags & PF_W;
cur_module.addAddressRange(cur_beg, cur_end, executable,
writable);
+ } else if (phdr->p_type == PT_NOTE) {
+# ifdef NT_GNU_BUILD_ID
+ uptr off = 0;
+ while (off + sizeof(ElfW(Nhdr)) < phdr->p_memsz) {
+ auto *nhdr = reinterpret_cast<const ElfW(Nhdr) *>(info->dlpi_addr +
+ phdr->p_vaddr + off);
+ constexpr auto kGnuNamesz = 4; // "GNU" with NUL-byte.
+ static_assert(kGnuNamesz % 4 == 0, "kGnuNameSize is aligned to 4.");
+ if (nhdr->n_type == NT_GNU_BUILD_ID && nhdr->n_namesz == kGnuNamesz) {
+ if (off + sizeof(ElfW(Nhdr)) + nhdr->n_namesz + nhdr->n_descsz >
+ phdr->p_memsz) {
+ // Something is very wrong, bail out instead of reading potentially
+ // arbitrary memory.
+ break;
+ }
+ const char *name =
+ reinterpret_cast<const char *>(nhdr) + sizeof(*nhdr);
+ if (internal_memcmp(name, "GNU", 3) == 0) {
+ const char *value = reinterpret_cast<const char *>(nhdr) +
+ sizeof(*nhdr) + kGnuNamesz;
+ cur_module.setUuid(value, nhdr->n_descsz);
+ break;
+ }
+ }
+ off += sizeof(*nhdr) + RoundUpTo(nhdr->n_namesz, 4) +
+ RoundUpTo(nhdr->n_descsz, 4);
+ }
+# endif
}
}
modules->push_back(cur_module);
uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg,
int *parent_tidptr, void *newtls, int *child_tidptr) {
- if (!fn || !child_stack)
- return -EINVAL;
+ if (!fn || !child_stack) {
+ errno = EINVAL;
+ return -1;
+ }
CHECK_EQ(0, (uptr)child_stack % 16);
// Minimum frame size.
#ifdef __s390x__
// And pass parameters.
((unsigned long *)child_stack)[1] = (uptr)fn;
((unsigned long *)child_stack)[2] = (uptr)arg;
- register long res __asm__("r2");
+ register uptr res __asm__("r2");
register void *__cstack __asm__("r2") = child_stack;
- register int __flags __asm__("r3") = flags;
+ register long __flags __asm__("r3") = flags;
register int * __ptidptr __asm__("r4") = parent_tidptr;
register int * __ctidptr __asm__("r5") = child_tidptr;
register void * __newtls __asm__("r6") = newtls;
"r"(__ctidptr),
"r"(__newtls)
: "memory", "cc");
+ if (res >= (uptr)-4095) {
+ errno = -res;
+ return -1;
+ }
return res;
}
// instantiated with the `LocalAddressSpaceView` type. This type is used to
// load any pointers in instance methods. This implementation is effectively
// a no-op. When an object is to be used in an out-of-process manner it is
-// instansiated with the `RemoteAddressSpaceView` type.
+// instantiated with the `RemoteAddressSpaceView` type.
//
// By making `AddressSpaceView` a template parameter of an object, it can
// change its implementation at compile time which has no run time overhead.
--- /dev/null
+//===-- sanitizer_lzw.h -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Lempel–Ziv–Welch encoding/decoding
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_LZW_H
+#define SANITIZER_LZW_H
+
+#include "sanitizer_dense_map.h"
+
+namespace __sanitizer {
+
+using LzwCodeType = u32;
+
+template <class T, class ItIn, class ItOut>
+ItOut LzwEncode(ItIn begin, ItIn end, ItOut out) {
+ using Substring =
+ detail::DenseMapPair<LzwCodeType /* Prefix */, T /* Next input */>;
+
+ // Sentinel value for substrings of len 1.
+ static constexpr LzwCodeType kNoPrefix =
+ Min(DenseMapInfo<Substring>::getEmptyKey().first,
+ DenseMapInfo<Substring>::getTombstoneKey().first) -
+ 1;
+ DenseMap<Substring, LzwCodeType> prefix_to_code;
+ {
+ // Add all substring of len 1 as initial dictionary.
+ InternalMmapVector<T> dict_len1;
+ for (auto it = begin; it != end; ++it)
+ if (prefix_to_code.try_emplace({kNoPrefix, *it}, 0).second)
+ dict_len1.push_back(*it);
+
+ // Slightly helps with later delta encoding.
+ Sort(dict_len1.data(), dict_len1.size());
+
+ // For large sizeof(T) we have to store dict_len1. Smaller types like u8 can
+ // just generate them.
+ *out = dict_len1.size();
+ ++out;
+
+ for (uptr i = 0; i != dict_len1.size(); ++i) {
+ // Remap after the Sort.
+ prefix_to_code[{kNoPrefix, dict_len1[i]}] = i;
+ *out = dict_len1[i];
+ ++out;
+ }
+ CHECK_EQ(prefix_to_code.size(), dict_len1.size());
+ }
+
+ if (begin == end)
+ return out;
+
+ // Main LZW encoding loop.
+ LzwCodeType match = prefix_to_code.find({kNoPrefix, *begin})->second;
+ ++begin;
+ for (auto it = begin; it != end; ++it) {
+ // Extend match with the new item.
+ auto ins = prefix_to_code.try_emplace({match, *it}, prefix_to_code.size());
+ if (ins.second) {
+ // This is a new substring, but emit the code for the current match
+ // (before extend). This allows LZW decoder to recover the dictionary.
+ *out = match;
+ ++out;
+ // Reset the match to a single item, which must be already in the map.
+ match = prefix_to_code.find({kNoPrefix, *it})->second;
+ } else {
+ // Already known, use as the current match.
+ match = ins.first->second;
+ }
+ }
+
+ *out = match;
+ ++out;
+
+ return out;
+}
+
+template <class T, class ItIn, class ItOut>
+ItOut LzwDecode(ItIn begin, ItIn end, ItOut out) {
+ if (begin == end)
+ return out;
+
+ // Load dictionary of len 1 substrings. Theses correspont to lowest codes.
+ InternalMmapVector<T> dict_len1(*begin);
+ ++begin;
+
+ if (begin == end)
+ return out;
+
+ for (auto& v : dict_len1) {
+ v = *begin;
+ ++begin;
+ }
+
+ // Substrings of len 2 and up. Indexes are shifted because [0,
+ // dict_len1.size()) stored in dict_len1. Substings get here after being
+ // emitted to the output, so we can use output position.
+ InternalMmapVector<detail::DenseMapPair<ItOut /* begin. */, ItOut /* end */>>
+ code_to_substr;
+
+ // Copies already emitted substrings into the output again.
+ auto copy = [&code_to_substr, &dict_len1](LzwCodeType code, ItOut out) {
+ if (code < dict_len1.size()) {
+ *out = dict_len1[code];
+ ++out;
+ return out;
+ }
+ const auto& s = code_to_substr[code - dict_len1.size()];
+
+ for (ItOut it = s.first; it != s.second; ++it, ++out) *out = *it;
+ return out;
+ };
+
+ // Returns lens of the substring with the given code.
+ auto code_to_len = [&code_to_substr, &dict_len1](LzwCodeType code) -> uptr {
+ if (code < dict_len1.size())
+ return 1;
+ const auto& s = code_to_substr[code - dict_len1.size()];
+ return s.second - s.first;
+ };
+
+ // Main LZW decoding loop.
+ LzwCodeType prev_code = *begin;
+ ++begin;
+ out = copy(prev_code, out);
+ for (auto it = begin; it != end; ++it) {
+ LzwCodeType code = *it;
+ auto start = out;
+ if (code == dict_len1.size() + code_to_substr.size()) {
+ // Special LZW case. The code is not in the dictionary yet. This is
+ // possible only when the new substring is the same as previous one plus
+ // the first item of the previous substring. We can emit that in two
+ // steps.
+ out = copy(prev_code, out);
+ *out = *start;
+ ++out;
+ } else {
+ out = copy(code, out);
+ }
+
+ // Every time encoded emits the code, it also creates substing of len + 1
+ // including the first item of the just emmited substring. Do the same here.
+ uptr len = code_to_len(prev_code);
+ code_to_substr.push_back({start - len, start + 1});
+
+ prev_code = code;
+ }
+ return out;
+}
+
+} // namespace __sanitizer
+#endif
//===----------------------------------------------------------------------===//
#include "sanitizer_platform.h"
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
#include "sanitizer_mac.h"
#include "interception/interception.h"
#include "sanitizer_common.h"
#include "sanitizer_file.h"
#include "sanitizer_flags.h"
+#include "sanitizer_interface_internal.h"
#include "sanitizer_internal_defs.h"
#include "sanitizer_libc.h"
#include "sanitizer_platform_limits_posix.h"
#include <malloc/malloc.h>
#include <os/log.h>
#include <pthread.h>
+#include <pthread/introspection.h>
#include <sched.h>
#include <signal.h>
#include <spawn.h>
static fd_t internal_spawn_impl(const char *argv[], const char *envp[],
pid_t *pid) {
- fd_t master_fd = kInvalidFd;
- fd_t slave_fd = kInvalidFd;
+ fd_t primary_fd = kInvalidFd;
+ fd_t secondary_fd = kInvalidFd;
auto fd_closer = at_scope_exit([&] {
- internal_close(master_fd);
- internal_close(slave_fd);
+ internal_close(primary_fd);
+ internal_close(secondary_fd);
});
// We need a new pseudoterminal to avoid buffering problems. The 'atos' tool
// in particular detects when it's talking to a pipe and forgets to flush the
// output stream after sending a response.
- master_fd = posix_openpt(O_RDWR);
- if (master_fd == kInvalidFd) return kInvalidFd;
+ primary_fd = posix_openpt(O_RDWR);
+ if (primary_fd == kInvalidFd)
+ return kInvalidFd;
- int res = grantpt(master_fd) || unlockpt(master_fd);
+ int res = grantpt(primary_fd) || unlockpt(primary_fd);
if (res != 0) return kInvalidFd;
// Use TIOCPTYGNAME instead of ptsname() to avoid threading problems.
- char slave_pty_name[128];
- res = ioctl(master_fd, TIOCPTYGNAME, slave_pty_name);
+ char secondary_pty_name[128];
+ res = ioctl(primary_fd, TIOCPTYGNAME, secondary_pty_name);
if (res == -1) return kInvalidFd;
- slave_fd = internal_open(slave_pty_name, O_RDWR);
- if (slave_fd == kInvalidFd) return kInvalidFd;
+ secondary_fd = internal_open(secondary_pty_name, O_RDWR);
+ if (secondary_fd == kInvalidFd)
+ return kInvalidFd;
// File descriptor actions
posix_spawn_file_actions_t acts;
posix_spawn_file_actions_destroy(&acts);
});
- res = posix_spawn_file_actions_adddup2(&acts, slave_fd, STDIN_FILENO) ||
- posix_spawn_file_actions_adddup2(&acts, slave_fd, STDOUT_FILENO) ||
- posix_spawn_file_actions_addclose(&acts, slave_fd);
+ res = posix_spawn_file_actions_adddup2(&acts, secondary_fd, STDIN_FILENO) ||
+ posix_spawn_file_actions_adddup2(&acts, secondary_fd, STDOUT_FILENO) ||
+ posix_spawn_file_actions_addclose(&acts, secondary_fd);
if (res != 0) return kInvalidFd;
// Spawn attributes
// Disable echo in the new terminal, disable CR.
struct termios termflags;
- tcgetattr(master_fd, &termflags);
+ tcgetattr(primary_fd, &termflags);
termflags.c_oflag &= ~ONLCR;
termflags.c_lflag &= ~ECHO;
- tcsetattr(master_fd, TCSANOW, &termflags);
+ tcsetattr(primary_fd, TCSANOW, &termflags);
- // On success, do not close master_fd on scope exit.
- fd_t fd = master_fd;
- master_fd = kInvalidFd;
+ // On success, do not close primary_fd on scope exit.
+ fd_t fd = primary_fd;
+ primary_fd = kInvalidFd;
return fd;
}
return S_ISREG(st.st_mode);
}
+bool DirExists(const char *path) {
+ struct stat st;
+ if (stat(path, &st))
+ return false;
+ return S_ISDIR(st.st_mode);
+}
+
tid_t GetTid() {
tid_t tid;
pthread_threadid_np(nullptr, &tid);
void FutexWake(atomic_uint32_t *p, u32 count) {}
-BlockingMutex::BlockingMutex() {
- internal_memset(this, 0, sizeof(*this));
-}
-
-void BlockingMutex::Lock() {
- CHECK(sizeof(OSSpinLock) <= sizeof(opaque_storage_));
- CHECK_EQ(OS_SPINLOCK_INIT, 0);
- CHECK_EQ(owner_, 0);
- OSSpinLockLock((OSSpinLock*)&opaque_storage_);
-}
-
-void BlockingMutex::Unlock() {
- OSSpinLockUnlock((OSSpinLock*)&opaque_storage_);
-}
-
-void BlockingMutex::CheckLocked() const {
- CHECK_NE(*(const OSSpinLock*)&opaque_storage_, 0);
-}
-
u64 NanoTime() {
timeval tv;
internal_memset(&tv, 0, sizeof(tv));
asm("movq %%gs:0,%0" : "=r"(segbase));
#elif defined(__i386__)
asm("movl %%gs:0,%0" : "=r"(segbase));
+#elif defined(__aarch64__)
+ asm("mrs %x0, tpidrro_el0" : "=r"(segbase));
+ segbase &= 0x07ul; // clearing lower bits, cpu id stored there
#endif
return segbase;
}
void internal_join_thread(void *th) { pthread_join((pthread_t)th, 0); }
#if !SANITIZER_GO
-static BlockingMutex syslog_lock(LINKER_INITIALIZED);
-#endif
+static Mutex syslog_lock;
+# endif
void WriteOneLineToSyslog(const char *s) {
#if !SANITIZER_GO
// buffer to store crash report application information
static char crashreporter_info_buff[__sanitizer::kErrorMessageBufferSize] = {};
-static BlockingMutex crashreporter_info_mutex(LINKER_INITIALIZED);
+static Mutex crashreporter_info_mutex;
extern "C" {
// Integrate with crash reporter libraries.
} // extern "C"
static void CRAppendCrashLogMessage(const char *msg) {
- BlockingMutexLock l(&crashreporter_info_mutex);
+ Lock l(&crashreporter_info_mutex);
internal_strlcat(crashreporter_info_buff, msg,
sizeof(crashreporter_info_buff));
#if HAVE_CRASHREPORTERCLIENT_H
// the reporting thread holds the thread registry mutex, and asl_log waits
// for GCD to dispatch a new thread, the process will deadlock, because the
// pthread_create wrapper needs to acquire the lock as well.
- BlockingMutexLock l(&syslog_lock);
+ Lock l(&syslog_lock);
if (common_flags()->log_to_syslog)
WriteToSyslog(buffer);
SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
#if defined(__x86_64__) || defined(__i386__)
ucontext_t *ucontext = static_cast<ucontext_t*>(context);
- return ucontext->uc_mcontext->__es.__err & 2 /*T_PF_WRITE*/ ? WRITE : READ;
+ return ucontext->uc_mcontext->__es.__err & 2 /*T_PF_WRITE*/ ? Write : Read;
+#elif defined(__arm64__)
+ ucontext_t *ucontext = static_cast<ucontext_t*>(context);
+ return ucontext->uc_mcontext->__es.__esr & 0x40 /*ISS_DA_WNR*/ ? Write : Read;
#else
- return UNKNOWN;
+ return Unknown;
#endif
}
(uptr)ptrauth_strip( \
(void *)arm_thread_state64_get_##r(ucontext->uc_mcontext->__ss), 0)
#else
- #define AARCH64_GET_REG(r) ucontext->uc_mcontext->__ss.__##r
+ #define AARCH64_GET_REG(r) (uptr)ucontext->uc_mcontext->__ss.__##r
#endif
static void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) {
ucontext_t *ucontext = (ucontext_t*)context;
# if defined(__aarch64__)
*pc = AARCH64_GET_REG(pc);
-# if defined(__IPHONE_8_0) && __IPHONE_OS_VERSION_MAX_ALLOWED >= __IPHONE_8_0
*bp = AARCH64_GET_REG(fp);
-# else
- *bp = AARCH64_GET_REG(lr);
-# endif
*sp = AARCH64_GET_REG(sp);
# elif defined(__x86_64__)
*pc = ucontext->uc_mcontext->__ss.__rip;
set_behavior(mach_task_self(), task_exc_guard_none);
}
+static void VerifyInterceptorsWorking();
+static void StripEnv();
+
void InitializePlatformEarly() {
// Only use xnu_fast_mmap when on x86_64 and the kernel supports it.
use_xnu_fast_mmap =
#endif
if (GetDarwinKernelVersion() >= DarwinKernelVersion(19, 0))
DisableMmapExcGuardExceptions();
+
+# if !SANITIZER_GO
+ MonotonicNanoTime(); // Call to initialize mach_timebase_info
+ VerifyInterceptorsWorking();
+ StripEnv();
+# endif
}
#if !SANITIZER_GO
static const char kDyldInsertLibraries[] = "DYLD_INSERT_LIBRARIES";
LowLevelAllocator allocator_for_env;
+static bool ShouldCheckInterceptors() {
+ // Restrict "interceptors working?" check to ASan and TSan.
+ const char *sanitizer_names[] = {"AddressSanitizer", "ThreadSanitizer"};
+ size_t count = sizeof(sanitizer_names) / sizeof(sanitizer_names[0]);
+ for (size_t i = 0; i < count; i++) {
+ if (internal_strcmp(sanitizer_names[i], SanitizerToolName) == 0)
+ return true;
+ }
+ return false;
+}
+
+static void VerifyInterceptorsWorking() {
+ if (!common_flags()->verify_interceptors || !ShouldCheckInterceptors())
+ return;
+
+ // Verify that interceptors really work. We'll use dlsym to locate
+ // "puts", if interceptors are working, it should really point to
+ // "wrap_puts" within our own dylib.
+ Dl_info info_puts, info_runtime;
+ RAW_CHECK(dladdr(dlsym(RTLD_DEFAULT, "puts"), &info_puts));
+ RAW_CHECK(dladdr((void *)__sanitizer_report_error_summary, &info_runtime));
+ if (internal_strcmp(info_puts.dli_fname, info_runtime.dli_fname) != 0) {
+ Report(
+ "ERROR: Interceptors are not working. This may be because %s is "
+ "loaded too late (e.g. via dlopen). Please launch the executable "
+ "with:\n%s=%s\n",
+ SanitizerToolName, kDyldInsertLibraries, info_runtime.dli_fname);
+ RAW_CHECK("interceptors not installed" && 0);
+ }
+}
+
// Change the value of the env var |name|, leaking the original value.
// If |name_value| is NULL, the variable is deleted from the environment,
// otherwise the corresponding "NAME=value" string is replaced with
// |name_value|.
-void LeakyResetEnv(const char *name, const char *name_value) {
+static void LeakyResetEnv(const char *name, const char *name_value) {
char **env = GetEnviron();
uptr name_len = internal_strlen(name);
while (*env != 0) {
}
}
-SANITIZER_WEAK_CXX_DEFAULT_IMPL
-bool ReexecDisabled() {
- return false;
-}
-
-static bool DyldNeedsEnvVariable() {
- // If running on OS X 10.11+ or iOS 9.0+, dyld will interpose even if
- // DYLD_INSERT_LIBRARIES is not set.
- return GetMacosAlignedVersion() < MacosVersion(10, 11);
-}
-
-void MaybeReexec() {
- // FIXME: This should really live in some "InitializePlatform" method.
- MonotonicNanoTime();
+static void StripEnv() {
+ if (!common_flags()->strip_env)
+ return;
- if (ReexecDisabled()) return;
+ char *dyld_insert_libraries =
+ const_cast<char *>(GetEnv(kDyldInsertLibraries));
+ if (!dyld_insert_libraries)
+ return;
- // Make sure the dynamic runtime library is preloaded so that the
- // wrappers work. If it is not, set DYLD_INSERT_LIBRARIES and re-exec
- // ourselves.
Dl_info info;
- RAW_CHECK(dladdr((void*)((uptr)&__sanitizer_report_error_summary), &info));
- char *dyld_insert_libraries =
- const_cast<char*>(GetEnv(kDyldInsertLibraries));
- uptr old_env_len = dyld_insert_libraries ?
- internal_strlen(dyld_insert_libraries) : 0;
- uptr fname_len = internal_strlen(info.dli_fname);
+ RAW_CHECK(dladdr((void *)__sanitizer_report_error_summary, &info));
const char *dylib_name = StripModuleName(info.dli_fname);
- uptr dylib_name_len = internal_strlen(dylib_name);
-
- bool lib_is_in_env = dyld_insert_libraries &&
- internal_strstr(dyld_insert_libraries, dylib_name);
- if (DyldNeedsEnvVariable() && !lib_is_in_env) {
- // DYLD_INSERT_LIBRARIES is not set or does not contain the runtime
- // library.
- InternalMmapVector<char> program_name(1024);
- uint32_t buf_size = program_name.size();
- _NSGetExecutablePath(program_name.data(), &buf_size);
- char *new_env = const_cast<char*>(info.dli_fname);
- if (dyld_insert_libraries) {
- // Append the runtime dylib name to the existing value of
- // DYLD_INSERT_LIBRARIES.
- new_env = (char*)allocator_for_env.Allocate(old_env_len + fname_len + 2);
- internal_strncpy(new_env, dyld_insert_libraries, old_env_len);
- new_env[old_env_len] = ':';
- // Copy fname_len and add a trailing zero.
- internal_strncpy(new_env + old_env_len + 1, info.dli_fname,
- fname_len + 1);
- // Ok to use setenv() since the wrappers don't depend on the value of
- // asan_inited.
- setenv(kDyldInsertLibraries, new_env, /*overwrite*/1);
- } else {
- // Set DYLD_INSERT_LIBRARIES equal to the runtime dylib name.
- setenv(kDyldInsertLibraries, info.dli_fname, /*overwrite*/0);
- }
- VReport(1, "exec()-ing the program with\n");
- VReport(1, "%s=%s\n", kDyldInsertLibraries, new_env);
- VReport(1, "to enable wrappers.\n");
- execv(program_name.data(), *_NSGetArgv());
-
- // We get here only if execv() failed.
- Report("ERROR: The process is launched without DYLD_INSERT_LIBRARIES, "
- "which is required for the sanitizer to work. We tried to set the "
- "environment variable and re-execute itself, but execv() failed, "
- "possibly because of sandbox restrictions. Make sure to launch the "
- "executable with:\n%s=%s\n", kDyldInsertLibraries, new_env);
- RAW_CHECK("execv failed" && 0);
- }
-
- // Verify that interceptors really work. We'll use dlsym to locate
- // "pthread_create", if interceptors are working, it should really point to
- // "wrap_pthread_create" within our own dylib.
- Dl_info info_pthread_create;
- void *dlopen_addr = dlsym(RTLD_DEFAULT, "pthread_create");
- RAW_CHECK(dladdr(dlopen_addr, &info_pthread_create));
- if (internal_strcmp(info.dli_fname, info_pthread_create.dli_fname) != 0) {
- Report(
- "ERROR: Interceptors are not working. This may be because %s is "
- "loaded too late (e.g. via dlopen). Please launch the executable "
- "with:\n%s=%s\n",
- SanitizerToolName, kDyldInsertLibraries, info.dli_fname);
- RAW_CHECK("interceptors not installed" && 0);
- }
-
+ bool lib_is_in_env = internal_strstr(dyld_insert_libraries, dylib_name);
if (!lib_is_in_env)
return;
- if (!common_flags()->strip_env)
- return;
-
// DYLD_INSERT_LIBRARIES is set and contains the runtime library. Let's remove
// the dylib from the environment variable, because interceptors are installed
// and we don't want our children to inherit the variable.
+ uptr old_env_len = internal_strlen(dyld_insert_libraries);
+ uptr dylib_name_len = internal_strlen(dylib_name);
uptr env_name_len = internal_strlen(kDyldInsertLibraries);
// Allocate memory to hold the previous env var name, its value, the '='
// sign and the '\0' char.
uptr largest_gap_found = 0;
uptr max_occupied_addr = 0;
- VReport(2, "FindDynamicShadowStart, space_size = %p\n", space_size);
+ VReport(2, "FindDynamicShadowStart, space_size = %p\n", (void *)space_size);
uptr shadow_start =
FindAvailableMemoryRange(space_size, alignment, granularity,
&largest_gap_found, &max_occupied_addr);
VReport(
2,
"Shadow doesn't fit, largest_gap_found = %p, max_occupied_addr = %p\n",
- largest_gap_found, max_occupied_addr);
+ (void *)largest_gap_found, (void *)max_occupied_addr);
uptr new_max_vm = RoundDownTo(largest_gap_found << shadow_scale, alignment);
if (new_max_vm < max_occupied_addr) {
Report("Unable to find a memory range for dynamic shadow.\n");
Report(
"space_size = %p, largest_gap_found = %p, max_occupied_addr = %p, "
"new_max_vm = %p\n",
- space_size, largest_gap_found, max_occupied_addr, new_max_vm);
+ (void *)space_size, (void *)largest_gap_found,
+ (void *)max_occupied_addr, (void *)new_max_vm);
CHECK(0 && "cannot place shadow");
}
RestrictMemoryToMaxAddress(new_max_vm);
high_mem_end = new_max_vm - 1;
space_size = (high_mem_end >> shadow_scale) + left_padding;
- VReport(2, "FindDynamicShadowStart, space_size = %p\n", space_size);
+ VReport(2, "FindDynamicShadowStart, space_size = %p\n", (void *)space_size);
shadow_start = FindAvailableMemoryRange(space_size, alignment, granularity,
nullptr, nullptr);
if (shadow_start == 0) {
mach_vm_address_t start_address =
(SANITIZER_WORDSIZE == 32) ? 0x000000001000 : 0x000100000000;
+ const mach_vm_address_t max_vm_address = GetMaxVirtualAddress() + 1;
mach_vm_address_t address = start_address;
mach_vm_address_t free_begin = start_address;
kern_return_t kr = KERN_SUCCESS;
(vm_region_info_t)&vminfo, &count);
if (kr == KERN_INVALID_ADDRESS) {
// No more regions beyond "address", consider the gap at the end of VM.
- address = GetMaxVirtualAddress() + 1;
+ address = max_vm_address;
vmsize = 0;
} else {
if (max_occupied_addr) *max_occupied_addr = address + vmsize;
if (free_begin != address) {
// We found a free region [free_begin..address-1].
uptr gap_start = RoundUpTo((uptr)free_begin + left_padding, alignment);
- uptr gap_end = RoundDownTo((uptr)address, alignment);
+ uptr gap_end = RoundDownTo((uptr)Min(address, max_vm_address), alignment);
uptr gap_size = gap_end > gap_start ? gap_end - gap_start : 0;
if (size < gap_size) {
return gap_start;
}
// FIXME implement on this platform.
-void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size) { }
+void GetMemoryProfile(fill_profile_f cb, uptr *stats) {}
void SignalContext::DumpAllRegisters(void *context) {
Report("Register values:\n");
# define DUMPREG64(r) \
Printf("%s = 0x%016llx ", #r, ucontext->uc_mcontext->__ss.__ ## r);
# define DUMPREGA64(r) \
- Printf(" %s = 0x%016llx ", #r, AARCH64_GET_REG(r));
+ Printf(" %s = 0x%016lx ", #r, AARCH64_GET_REG(r));
# define DUMPREG32(r) \
Printf("%s = 0x%08x ", #r, ucontext->uc_mcontext->__ss.__ ## r);
# define DUMPREG_(r) Printf(" "); DUMPREG(r);
char uuid_str[128];
FormatUUID(uuid_str, sizeof(uuid_str), modules[i].uuid());
Printf("0x%zx-0x%zx %s (%s) %s\n", modules[i].base_address(),
- modules[i].max_executable_address(), modules[i].full_name(),
+ modules[i].max_address(), modules[i].full_name(),
ModuleArchToString(modules[i].arch()), uuid_str);
}
Printf("End of module map.\n");
void InitializePlatformCommonFlags(CommonFlags *cf) {}
+// Pthread introspection hook
+//
+// * GCD worker threads are created without a call to pthread_create(), but we
+// still need to register these threads (with ThreadCreate/Start()).
+// * We use the "pthread introspection hook" below to observe the creation of
+// such threads.
+// * GCD worker threads don't have parent threads and the CREATE event is
+// delivered in the context of the thread itself. CREATE events for regular
+// threads, are delivered on the parent. We use this to tell apart which
+// threads are GCD workers with `thread == pthread_self()`.
+//
+static pthread_introspection_hook_t prev_pthread_introspection_hook;
+static ThreadEventCallbacks thread_event_callbacks;
+
+static void sanitizer_pthread_introspection_hook(unsigned int event,
+ pthread_t thread, void *addr,
+ size_t size) {
+ // create -> start -> terminate -> destroy
+ // * create/destroy are usually (not guaranteed) delivered on the parent and
+ // track resource allocation/reclamation
+ // * start/terminate are guaranteed to be delivered in the context of the
+ // thread and give hooks into "just after (before) thread starts (stops)
+ // executing"
+ DCHECK(event >= PTHREAD_INTROSPECTION_THREAD_CREATE &&
+ event <= PTHREAD_INTROSPECTION_THREAD_DESTROY);
+
+ if (event == PTHREAD_INTROSPECTION_THREAD_CREATE) {
+ bool gcd_worker = (thread == pthread_self());
+ if (thread_event_callbacks.create)
+ thread_event_callbacks.create((uptr)thread, gcd_worker);
+ } else if (event == PTHREAD_INTROSPECTION_THREAD_START) {
+ CHECK_EQ(thread, pthread_self());
+ if (thread_event_callbacks.start)
+ thread_event_callbacks.start((uptr)thread);
+ }
+
+ if (prev_pthread_introspection_hook)
+ prev_pthread_introspection_hook(event, thread, addr, size);
+
+ if (event == PTHREAD_INTROSPECTION_THREAD_TERMINATE) {
+ CHECK_EQ(thread, pthread_self());
+ if (thread_event_callbacks.terminate)
+ thread_event_callbacks.terminate((uptr)thread);
+ } else if (event == PTHREAD_INTROSPECTION_THREAD_DESTROY) {
+ if (thread_event_callbacks.destroy)
+ thread_event_callbacks.destroy((uptr)thread);
+ }
+}
+
+void InstallPthreadIntrospectionHook(const ThreadEventCallbacks &callbacks) {
+ thread_event_callbacks = callbacks;
+ prev_pthread_introspection_hook =
+ pthread_introspection_hook_install(&sanitizer_pthread_introspection_hook);
+}
+
} // namespace __sanitizer
-#endif // SANITIZER_MAC
+#endif // SANITIZER_APPLE
// This file is shared between various sanitizers' runtime libraries and
// provides definitions for OSX-specific functions.
//===----------------------------------------------------------------------===//
-#ifndef SANITIZER_MAC_H
-#define SANITIZER_MAC_H
+#ifndef SANITIZER_APPLE_H
+#define SANITIZER_APPLE_H
#include "sanitizer_common.h"
#include "sanitizer_platform.h"
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
#include "sanitizer_posix.h"
namespace __sanitizer {
void RestrictMemoryToMaxAddress(uptr max_address);
+using ThreadEventCallback = void (*)(uptr thread);
+using ThreadCreateEventCallback = void (*)(uptr thread, bool gcd_worker);
+struct ThreadEventCallbacks {
+ ThreadCreateEventCallback create;
+ ThreadEventCallback start;
+ ThreadEventCallback terminate;
+ ThreadEventCallback destroy;
+};
+
+void InstallPthreadIntrospectionHook(const ThreadEventCallbacks &callbacks);
+
} // namespace __sanitizer
-#endif // SANITIZER_MAC
-#endif // SANITIZER_MAC_H
+#endif // SANITIZER_APPLE
+#endif // SANITIZER_APPLE_H
//===----------------------------------------------------------------------===//
#include "sanitizer_platform.h"
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
#include "sanitizer_mac.h"
#include <sys/mman.h>
} // namespace __sanitizer
-#endif // SANITIZER_MAC
+#endif // SANITIZER_APPLE
//===----------------------------------------------------------------------===//
#include "sanitizer_common/sanitizer_platform.h"
-#if !SANITIZER_MAC
+#if !SANITIZER_APPLE
#error "This file should only be compiled on Darwin."
#endif
#include <sys/mman.h>
#include "interception/interception.h"
+#include "sanitizer_common/sanitizer_allocator_dlsym.h"
#include "sanitizer_common/sanitizer_mac.h"
// Similar code is used in Google Perftools,
return p;
}
+struct DlsymAlloc : public DlSymAllocator<DlsymAlloc> {
+ static bool UseImpl() { return !COMMON_MALLOC_SANITIZER_INITIALIZED; }
+};
+
extern "C"
SANITIZER_INTERFACE_ATTRIBUTE
void *__sanitizer_mz_calloc(malloc_zone_t *zone, size_t nmemb, size_t size) {
- if (UNLIKELY(!COMMON_MALLOC_SANITIZER_INITIALIZED)) {
- // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
- const size_t kCallocPoolSize = 1024;
- static uptr calloc_memory_for_dlsym[kCallocPoolSize];
- static size_t allocated;
- size_t size_in_words = ((nmemb * size) + kWordSize - 1) / kWordSize;
- void *mem = (void*)&calloc_memory_for_dlsym[allocated];
- allocated += size_in_words;
- CHECK(allocated < kCallocPoolSize);
- return mem;
- }
+ if (DlsymAlloc::Use())
+ return DlsymAlloc::Callocate(nmemb, size);
COMMON_MALLOC_CALLOC(nmemb, size);
return p;
}
SANITIZER_INTERFACE_ATTRIBUTE
void __sanitizer_mz_free(malloc_zone_t *zone, void *ptr) {
if (!ptr) return;
+ if (DlsymAlloc::PointerIsMine(ptr))
+ return DlsymAlloc::Free(ptr);
COMMON_MALLOC_FREE(ptr);
}
// Build adjacency matrix.
bool leaf[kMutexTypeMax];
internal_memset(&leaf, 0, sizeof(leaf));
- int cnt[kMutexTypeMax] = {};
+ int cnt[kMutexTypeMax];
internal_memset(&cnt, 0, sizeof(cnt));
for (int t = 0; t < kMutexTypeMax; t++) {
mutex_type_count = t;
if (max_idx != MutexInvalid && !mutex_can_lock[max_idx][type]) {
Printf("%s: internal deadlock: can't lock %s under %s mutex\n", SanitizerToolName,
mutex_meta[type].name, mutex_meta[max_idx].name);
- PrintMutexPC(pc);
+ PrintMutexPC(locked[max_idx].pc);
CHECK(0);
}
locked[type].seq = ++sequence;
namespace __sanitizer {
-class MUTEX StaticSpinMutex {
+class SANITIZER_MUTEX StaticSpinMutex {
public:
void Init() {
atomic_store(&state_, 0, memory_order_relaxed);
}
- void Lock() ACQUIRE() {
+ void Lock() SANITIZER_ACQUIRE() {
if (LIKELY(TryLock()))
return;
LockSlow();
}
- bool TryLock() TRY_ACQUIRE(true) {
+ bool TryLock() SANITIZER_TRY_ACQUIRE(true) {
return atomic_exchange(&state_, 1, memory_order_acquire) == 0;
}
- void Unlock() RELEASE() { atomic_store(&state_, 0, memory_order_release); }
+ void Unlock() SANITIZER_RELEASE() {
+ atomic_store(&state_, 0, memory_order_release);
+ }
- void CheckLocked() const CHECK_LOCKED() {
+ void CheckLocked() const SANITIZER_CHECK_LOCKED() {
CHECK_EQ(atomic_load(&state_, memory_order_relaxed), 1);
}
void LockSlow();
};
-class MUTEX SpinMutex : public StaticSpinMutex {
+class SANITIZER_MUTEX SpinMutex : public StaticSpinMutex {
public:
SpinMutex() {
Init();
// Go linker does not support THREADLOCAL variables,
// so we can't use per-thread state.
-#define SANITIZER_CHECK_DEADLOCKS (SANITIZER_DEBUG && !SANITIZER_GO)
+// Disable checked locks on Darwin. Although Darwin platforms support
+// THREADLOCAL variables they are not usable early on during process init when
+// `__sanitizer::Mutex` is used.
+#define SANITIZER_CHECK_DEADLOCKS \
+ (SANITIZER_DEBUG && !SANITIZER_GO && SANITIZER_SUPPORTS_THREADLOCAL && !SANITIZER_APPLE)
#if SANITIZER_CHECK_DEADLOCKS
struct MutexMeta {
class CheckedMutex {
public:
- constexpr CheckedMutex(MutexType type)
+ explicit constexpr CheckedMutex(MutexType type)
#if SANITIZER_CHECK_DEADLOCKS
: type_(type)
#endif
// Derive from CheckedMutex for the purposes of EBO.
// We could make it a field marked with [[no_unique_address]],
// but this attribute is not supported by some older compilers.
-class MUTEX Mutex : CheckedMutex {
+class SANITIZER_MUTEX Mutex : CheckedMutex {
public:
- constexpr Mutex(MutexType type = MutexUnchecked) : CheckedMutex(type) {}
+ explicit constexpr Mutex(MutexType type = MutexUnchecked)
+ : CheckedMutex(type) {}
- void Lock() ACQUIRE() {
+ void Lock() SANITIZER_ACQUIRE() {
CheckedMutex::Lock();
u64 reset_mask = ~0ull;
u64 state = atomic_load_relaxed(&state_);
- const uptr kMaxSpinIters = 1500;
for (uptr spin_iters = 0;; spin_iters++) {
u64 new_state;
bool locked = (state & (kWriterLock | kReaderLockMask)) != 0;
// We've incremented waiting writers, so now block.
writers_.Wait();
spin_iters = 0;
- state = atomic_load(&state_, memory_order_relaxed);
- DCHECK_NE(state & kWriterSpinWait, 0);
} else {
// We've set kWriterSpinWait, but we are still in active spinning.
}
// Either way we need to reset kWriterSpinWait
// next time we take the lock or block again.
reset_mask = ~kWriterSpinWait;
+ state = atomic_load(&state_, memory_order_relaxed);
+ DCHECK_NE(state & kWriterSpinWait, 0);
+ }
+ }
+
+ bool TryLock() SANITIZER_TRY_ACQUIRE(true) {
+ u64 state = atomic_load_relaxed(&state_);
+ for (;;) {
+ if (UNLIKELY(state & (kWriterLock | kReaderLockMask)))
+ return false;
+ // The mutex is not read-/write-locked, try to lock.
+ if (LIKELY(atomic_compare_exchange_weak(
+ &state_, &state, state | kWriterLock, memory_order_acquire))) {
+ CheckedMutex::Lock();
+ return true;
+ }
}
}
- void Unlock() RELEASE() {
+ void Unlock() SANITIZER_RELEASE() {
CheckedMutex::Unlock();
bool wake_writer;
u64 wake_readers;
DCHECK_NE(state & kWriterLock, 0);
DCHECK_EQ(state & kReaderLockMask, 0);
new_state = state & ~kWriterLock;
- wake_writer =
- (state & kWriterSpinWait) == 0 && (state & kWaitingWriterMask) != 0;
+ wake_writer = (state & (kWriterSpinWait | kReaderSpinWait)) == 0 &&
+ (state & kWaitingWriterMask) != 0;
if (wake_writer)
new_state = (new_state - kWaitingWriterInc) | kWriterSpinWait;
wake_readers =
- (state & (kWriterSpinWait | kWaitingWriterMask)) != 0
+ wake_writer || (state & kWriterSpinWait) != 0
? 0
: ((state & kWaitingReaderMask) >> kWaitingReaderShift);
if (wake_readers)
- new_state = (new_state & ~kWaitingReaderMask) +
- (wake_readers << kReaderLockShift);
+ new_state = (new_state & ~kWaitingReaderMask) | kReaderSpinWait;
} while (UNLIKELY(!atomic_compare_exchange_weak(&state_, &state, new_state,
memory_order_release)));
if (UNLIKELY(wake_writer))
readers_.Post(wake_readers);
}
- void ReadLock() ACQUIRE_SHARED() {
+ void ReadLock() SANITIZER_ACQUIRE_SHARED() {
CheckedMutex::Lock();
- bool locked;
- u64 new_state;
+ u64 reset_mask = ~0ull;
u64 state = atomic_load_relaxed(&state_);
- do {
- locked =
- (state & kReaderLockMask) == 0 &&
- (state & (kWriterLock | kWriterSpinWait | kWaitingWriterMask)) != 0;
+ for (uptr spin_iters = 0;; spin_iters++) {
+ bool locked = (state & kWriterLock) != 0;
+ u64 new_state;
+ if (LIKELY(!locked)) {
+ new_state = (state + kReaderLockInc) & reset_mask;
+ } else if (spin_iters > kMaxSpinIters) {
+ new_state = (state + kWaitingReaderInc) & reset_mask;
+ } else if ((state & kReaderSpinWait) == 0) {
+ // Active spinning, but denote our presence so that unlocking
+ // thread does not wake up other threads.
+ new_state = state | kReaderSpinWait;
+ } else {
+ // Active spinning.
+ state = atomic_load(&state_, memory_order_relaxed);
+ continue;
+ }
+ if (UNLIKELY(!atomic_compare_exchange_weak(&state_, &state, new_state,
+ memory_order_acquire)))
+ continue;
if (LIKELY(!locked))
- new_state = state + kReaderLockInc;
- else
- new_state = state + kWaitingReaderInc;
- } while (UNLIKELY(!atomic_compare_exchange_weak(&state_, &state, new_state,
- memory_order_acquire)));
- if (UNLIKELY(locked))
- readers_.Wait();
- DCHECK_EQ(atomic_load_relaxed(&state_) & kWriterLock, 0);
- DCHECK_NE(atomic_load_relaxed(&state_) & kReaderLockMask, 0);
+ return; // We've locked the mutex.
+ if (spin_iters > kMaxSpinIters) {
+ // We've incremented waiting readers, so now block.
+ readers_.Wait();
+ spin_iters = 0;
+ } else {
+ // We've set kReaderSpinWait, but we are still in active spinning.
+ }
+ reset_mask = ~kReaderSpinWait;
+ state = atomic_load(&state_, memory_order_relaxed);
+ }
}
- void ReadUnlock() RELEASE_SHARED() {
+ void ReadUnlock() SANITIZER_RELEASE_SHARED() {
CheckedMutex::Unlock();
bool wake;
u64 new_state;
u64 state = atomic_load_relaxed(&state_);
do {
DCHECK_NE(state & kReaderLockMask, 0);
- DCHECK_EQ(state & (kWaitingReaderMask | kWriterLock), 0);
+ DCHECK_EQ(state & kWriterLock, 0);
new_state = state - kReaderLockInc;
- wake = (new_state & (kReaderLockMask | kWriterSpinWait)) == 0 &&
+ wake = (new_state &
+ (kReaderLockMask | kWriterSpinWait | kReaderSpinWait)) == 0 &&
(new_state & kWaitingWriterMask) != 0;
if (wake)
new_state = (new_state - kWaitingWriterInc) | kWriterSpinWait;
// owns the mutex but a child checks that it is locked. Rather than
// maintaining complex state to work around those situations, the check only
// checks that the mutex is owned.
- void CheckWriteLocked() const CHECK_LOCKED() {
+ void CheckWriteLocked() const SANITIZER_CHECK_LOCKED() {
CHECK(atomic_load(&state_, memory_order_relaxed) & kWriterLock);
}
- void CheckLocked() const CHECK_LOCKED() { CheckWriteLocked(); }
+ void CheckLocked() const SANITIZER_CHECK_LOCKED() { CheckWriteLocked(); }
- void CheckReadLocked() const CHECK_LOCKED() {
+ void CheckReadLocked() const SANITIZER_CHECK_LOCKED() {
CHECK(atomic_load(&state_, memory_order_relaxed) & kReaderLockMask);
}
// - a writer is awake and spin-waiting
// the flag is used to prevent thundering herd problem
// (new writers are not woken if this flag is set)
+ // - a reader is awake and spin-waiting
//
- // Writer support active spinning, readers does not.
+ // Both writers and readers use active spinning before blocking.
// But readers are more aggressive and always take the mutex
// if there are any other readers.
- // Writers hand off the mutex to readers: after wake up readers
- // already assume ownership of the mutex (don't need to do any
- // state updates). But the mutex is not handed off to writers,
- // after wake up writers compete to lock the mutex again.
- // This is needed to allow repeated write locks even in presence
- // of other blocked writers.
+ // After wake up both writers and readers compete to lock the
+ // mutex again. This is needed to allow repeated locks even in presence
+ // of other blocked threads.
static constexpr u64 kCounterWidth = 20;
static constexpr u64 kReaderLockShift = 0;
static constexpr u64 kReaderLockInc = 1ull << kReaderLockShift;
<< kWaitingWriterShift;
static constexpr u64 kWriterLock = 1ull << (3 * kCounterWidth);
static constexpr u64 kWriterSpinWait = 1ull << (3 * kCounterWidth + 1);
+ static constexpr u64 kReaderSpinWait = 1ull << (3 * kCounterWidth + 2);
+
+ static constexpr uptr kMaxSpinIters = 1500;
+ Mutex(LinkerInitialized) = delete;
Mutex(const Mutex &) = delete;
void operator=(const Mutex &) = delete;
};
void FutexWait(atomic_uint32_t *p, u32 cmp);
void FutexWake(atomic_uint32_t *p, u32 count);
-class MUTEX BlockingMutex {
- public:
- explicit constexpr BlockingMutex(LinkerInitialized)
- : opaque_storage_ {0, }, owner_ {0} {}
- BlockingMutex();
- void Lock() ACQUIRE();
- void Unlock() RELEASE();
-
- // This function does not guarantee an explicit check that the calling thread
- // is the thread which owns the mutex. This behavior, while more strictly
- // correct, causes problems in cases like StopTheWorld, where a parent thread
- // owns the mutex but a child checks that it is locked. Rather than
- // maintaining complex state to work around those situations, the check only
- // checks that the mutex is owned, and assumes callers to be generally
- // well-behaved.
- void CheckLocked() const CHECK_LOCKED();
-
- private:
- // Solaris mutex_t has a member that requires 64-bit alignment.
- ALIGNED(8) uptr opaque_storage_[10];
- uptr owner_; // for debugging
-};
-
-// Reader-writer spin mutex.
-class MUTEX RWMutex {
+template <typename MutexType>
+class SANITIZER_SCOPED_LOCK GenericScopedLock {
public:
- RWMutex() {
- atomic_store(&state_, kUnlocked, memory_order_relaxed);
- }
-
- ~RWMutex() {
- CHECK_EQ(atomic_load(&state_, memory_order_relaxed), kUnlocked);
- }
-
- void Lock() ACQUIRE() {
- u32 cmp = kUnlocked;
- if (atomic_compare_exchange_strong(&state_, &cmp, kWriteLock,
- memory_order_acquire))
- return;
- LockSlow();
- }
-
- void Unlock() RELEASE() {
- u32 prev = atomic_fetch_sub(&state_, kWriteLock, memory_order_release);
- DCHECK_NE(prev & kWriteLock, 0);
- (void)prev;
- }
-
- void ReadLock() ACQUIRE_SHARED() {
- u32 prev = atomic_fetch_add(&state_, kReadLock, memory_order_acquire);
- if ((prev & kWriteLock) == 0)
- return;
- ReadLockSlow();
- }
-
- void ReadUnlock() RELEASE_SHARED() {
- u32 prev = atomic_fetch_sub(&state_, kReadLock, memory_order_release);
- DCHECK_EQ(prev & kWriteLock, 0);
- DCHECK_GT(prev & ~kWriteLock, 0);
- (void)prev;
+ explicit GenericScopedLock(MutexType *mu) SANITIZER_ACQUIRE(mu) : mu_(mu) {
+ mu_->Lock();
}
- void CheckLocked() const CHECK_LOCKED() {
- CHECK_NE(atomic_load(&state_, memory_order_relaxed), kUnlocked);
- }
+ ~GenericScopedLock() SANITIZER_RELEASE() { mu_->Unlock(); }
private:
- atomic_uint32_t state_;
-
- enum {
- kUnlocked = 0,
- kWriteLock = 1,
- kReadLock = 2
- };
-
- void NOINLINE LockSlow() {
- for (int i = 0;; i++) {
- if (i < 10)
- proc_yield(10);
- else
- internal_sched_yield();
- u32 cmp = atomic_load(&state_, memory_order_relaxed);
- if (cmp == kUnlocked &&
- atomic_compare_exchange_weak(&state_, &cmp, kWriteLock,
- memory_order_acquire))
- return;
- }
- }
-
- void NOINLINE ReadLockSlow() {
- for (int i = 0;; i++) {
- if (i < 10)
- proc_yield(10);
- else
- internal_sched_yield();
- u32 prev = atomic_load(&state_, memory_order_acquire);
- if ((prev & kWriteLock) == 0)
- return;
- }
- }
+ MutexType *mu_;
- RWMutex(const RWMutex &) = delete;
- void operator=(const RWMutex &) = delete;
+ GenericScopedLock(const GenericScopedLock &) = delete;
+ void operator=(const GenericScopedLock &) = delete;
};
template <typename MutexType>
-class SCOPED_LOCK GenericScopedLock {
+class SANITIZER_SCOPED_LOCK GenericScopedReadLock {
public:
- explicit GenericScopedLock(MutexType *mu) ACQUIRE(mu) : mu_(mu) {
- mu_->Lock();
+ explicit GenericScopedReadLock(MutexType *mu) SANITIZER_ACQUIRE(mu)
+ : mu_(mu) {
+ mu_->ReadLock();
}
- ~GenericScopedLock() RELEASE() { mu_->Unlock(); }
+ ~GenericScopedReadLock() SANITIZER_RELEASE() { mu_->ReadUnlock(); }
private:
MutexType *mu_;
- GenericScopedLock(const GenericScopedLock &) = delete;
- void operator=(const GenericScopedLock &) = delete;
+ GenericScopedReadLock(const GenericScopedReadLock &) = delete;
+ void operator=(const GenericScopedReadLock &) = delete;
};
template <typename MutexType>
-class SCOPED_LOCK GenericScopedReadLock {
+class SANITIZER_SCOPED_LOCK GenericScopedRWLock {
public:
- explicit GenericScopedReadLock(MutexType *mu) ACQUIRE(mu) : mu_(mu) {
- mu_->ReadLock();
+ ALWAYS_INLINE explicit GenericScopedRWLock(MutexType *mu, bool write)
+ SANITIZER_ACQUIRE(mu)
+ : mu_(mu), write_(write) {
+ if (write_)
+ mu_->Lock();
+ else
+ mu_->ReadLock();
}
- ~GenericScopedReadLock() RELEASE() { mu_->ReadUnlock(); }
+ ALWAYS_INLINE ~GenericScopedRWLock() SANITIZER_RELEASE() {
+ if (write_)
+ mu_->Unlock();
+ else
+ mu_->ReadUnlock();
+ }
private:
MutexType *mu_;
+ bool write_;
- GenericScopedReadLock(const GenericScopedReadLock &) = delete;
- void operator=(const GenericScopedReadLock &) = delete;
+ GenericScopedRWLock(const GenericScopedRWLock &) = delete;
+ void operator=(const GenericScopedRWLock &) = delete;
};
typedef GenericScopedLock<StaticSpinMutex> SpinMutexLock;
-typedef GenericScopedLock<BlockingMutex> BlockingMutexLock;
-typedef GenericScopedLock<RWMutex> RWMutexLock;
-typedef GenericScopedReadLock<RWMutex> RWMutexReadLock;
typedef GenericScopedLock<Mutex> Lock;
typedef GenericScopedReadLock<Mutex> ReadLock;
+typedef GenericScopedRWLock<Mutex> RWLock;
} // namespace __sanitizer
#define SI_LINUX 0
#endif
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
#define SI_MAC 1
#define SI_NOT_MAC 0
#else
#define SI_SOLARIS32 0
#endif
-#if SANITIZER_POSIX && !SANITIZER_MAC
+#if SANITIZER_POSIX && !SANITIZER_APPLE
#define SI_POSIX_NOT_MAC 1
#else
#define SI_POSIX_NOT_MAC 0
(SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
#define SANITIZER_INTERCEPT_CLOCK_GETTIME \
(SI_FREEBSD || SI_NETBSD || SI_LINUX || SI_SOLARIS)
-#define SANITIZER_INTERCEPT_CLOCK_GETCPUCLOCKID SI_LINUX
+#define SANITIZER_INTERCEPT_CLOCK_GETCPUCLOCKID \
+ (SI_LINUX || SI_FREEBSD || SI_NETBSD)
#define SANITIZER_INTERCEPT_GETITIMER SI_POSIX
#define SANITIZER_INTERCEPT_TIME SI_POSIX
#define SANITIZER_INTERCEPT_GLOB (SI_GLIBC || SI_SOLARIS)
#define SANITIZER_INTERCEPT_GLOB64 SI_GLIBC
+#define SANITIZER_INTERCEPT___B64_TO SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_DN_COMP_EXPAND SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_POSIX_SPAWN SI_POSIX
#define SANITIZER_INTERCEPT_WAIT SI_POSIX
#define SANITIZER_INTERCEPT_INET SI_POSIX
#define SANITIZER_INTERCEPT_PTHREAD_GETSCHEDPARAM SI_POSIX
#define SANITIZER_INTERCEPT_GETHOSTENT_R (SI_FREEBSD || SI_GLIBC || SI_SOLARIS)
#define SANITIZER_INTERCEPT_GETSOCKOPT SI_POSIX
#define SANITIZER_INTERCEPT_ACCEPT SI_POSIX
-#define SANITIZER_INTERCEPT_ACCEPT4 (SI_LINUX_NOT_ANDROID || SI_NETBSD)
+#define SANITIZER_INTERCEPT_ACCEPT4 \
+ (SI_LINUX_NOT_ANDROID || SI_NETBSD || SI_FREEBSD)
#define SANITIZER_INTERCEPT_PACCEPT SI_NETBSD
#define SANITIZER_INTERCEPT_MODF SI_POSIX
#define SANITIZER_INTERCEPT_RECVMSG SI_POSIX
#define SANITIZER_INTERCEPT_INET_ATON SI_POSIX
#define SANITIZER_INTERCEPT_SYSINFO SI_LINUX
#define SANITIZER_INTERCEPT_READDIR SI_POSIX
-#define SANITIZER_INTERCEPT_READDIR64 SI_LINUX_NOT_ANDROID || SI_SOLARIS32
+#define SANITIZER_INTERCEPT_READDIR64 SI_GLIBC || SI_SOLARIS32
#if SI_LINUX_NOT_ANDROID && \
(defined(__i386) || defined(__x86_64) || defined(__mips64) || \
defined(__powerpc64__) || defined(__aarch64__) || defined(__arm__) || \
- defined(__s390__) || SANITIZER_RISCV64)
+ defined(__s390__) || defined(__loongarch__) || SANITIZER_RISCV64)
#define SANITIZER_INTERCEPT_PTRACE 1
#else
#define SANITIZER_INTERCEPT_PTRACE 0
#define SANITIZER_INTERCEPT_XPG_STRERROR_R SI_LINUX_NOT_ANDROID
#define SANITIZER_INTERCEPT_SCANDIR \
(SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
-#define SANITIZER_INTERCEPT_SCANDIR64 SI_LINUX_NOT_ANDROID || SI_SOLARIS32
+#define SANITIZER_INTERCEPT_SCANDIR64 SI_GLIBC || SI_SOLARIS32
#define SANITIZER_INTERCEPT_GETGROUPS SI_POSIX
#define SANITIZER_INTERCEPT_POLL SI_POSIX
#define SANITIZER_INTERCEPT_PPOLL SI_LINUX_NOT_ANDROID || SI_SOLARIS
#define SANITIZER_INTERCEPT_WORDEXP \
(SI_FREEBSD || SI_NETBSD || (SI_MAC && !SI_IOS) || SI_LINUX_NOT_ANDROID || \
- SI_SOLARIS) // NOLINT
+ SI_SOLARIS)
#define SANITIZER_INTERCEPT_SIGWAIT SI_POSIX
#define SANITIZER_INTERCEPT_SIGWAITINFO SI_LINUX_NOT_ANDROID || SI_SOLARIS
#define SANITIZER_INTERCEPT_SIGTIMEDWAIT SI_LINUX_NOT_ANDROID || SI_SOLARIS
#define SANITIZER_INTERCEPT_GETMNTENT_R SI_LINUX_NOT_ANDROID
#define SANITIZER_INTERCEPT_STATFS \
(SI_FREEBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
-#define SANITIZER_INTERCEPT_STATFS64 \
- (((SI_MAC && !TARGET_CPU_ARM64) && !SI_IOS) || SI_LINUX_NOT_ANDROID)
+#define SANITIZER_INTERCEPT_STATFS64 SI_GLIBC && SANITIZER_HAS_STATFS64
#define SANITIZER_INTERCEPT_STATVFS \
(SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID)
-#define SANITIZER_INTERCEPT_STATVFS64 SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_STATVFS64 SI_GLIBC
#define SANITIZER_INTERCEPT_INITGROUPS SI_POSIX
#define SANITIZER_INTERCEPT_ETHER_NTOA_ATON SI_POSIX
#define SANITIZER_INTERCEPT_ETHER_HOST \
#define SANITIZER_INTERCEPT_ETHER_R (SI_FREEBSD || SI_LINUX_NOT_ANDROID)
#define SANITIZER_INTERCEPT_SHMCTL \
(((SI_FREEBSD || SI_LINUX_NOT_ANDROID) && SANITIZER_WORDSIZE == 64) || \
- SI_NETBSD || SI_SOLARIS) // NOLINT
+ SI_NETBSD || SI_SOLARIS)
#define SANITIZER_INTERCEPT_RANDOM_R SI_GLIBC
#define SANITIZER_INTERCEPT_PTHREAD_ATTR_GET SI_POSIX
#define SANITIZER_INTERCEPT_PTHREAD_ATTR_GETINHERITSCHED \
(SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
#define SANITIZER_INTERCEPT_PTHREAD_ATTR_GETAFFINITY_NP SI_GLIBC
+#define SANITIZER_INTERCEPT_PTHREAD_GETAFFINITY_NP SI_LINUX_NOT_ANDROID
#define SANITIZER_INTERCEPT_PTHREAD_ATTR_GET_SCHED SI_POSIX
#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPSHARED \
(SI_POSIX && !SI_NETBSD)
#define SANITIZER_INTERCEPT__EXIT \
(SI_LINUX || SI_FREEBSD || SI_NETBSD || SI_MAC || SI_SOLARIS)
-#define SANITIZER_INTERCEPT_PTHREAD_MUTEX SI_POSIX
-#define SANITIZER_INTERCEPT___PTHREAD_MUTEX SI_GLIBC
#define SANITIZER_INTERCEPT___LIBC_MUTEX SI_NETBSD
#define SANITIZER_INTERCEPT_PTHREAD_SETNAME_NP \
(SI_FREEBSD || SI_NETBSD || SI_GLIBC || SI_SOLARIS)
(SI_FREEBSD || SI_NETBSD || SI_GLIBC || SI_SOLARIS)
#define SANITIZER_INTERCEPT_TLS_GET_ADDR \
- (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+ (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID)
#define SANITIZER_INTERCEPT_LISTXATTR SI_LINUX
#define SANITIZER_INTERCEPT_GETXATTR SI_LINUX
#define SANITIZER_INTERCEPT_SEM \
(SI_LINUX || SI_FREEBSD || SI_NETBSD || SI_SOLARIS)
#define SANITIZER_INTERCEPT_PTHREAD_SETCANCEL SI_POSIX
-#define SANITIZER_INTERCEPT_MINCORE (SI_LINUX || SI_NETBSD || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_MINCORE \
+ (SI_LINUX || SI_NETBSD || SI_FREEBSD || SI_SOLARIS)
#define SANITIZER_INTERCEPT_PROCESS_VM_READV SI_LINUX
#define SANITIZER_INTERCEPT_CTERMID \
(SI_LINUX || SI_MAC || SI_FREEBSD || SI_NETBSD || SI_SOLARIS)
#define SANITIZER_INTERCEPT_SEND_SENDTO SI_POSIX
#define SANITIZER_INTERCEPT_EVENTFD_READ_WRITE SI_LINUX
-#define SANITIZER_INTERCEPT_STAT \
- (SI_FREEBSD || SI_MAC || SI_ANDROID || SI_NETBSD || SI_SOLARIS)
-#define SANITIZER_INTERCEPT_LSTAT (SI_NETBSD || SI_FREEBSD)
-#define SANITIZER_INTERCEPT___XSTAT (!SANITIZER_INTERCEPT_STAT && SI_POSIX)
-#define SANITIZER_INTERCEPT___XSTAT64 SI_LINUX_NOT_ANDROID
+#define SI_STAT_LINUX (SI_LINUX && __GLIBC_PREREQ(2, 33))
+#define SANITIZER_INTERCEPT_STAT \
+ (SI_FREEBSD || SI_MAC || SI_ANDROID || SI_NETBSD || SI_SOLARIS || \
+ SI_STAT_LINUX)
+#define SANITIZER_INTERCEPT_STAT64 SI_STAT_LINUX && SANITIZER_HAS_STAT64
+#define SANITIZER_INTERCEPT_LSTAT (SI_NETBSD || SI_FREEBSD || SI_STAT_LINUX)
+#define SANITIZER_INTERCEPT___XSTAT \
+ ((!SANITIZER_INTERCEPT_STAT && SI_POSIX) || SI_STAT_LINUX)
+#define SANITIZER_INTERCEPT___XSTAT64 SI_GLIBC
#define SANITIZER_INTERCEPT___LXSTAT SANITIZER_INTERCEPT___XSTAT
-#define SANITIZER_INTERCEPT___LXSTAT64 SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT___LXSTAT64 SI_GLIBC
#define SANITIZER_INTERCEPT_UTMP \
(SI_POSIX && !SI_MAC && !SI_FREEBSD && !SI_NETBSD)
(SI_LINUX_NOT_ANDROID || SI_MAC || SI_FREEBSD || SI_NETBSD)
#define SANITIZER_INTERCEPT_MMAP SI_POSIX
-#define SANITIZER_INTERCEPT_MMAP64 SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_MMAP64 SI_GLIBC || SI_SOLARIS
#define SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO (SI_GLIBC || SI_ANDROID)
#define SANITIZER_INTERCEPT_MEMALIGN (!SI_FREEBSD && !SI_MAC && !SI_NETBSD)
#define SANITIZER_INTERCEPT___LIBC_MEMALIGN SI_GLIBC
#define SANITIZER_INTERCEPT_GID_FROM_GROUP SI_NETBSD
#define SANITIZER_INTERCEPT_ACCESS (SI_NETBSD || SI_FREEBSD)
#define SANITIZER_INTERCEPT_FACCESSAT (SI_NETBSD || SI_FREEBSD)
-#define SANITIZER_INTERCEPT_GETGROUPLIST SI_NETBSD
+#define SANITIZER_INTERCEPT_GETGROUPLIST \
+ (SI_NETBSD || SI_FREEBSD || SI_LINUX)
#define SANITIZER_INTERCEPT_STRLCPY \
(SI_NETBSD || SI_FREEBSD || SI_MAC || SI_ANDROID)
#define SANITIZER_INTERCEPT_DEVNAME_R (SI_NETBSD || SI_FREEBSD)
#define SANITIZER_INTERCEPT_FGETLN (SI_NETBSD || SI_FREEBSD)
#define SANITIZER_INTERCEPT_STRMODE (SI_NETBSD || SI_FREEBSD)
-#define SANITIZER_INTERCEPT_TTYENT SI_NETBSD
-#define SANITIZER_INTERCEPT_PROTOENT (SI_NETBSD || SI_LINUX)
+#define SANITIZER_INTERCEPT_TTYENT (SI_NETBSD || SI_FREEBSD)
+#define SANITIZER_INTERCEPT_TTYENTPATH SI_NETBSD
+#define SANITIZER_INTERCEPT_PROTOENT (SI_LINUX || SI_NETBSD || SI_FREEBSD)
#define SANITIZER_INTERCEPT_PROTOENT_R SI_GLIBC
-#define SANITIZER_INTERCEPT_NETENT SI_NETBSD
+#define SANITIZER_INTERCEPT_NETENT (SI_LINUX || SI_NETBSD || SI_FREEBSD)
#define SANITIZER_INTERCEPT_SETVBUF \
(SI_NETBSD || SI_FREEBSD || SI_LINUX || SI_MAC)
#define SANITIZER_INTERCEPT_GETMNTINFO (SI_NETBSD || SI_FREEBSD || SI_MAC)
#define SANITIZER_INTERCEPT_MODCTL SI_NETBSD
#define SANITIZER_INTERCEPT_CAPSICUM SI_FREEBSD
#define SANITIZER_INTERCEPT_STRTONUM (SI_NETBSD || SI_FREEBSD)
-#define SANITIZER_INTERCEPT_FPARSELN SI_NETBSD
+#define SANITIZER_INTERCEPT_FPARSELN (SI_NETBSD || SI_FREEBSD)
#define SANITIZER_INTERCEPT_STATVFS1 SI_NETBSD
#define SANITIZER_INTERCEPT_STRTOI SI_NETBSD
#define SANITIZER_INTERCEPT_CAPSICUM SI_FREEBSD
#define SANITIZER_INTERCEPT_SHA1 SI_NETBSD
#define SANITIZER_INTERCEPT_MD4 SI_NETBSD
#define SANITIZER_INTERCEPT_RMD160 SI_NETBSD
-#define SANITIZER_INTERCEPT_MD5 SI_NETBSD
+#define SANITIZER_INTERCEPT_MD5 (SI_NETBSD || SI_FREEBSD)
#define SANITIZER_INTERCEPT_FSEEK (SI_NETBSD || SI_FREEBSD)
#define SANITIZER_INTERCEPT_MD2 SI_NETBSD
-#define SANITIZER_INTERCEPT_SHA2 SI_NETBSD
+#define SANITIZER_INTERCEPT_SHA2 (SI_NETBSD || SI_FREEBSD)
#define SANITIZER_INTERCEPT_CDB SI_NETBSD
#define SANITIZER_INTERCEPT_VIS (SI_NETBSD || SI_FREEBSD)
#define SANITIZER_INTERCEPT_POPEN SI_POSIX
#define SANITIZER_INTERCEPT_QSORT \
(SI_POSIX && !SI_IOSSIM && !SI_WATCHOS && !SI_TVOS && !SI_ANDROID)
#define SANITIZER_INTERCEPT_QSORT_R SI_GLIBC
+#define SANITIZER_INTERCEPT_BSEARCH \
+ (SI_POSIX && !SI_IOSSIM && !SI_WATCHOS && !SI_TVOS && !SI_ANDROID)
// sigaltstack on i386 macOS cannot be intercepted due to setjmp()
// calling it and assuming that it does not clobber registers.
#define SANITIZER_INTERCEPT_SIGALTSTACK \
- (SI_POSIX && !(SANITIZER_MAC && SANITIZER_I386))
+ (SI_POSIX && !(SANITIZER_APPLE && SANITIZER_I386))
#define SANITIZER_INTERCEPT_UNAME (SI_POSIX && !SI_FREEBSD)
#define SANITIZER_INTERCEPT___XUNAME SI_FREEBSD
#define SANITIZER_INTERCEPT_FLOPEN SI_FREEBSD
+#define SANITIZER_INTERCEPT_PROCCTL SI_FREEBSD
+#define SANITIZER_INTERCEPT_HEXDUMP SI_FREEBSD
// This macro gives a way for downstream users to override the above
// interceptor macros irrespective of the platform they are on. They have
#include <semaphore.h>
#include <signal.h>
#include <stddef.h>
+#include <md5.h>
+#include <sha224.h>
+#include <sha256.h>
+#include <sha384.h>
+#include <sha512.h>
#include <stdio.h>
#include <stringlist.h>
#include <term.h>
#include <termios.h>
#include <time.h>
+#include <ttyent.h>
#include <utime.h>
#include <utmpx.h>
#include <vis.h>
unsigned struct_sched_param_sz = sizeof(struct sched_param);
unsigned struct_statfs_sz = sizeof(struct statfs);
unsigned struct_sockaddr_sz = sizeof(struct sockaddr);
-unsigned ucontext_t_sz = sizeof(ucontext_t);
+unsigned ucontext_t_sz(void *ctx) { return sizeof(ucontext_t); }
unsigned struct_rlimit_sz = sizeof(struct rlimit);
unsigned struct_timespec_sz = sizeof(struct timespec);
unsigned struct_utimbuf_sz = sizeof(struct utimbuf);
unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr);
int glob_nomatch = GLOB_NOMATCH;
int glob_altdirfunc = GLOB_ALTDIRFUNC;
+const int wordexp_wrde_dooffs = WRDE_DOOFFS;
unsigned path_max = PATH_MAX;
+int struct_ttyent_sz = sizeof(struct ttyent);
+
// ioctl arguments
unsigned struct_ifreq_sz = sizeof(struct ifreq);
unsigned struct_termios_sz = sizeof(struct termios);
unsigned struct_ppp_stats_sz = sizeof(struct ppp_stats);
unsigned struct_sioc_sg_req_sz = sizeof(struct sioc_sg_req);
unsigned struct_sioc_vif_req_sz = sizeof(struct sioc_vif_req);
+unsigned struct_procctl_reaper_status_sz = sizeof(struct __sanitizer_procctl_reaper_status);
+unsigned struct_procctl_reaper_pidinfo_sz = sizeof(struct __sanitizer_procctl_reaper_pidinfo);
+unsigned struct_procctl_reaper_pids_sz = sizeof(struct __sanitizer_procctl_reaper_pids);
+unsigned struct_procctl_reaper_kill_sz = sizeof(struct __sanitizer_procctl_reaper_kill);
const unsigned long __sanitizer_bufsiz = BUFSIZ;
const unsigned IOCTL_NOT_PRESENT = 0;
const int si_SEGV_ACCERR = SEGV_ACCERR;
const int unvis_valid = UNVIS_VALID;
const int unvis_validpush = UNVIS_VALIDPUSH;
+
+const unsigned MD5_CTX_sz = sizeof(MD5_CTX);
+const unsigned MD5_return_length = MD5_DIGEST_STRING_LENGTH;
+
+#define SHA2_CONST(LEN) \
+ const unsigned SHA##LEN##_CTX_sz = sizeof(SHA##LEN##_CTX); \
+ const unsigned SHA##LEN##_return_length = SHA##LEN##_DIGEST_STRING_LENGTH; \
+ const unsigned SHA##LEN##_block_length = SHA##LEN##_BLOCK_LENGTH; \
+ const unsigned SHA##LEN##_digest_length = SHA##LEN##_DIGEST_LENGTH
+
+SHA2_CONST(224);
+SHA2_CONST(256);
+SHA2_CONST(384);
+SHA2_CONST(512);
+
+#undef SHA2_CONST
} // namespace __sanitizer
using namespace __sanitizer;
#if SANITIZER_FREEBSD
-#include "sanitizer_internal_defs.h"
-#include "sanitizer_platform.h"
-#include "sanitizer_platform_limits_posix.h"
+# include "sanitizer_internal_defs.h"
+# include "sanitizer_platform.h"
+# include "sanitizer_platform_limits_posix.h"
// Get sys/_types.h, because that tells us whether 64-bit inodes are
// used in struct dirent below.
-#include <sys/_types.h>
+# include <sys/_types.h>
namespace __sanitizer {
void *__sanitizer_get_link_map_by_dlopen_handle(void *handle);
-#define GET_LINK_MAP_BY_DLOPEN_HANDLE(handle) \
- (link_map *)__sanitizer_get_link_map_by_dlopen_handle(handle)
+# define GET_LINK_MAP_BY_DLOPEN_HANDLE(handle) \
+ (link_map *)__sanitizer_get_link_map_by_dlopen_handle(handle)
extern unsigned struct_utsname_sz;
extern unsigned struct_stat_sz;
-#if defined(__powerpc64__)
+# if defined(__powerpc64__)
const unsigned struct___old_kernel_stat_sz = 0;
-#else
+# else
const unsigned struct___old_kernel_stat_sz = 32;
-#endif
+# endif
extern unsigned struct_rusage_sz;
extern unsigned siginfo_t_sz;
extern unsigned struct_itimerval_sz;
extern unsigned struct_statfs64_sz;
extern unsigned struct_statfs_sz;
extern unsigned struct_sockaddr_sz;
-extern unsigned ucontext_t_sz;
+unsigned ucontext_t_sz(void *ctx);
extern unsigned struct_rlimit_sz;
extern unsigned struct_utimbuf_sz;
extern unsigned struct_timespec_sz;
long key;
};
-#if !defined(__i386__)
+struct __sanitizer_protoent {
+ char *p_name;
+ char **p_aliases;
+ int p_proto;
+};
+
+struct __sanitizer_netent {
+ char *n_name;
+ char **n_aliases;
+ int n_addrtype;
+ u32 n_net;
+};
+
+# if !defined(__i386__)
typedef long long __sanitizer_time_t;
-#else
+# else
typedef long __sanitizer_time_t;
-#endif
+# endif
struct __sanitizer_shmid_ds {
__sanitizer_ipc_perm shm_perm;
unsigned int ifa_flags;
void *ifa_addr; // (struct sockaddr *)
void *ifa_netmask; // (struct sockaddr *)
-#undef ifa_dstaddr
+# undef ifa_dstaddr
void *ifa_dstaddr; // (struct sockaddr *)
void *ifa_data;
};
};
struct __sanitizer_dirent {
-#if defined(__INO64)
+# if defined(__INO64)
unsigned long long d_fileno;
unsigned long long d_off;
-#else
+# else
unsigned int d_fileno;
-#endif
+# endif
unsigned short d_reclen;
// more fields that we don't care about
};
typedef int __sanitizer_clock_t;
typedef int __sanitizer_clockid_t;
-#if defined(_LP64) || defined(__x86_64__) || defined(__powerpc__) || \
- defined(__mips__)
+# if defined(_LP64) || defined(__x86_64__) || defined(__powerpc__) || \
+ defined(__mips__)
typedef unsigned __sanitizer___kernel_uid_t;
typedef unsigned __sanitizer___kernel_gid_t;
-#else
+# else
typedef unsigned short __sanitizer___kernel_uid_t;
typedef unsigned short __sanitizer___kernel_gid_t;
-#endif
+# endif
typedef long long __sanitizer___kernel_off_t;
-#if defined(__powerpc__) || defined(__mips__)
+# if defined(__powerpc__) || defined(__mips__)
typedef unsigned int __sanitizer___kernel_old_uid_t;
typedef unsigned int __sanitizer___kernel_old_gid_t;
-#else
+# else
typedef unsigned short __sanitizer___kernel_old_uid_t;
typedef unsigned short __sanitizer___kernel_old_gid_t;
-#endif
+# endif
typedef long long __sanitizer___kernel_loff_t;
typedef struct {
extern int glob_nomatch;
extern int glob_altdirfunc;
+extern const int wordexp_wrde_dooffs;
extern unsigned path_max;
+extern int struct_ttyent_sz;
+
struct __sanitizer_wordexp_t {
uptr we_wordc;
char **we_wordv;
} ifc_ifcu;
};
-#define IOC_NRBITS 8
-#define IOC_TYPEBITS 8
-#if defined(__powerpc__) || defined(__powerpc64__) || defined(__mips__)
-#define IOC_SIZEBITS 13
-#define IOC_DIRBITS 3
-#define IOC_NONE 1U
-#define IOC_WRITE 4U
-#define IOC_READ 2U
-#else
-#define IOC_SIZEBITS 14
-#define IOC_DIRBITS 2
-#define IOC_NONE 0U
-#define IOC_WRITE 1U
-#define IOC_READ 2U
-#endif
-#define IOC_NRMASK ((1 << IOC_NRBITS) - 1)
-#define IOC_TYPEMASK ((1 << IOC_TYPEBITS) - 1)
-#define IOC_SIZEMASK ((1 << IOC_SIZEBITS) - 1)
-#if defined(IOC_DIRMASK)
-#undef IOC_DIRMASK
-#endif
-#define IOC_DIRMASK ((1 << IOC_DIRBITS) - 1)
-#define IOC_NRSHIFT 0
-#define IOC_TYPESHIFT (IOC_NRSHIFT + IOC_NRBITS)
-#define IOC_SIZESHIFT (IOC_TYPESHIFT + IOC_TYPEBITS)
-#define IOC_DIRSHIFT (IOC_SIZESHIFT + IOC_SIZEBITS)
-#define EVIOC_EV_MAX 0x1f
-#define EVIOC_ABS_MAX 0x3f
-
-#define IOC_DIR(nr) (((nr) >> IOC_DIRSHIFT) & IOC_DIRMASK)
-#define IOC_TYPE(nr) (((nr) >> IOC_TYPESHIFT) & IOC_TYPEMASK)
-#define IOC_NR(nr) (((nr) >> IOC_NRSHIFT) & IOC_NRMASK)
-#define IOC_SIZE(nr) (((nr) >> IOC_SIZESHIFT) & IOC_SIZEMASK)
+struct __sanitizer__ttyent {
+ char *ty_name;
+ char *ty_getty;
+ char *ty_type;
+ int ty_status;
+ char *ty_window;
+ char *ty_comment;
+ char *ty_group;
+};
+
+// procctl reaper data for PROCCTL_REAPER flags
+struct __sanitizer_procctl_reaper_status {
+ unsigned int rs_flags;
+ unsigned int rs_children;
+ unsigned int rs_descendants;
+ pid_t rs_reaper;
+ pid_t rs_pid;
+ unsigned int rs_pad0[15];
+};
+
+struct __sanitizer_procctl_reaper_pidinfo {
+ pid_t pi_pid;
+ pid_t pi_subtree;
+ unsigned int pi_flags;
+ unsigned int pi_pad0[15];
+};
+
+struct __sanitizer_procctl_reaper_pids {
+ unsigned int rp_count;
+ unsigned int rp_pad0[15];
+ struct __sanitize_procctl_reapper_pidinfo *rp_pids;
+};
+
+struct __sanitizer_procctl_reaper_kill {
+ int rk_sig;
+ unsigned int rk_flags;
+ pid_t rk_subtree;
+ unsigned int rk_killed;
+ pid_t rk_fpid;
+ unsigned int rk_pad[15];
+};
+
+# define IOC_NRBITS 8
+# define IOC_TYPEBITS 8
+# if defined(__powerpc__) || defined(__powerpc64__) || defined(__mips__)
+# define IOC_SIZEBITS 13
+# define IOC_DIRBITS 3
+# define IOC_NONE 1U
+# define IOC_WRITE 4U
+# define IOC_READ 2U
+# else
+# define IOC_SIZEBITS 14
+# define IOC_DIRBITS 2
+# define IOC_NONE 0U
+# define IOC_WRITE 1U
+# define IOC_READ 2U
+# endif
+# define IOC_NRMASK ((1 << IOC_NRBITS) - 1)
+# define IOC_TYPEMASK ((1 << IOC_TYPEBITS) - 1)
+# define IOC_SIZEMASK ((1 << IOC_SIZEBITS) - 1)
+# if defined(IOC_DIRMASK)
+# undef IOC_DIRMASK
+# endif
+# define IOC_DIRMASK ((1 << IOC_DIRBITS) - 1)
+# define IOC_NRSHIFT 0
+# define IOC_TYPESHIFT (IOC_NRSHIFT + IOC_NRBITS)
+# define IOC_SIZESHIFT (IOC_TYPESHIFT + IOC_TYPEBITS)
+# define IOC_DIRSHIFT (IOC_SIZESHIFT + IOC_SIZEBITS)
+# define EVIOC_EV_MAX 0x1f
+# define EVIOC_ABS_MAX 0x3f
+
+# define IOC_DIR(nr) (((nr) >> IOC_DIRSHIFT) & IOC_DIRMASK)
+# define IOC_TYPE(nr) (((nr) >> IOC_TYPESHIFT) & IOC_TYPEMASK)
+# define IOC_NR(nr) (((nr) >> IOC_NRSHIFT) & IOC_NRMASK)
+# define IOC_SIZE(nr) (((nr) >> IOC_SIZESHIFT) & IOC_SIZEMASK)
extern unsigned struct_ifreq_sz;
extern unsigned struct_termios_sz;
extern unsigned struct_sioc_sg_req_sz;
extern unsigned struct_sioc_vif_req_sz;
+extern unsigned struct_procctl_reaper_status_sz;
+extern unsigned struct_procctl_reaper_pidinfo_sz;
+extern unsigned struct_procctl_reaper_pids_sz;
+extern unsigned struct_procctl_reaper_kill_sz;
+
// ioctl request identifiers
// A special value to mark ioctls that are not present on the target platform,
extern const int si_SEGV_MAPERR;
extern const int si_SEGV_ACCERR;
+extern const unsigned MD5_CTX_sz;
+extern const unsigned MD5_return_length;
+
+#define SHA2_EXTERN(LEN) \
+ extern const unsigned SHA##LEN##_CTX_sz; \
+ extern const unsigned SHA##LEN##_return_length; \
+ extern const unsigned SHA##LEN##_block_length; \
+ extern const unsigned SHA##LEN##_digest_length
+
+SHA2_EXTERN(224);
+SHA2_EXTERN(256);
+SHA2_EXTERN(384);
+SHA2_EXTERN(512);
+
+#undef SHA2_EXTERN
+
struct __sanitizer_cap_rights {
u64 cr_rights[2];
};
extern unsigned struct_StringList_sz;
} // namespace __sanitizer
-#define CHECK_TYPE_SIZE(TYPE) \
- COMPILER_CHECK(sizeof(__sanitizer_##TYPE) == sizeof(TYPE))
+# define CHECK_TYPE_SIZE(TYPE) \
+ COMPILER_CHECK(sizeof(__sanitizer_##TYPE) == sizeof(TYPE))
-#define CHECK_SIZE_AND_OFFSET(CLASS, MEMBER) \
- COMPILER_CHECK(sizeof(((__sanitizer_##CLASS *)NULL)->MEMBER) == \
- sizeof(((CLASS *)NULL)->MEMBER)); \
- COMPILER_CHECK(offsetof(__sanitizer_##CLASS, MEMBER) == \
- offsetof(CLASS, MEMBER))
+# define CHECK_SIZE_AND_OFFSET(CLASS, MEMBER) \
+ COMPILER_CHECK(sizeof(((__sanitizer_##CLASS *)NULL)->MEMBER) == \
+ sizeof(((CLASS *)NULL)->MEMBER)); \
+ COMPILER_CHECK(offsetof(__sanitizer_##CLASS, MEMBER) == \
+ offsetof(CLASS, MEMBER))
// For sigaction, which is a function and struct at the same time,
// and thus requires explicit "struct" in sizeof() expression.
-#define CHECK_STRUCT_SIZE_AND_OFFSET(CLASS, MEMBER) \
- COMPILER_CHECK(sizeof(((struct __sanitizer_##CLASS *)NULL)->MEMBER) == \
- sizeof(((struct CLASS *)NULL)->MEMBER)); \
- COMPILER_CHECK(offsetof(struct __sanitizer_##CLASS, MEMBER) == \
- offsetof(struct CLASS, MEMBER))
+# define CHECK_STRUCT_SIZE_AND_OFFSET(CLASS, MEMBER) \
+ COMPILER_CHECK(sizeof(((struct __sanitizer_##CLASS *)NULL)->MEMBER) == \
+ sizeof(((struct CLASS *)NULL)->MEMBER)); \
+ COMPILER_CHECK(offsetof(struct __sanitizer_##CLASS, MEMBER) == \
+ offsetof(struct CLASS, MEMBER))
-#define SIGACTION_SYMNAME sigaction
+# define SIGACTION_SYMNAME sigaction
#endif
// are not defined anywhere in userspace headers. Fake them. This seems to work
// fine with newer headers, too.
#include <linux/posix_types.h>
-#if defined(__x86_64__) || defined(__mips__)
-#include <sys/stat.h>
-#else
-#define ino_t __kernel_ino_t
-#define mode_t __kernel_mode_t
-#define nlink_t __kernel_nlink_t
-#define uid_t __kernel_uid_t
-#define gid_t __kernel_gid_t
-#define off_t __kernel_off_t
-#define time_t __kernel_time_t
+# if defined(__x86_64__) || defined(__mips__) || defined(__hexagon__)
+# include <sys/stat.h>
+# else
+# define ino_t __kernel_ino_t
+# define mode_t __kernel_mode_t
+# define nlink_t __kernel_nlink_t
+# define uid_t __kernel_uid_t
+# define gid_t __kernel_gid_t
+# define off_t __kernel_off_t
+# define time_t __kernel_time_t
// This header seems to contain the definitions of _kernel_ stat* structs.
-#include <asm/stat.h>
-#undef ino_t
-#undef mode_t
-#undef nlink_t
-#undef uid_t
-#undef gid_t
-#undef off_t
-#endif
-
-#include <linux/aio_abi.h>
-
-#if !SANITIZER_ANDROID
-#include <sys/statfs.h>
-#include <linux/perf_event.h>
-#endif
+# undef ino_t
+# undef mode_t
+# undef nlink_t
+# undef uid_t
+# undef gid_t
+# undef off_t
+# endif
+
+# include <linux/aio_abi.h>
+
+# if !SANITIZER_ANDROID
+# include <sys/statfs.h>
+# include <linux/perf_event.h>
+# endif
using namespace __sanitizer;
-namespace __sanitizer {
-#if !SANITIZER_ANDROID
- unsigned struct_statfs64_sz = sizeof(struct statfs64);
-#endif
-} // namespace __sanitizer
-
-#if !defined(__powerpc64__) && !defined(__x86_64__) && !defined(__aarch64__)\
- && !defined(__mips__) && !defined(__s390__)\
- && !defined(__sparc__) && !defined(__riscv)
+# if !defined(__powerpc64__) && !defined(__x86_64__) && \
+ !defined(__aarch64__) && !defined(__mips__) && !defined(__s390__) && \
+ !defined(__sparc__) && !defined(__riscv) && !defined(__hexagon__) && \
+ !defined(__loongarch__)
COMPILER_CHECK(struct___old_kernel_stat_sz == sizeof(struct __old_kernel_stat));
#endif
unsigned struct_sigevent_sz = sizeof(struct sigevent);
unsigned struct_sched_param_sz = sizeof(struct sched_param);
unsigned struct_sockaddr_sz = sizeof(struct sockaddr);
-unsigned ucontext_t_sz = sizeof(ucontext_t);
+unsigned ucontext_t_sz(void *ctx) { return sizeof(ucontext_t); }
unsigned struct_rlimit_sz = sizeof(struct rlimit);
unsigned struct_timespec_sz = sizeof(struct timespec);
unsigned struct_sembuf_sz = sizeof(struct sembuf);
int glob_nomatch = GLOB_NOMATCH;
int glob_altdirfunc = GLOB_ALTDIRFUNC;
+const int wordexp_wrde_dooffs = WRDE_DOOFFS;
unsigned path_max = PATH_MAX;
extern unsigned struct_sched_param_sz;
extern unsigned struct_statfs_sz;
extern unsigned struct_sockaddr_sz;
-extern unsigned ucontext_t_sz;
+unsigned ucontext_t_sz(void *ctx);
extern unsigned struct_rlimit_sz;
extern unsigned struct_utimbuf_sz;
extern int glob_nomatch;
extern int glob_altdirfunc;
+extern const int wordexp_wrde_dooffs;
extern unsigned path_max;
// Must go after undef _FILE_OFFSET_BITS.
#include "sanitizer_platform.h"
-#if SANITIZER_LINUX || SANITIZER_MAC
+#if SANITIZER_LINUX || SANITIZER_APPLE
// Must go after undef _FILE_OFFSET_BITS.
#include "sanitizer_glibc_version.h"
#include <time.h>
#include <wchar.h>
#include <regex.h>
-#if !SANITIZER_MAC
+#if !SANITIZER_APPLE
#include <utmp.h>
#endif
#include <sys/vt.h>
#include <linux/cdrom.h>
#include <linux/fd.h>
+#if SANITIZER_ANDROID
#include <linux/fs.h>
+#endif
#include <linux/hdreg.h>
#include <linux/input.h>
#include <linux/ioctl.h>
#if SANITIZER_LINUX
# include <utime.h>
# include <sys/ptrace.h>
-#if defined(__mips64) || defined(__aarch64__) || defined(__arm__) || \
- SANITIZER_RISCV64
-# include <asm/ptrace.h>
-# ifdef __arm__
+# if defined(__mips64) || defined(__aarch64__) || defined(__arm__) || \
+ defined(__hexagon__) || defined(__loongarch__) ||SANITIZER_RISCV64
+#
include <asm/ptrace.h>
+# ifdef __arm__
typedef struct user_fpregs elf_fpregset_t;
# define ARM_VFPREGS_SIZE_ASAN (32 * 8 /*fpregs*/ + 4 /*fpscr*/)
# if !defined(ARM_VFPREGS_SIZE)
#include <linux/serial.h>
#include <sys/msg.h>
#include <sys/ipc.h>
-#include <crypt.h>
#endif // SANITIZER_ANDROID
#include <link.h>
#include <fstab.h>
#endif // SANITIZER_LINUX
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
#include <net/ethernet.h>
#include <sys/filio.h>
#include <sys/sockio.h>
#endif
// Include these after system headers to avoid name clashes and ambiguities.
-#include "sanitizer_internal_defs.h"
-#include "sanitizer_platform_limits_posix.h"
+# include "sanitizer_common.h"
+# include "sanitizer_internal_defs.h"
+# include "sanitizer_platform_interceptors.h"
+# include "sanitizer_platform_limits_posix.h"
+
+#if SANITIZER_INTERCEPT_CRYPT_R
+#include <crypt.h>
+#endif
namespace __sanitizer {
unsigned struct_utsname_sz = sizeof(struct utsname);
unsigned struct_stat_sz = sizeof(struct stat);
-#if !SANITIZER_IOS && !(SANITIZER_MAC && TARGET_CPU_ARM64)
+#if SANITIZER_HAS_STAT64
unsigned struct_stat64_sz = sizeof(struct stat64);
-#endif // !SANITIZER_IOS && !(SANITIZER_MAC && TARGET_CPU_ARM64)
+#endif // SANITIZER_HAS_STAT64
unsigned struct_rusage_sz = sizeof(struct rusage);
unsigned struct_tm_sz = sizeof(struct tm);
unsigned struct_passwd_sz = sizeof(struct passwd);
unsigned struct_regex_sz = sizeof(regex_t);
unsigned struct_regmatch_sz = sizeof(regmatch_t);
-#if (SANITIZER_MAC && !TARGET_CPU_ARM64) && !SANITIZER_IOS
+#if SANITIZER_HAS_STATFS64
unsigned struct_statfs64_sz = sizeof(struct statfs64);
-#endif // (SANITIZER_MAC && !TARGET_CPU_ARM64) && !SANITIZER_IOS
+#endif // SANITIZER_HAS_STATFS64
-#if SANITIZER_GLIBC || SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_MAC
+#if SANITIZER_GLIBC || SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_APPLE
unsigned struct_fstab_sz = sizeof(struct fstab);
#endif // SANITIZER_GLIBC || SANITIZER_FREEBSD || SANITIZER_NETBSD ||
- // SANITIZER_MAC
+ // SANITIZER_APPLE
#if !SANITIZER_ANDROID
unsigned struct_statfs_sz = sizeof(struct statfs);
unsigned struct_sockaddr_sz = sizeof(struct sockaddr);
- unsigned ucontext_t_sz = sizeof(ucontext_t);
-#endif // !SANITIZER_ANDROID
-#if SANITIZER_LINUX
+ unsigned ucontext_t_sz(void *ctx) {
+# if SANITIZER_GLIBC && SANITIZER_X64
+ // Added in Linux kernel 3.4.0, merged to glibc in 2.16
+# ifndef FP_XSTATE_MAGIC1
+# define FP_XSTATE_MAGIC1 0x46505853U
+# endif
+ // See kernel arch/x86/kernel/fpu/signal.c for details.
+ const auto *fpregs = static_cast<ucontext_t *>(ctx)->uc_mcontext.fpregs;
+ // The member names differ across header versions, but the actual layout
+ // is always the same. So avoid using members, just use arithmetic.
+ const uint32_t *after_xmm =
+ reinterpret_cast<const uint32_t *>(fpregs + 1) - 24;
+ if (after_xmm[12] == FP_XSTATE_MAGIC1)
+ return reinterpret_cast<const char *>(fpregs) + after_xmm[13] -
+ static_cast<const char *>(ctx);
+# endif
+ return sizeof(ucontext_t);
+ }
+# endif // !SANITIZER_ANDROID
+
+# if SANITIZER_LINUX
unsigned struct_epoll_event_sz = sizeof(struct epoll_event);
unsigned struct_sysinfo_sz = sizeof(struct sysinfo);
unsigned __user_cap_header_struct_sz =
unsigned struct_itimerspec_sz = sizeof(struct itimerspec);
#endif // SANITIZER_LINUX
-#if SANITIZER_LINUX && !SANITIZER_ANDROID
+#if SANITIZER_GLIBC
// Use pre-computed size of struct ustat to avoid <sys/ustat.h> which
// has been removed from glibc 2.28.
#if defined(__aarch64__) || defined(__s390x__) || defined(__mips64) || \
defined(__powerpc64__) || defined(__arch64__) || defined(__sparcv9) || \
defined(__x86_64__) || SANITIZER_RISCV64
#define SIZEOF_STRUCT_USTAT 32
-#elif defined(__arm__) || defined(__i386__) || defined(__mips__) \
- || defined(__powerpc__) || defined(__s390__) || defined(__sparc__)
-#define SIZEOF_STRUCT_USTAT 20
-#else
-#error Unknown size of struct ustat
-#endif
+# elif defined(__arm__) || defined(__i386__) || defined(__mips__) || \
+ defined(__powerpc__) || defined(__s390__) || defined(__sparc__) || \
+ defined(__hexagon__)
+# define SIZEOF_STRUCT_USTAT 20
+# elif defined(__loongarch__)
+ // Not used. The minimum Glibc version available for LoongArch is 2.36
+ // so ustat() wrapper is already gone.
+# define SIZEOF_STRUCT_USTAT 0
+# else
+# error Unknown size of struct ustat
+# endif
unsigned struct_ustat_sz = SIZEOF_STRUCT_USTAT;
unsigned struct_rlimit64_sz = sizeof(struct rlimit64);
unsigned struct_statvfs64_sz = sizeof(struct statvfs64);
+#endif // SANITIZER_GLIBC
+
+#if SANITIZER_INTERCEPT_CRYPT_R
unsigned struct_crypt_data_sz = sizeof(struct crypt_data);
-#endif // SANITIZER_LINUX && !SANITIZER_ANDROID
+#endif
#if SANITIZER_LINUX && !SANITIZER_ANDROID
unsigned struct_timex_sz = sizeof(struct timex);
int shmctl_shm_stat = (int)SHM_STAT;
#endif
-#if !SANITIZER_MAC && !SANITIZER_FREEBSD
+#if !SANITIZER_APPLE && !SANITIZER_FREEBSD
unsigned struct_utmp_sz = sizeof(struct utmp);
#endif
#if !SANITIZER_ANDROID
int glob_altdirfunc = GLOB_ALTDIRFUNC;
#endif
+# if !SANITIZER_ANDROID
+ const int wordexp_wrde_dooffs = WRDE_DOOFFS;
+# endif // !SANITIZER_ANDROID
+
#if SANITIZER_LINUX && !SANITIZER_ANDROID && \
(defined(__i386) || defined(__x86_64) || defined(__mips64) || \
defined(__powerpc64__) || defined(__aarch64__) || defined(__arm__) || \
- defined(__s390__) || SANITIZER_RISCV64)
+ defined(__s390__) || defined(__loongarch__)|| SANITIZER_RISCV64)
#if defined(__mips64) || defined(__powerpc64__) || defined(__arm__)
unsigned struct_user_regs_struct_sz = sizeof(struct pt_regs);
unsigned struct_user_fpregs_struct_sz = sizeof(elf_fpregset_t);
#elif defined(__aarch64__)
unsigned struct_user_regs_struct_sz = sizeof(struct user_pt_regs);
unsigned struct_user_fpregs_struct_sz = sizeof(struct user_fpsimd_state);
+#elif defined(__loongarch__)
+ unsigned struct_user_regs_struct_sz = sizeof(struct user_pt_regs);
+ unsigned struct_user_fpregs_struct_sz = sizeof(struct user_fp_state);
#elif defined(__s390__)
unsigned struct_user_regs_struct_sz = sizeof(struct _user_regs_struct);
unsigned struct_user_fpregs_struct_sz = sizeof(struct _user_fpregs_struct);
#else
unsigned struct_user_regs_struct_sz = sizeof(struct user_regs_struct);
unsigned struct_user_fpregs_struct_sz = sizeof(struct user_fpregs_struct);
-#endif // __mips64 || __powerpc64__ || __aarch64__
+#endif // __mips64 || __powerpc64__ || __aarch64__ || __loongarch__
#if defined(__x86_64) || defined(__mips64) || defined(__powerpc64__) || \
defined(__aarch64__) || defined(__arm__) || defined(__s390__) || \
- SANITIZER_RISCV64
+ defined(__loongarch__) || SANITIZER_RISCV64
unsigned struct_user_fpxregs_struct_sz = 0;
#else
unsigned struct_user_fpxregs_struct_sz = sizeof(struct user_fpxregs_struct);
#endif // __x86_64 || __mips64 || __powerpc64__ || __aarch64__ || __arm__
-// || __s390__
+// || __s390__ || __loongarch__
#ifdef __arm__
unsigned struct_user_vfpregs_struct_sz = ARM_VFPREGS_SIZE;
#else
unsigned struct_ppp_stats_sz = sizeof(struct ppp_stats);
#endif // SANITIZER_GLIBC
-#if !SANITIZER_ANDROID && !SANITIZER_MAC
+#if !SANITIZER_ANDROID && !SANITIZER_APPLE
unsigned struct_sioc_sg_req_sz = sizeof(struct sioc_sg_req);
unsigned struct_sioc_vif_req_sz = sizeof(struct sioc_vif_req);
#endif
unsigned IOCTL_BLKROGET = BLKROGET;
unsigned IOCTL_BLKROSET = BLKROSET;
unsigned IOCTL_BLKRRPART = BLKRRPART;
+ unsigned IOCTL_BLKFRASET = BLKFRASET;
+ unsigned IOCTL_BLKFRAGET = BLKFRAGET;
+ unsigned IOCTL_BLKSECTSET = BLKSECTSET;
+ unsigned IOCTL_BLKSECTGET = BLKSECTGET;
+ unsigned IOCTL_BLKSSZGET = BLKSSZGET;
+ unsigned IOCTL_BLKBSZGET = BLKBSZGET;
+ unsigned IOCTL_BLKBSZSET = BLKBSZSET;
+ unsigned IOCTL_BLKGETSIZE64 = BLKGETSIZE64;
unsigned IOCTL_CDROMAUDIOBUFSIZ = CDROMAUDIOBUFSIZ;
unsigned IOCTL_CDROMEJECT = CDROMEJECT;
unsigned IOCTL_CDROMEJECT_SW = CDROMEJECT_SW;
unsigned IOCTL_EVIOCGPROP = IOCTL_NOT_PRESENT;
unsigned IOCTL_EVIOCSKEYCODE_V2 = IOCTL_NOT_PRESENT;
#endif
- unsigned IOCTL_FS_IOC_GETFLAGS = FS_IOC_GETFLAGS;
- unsigned IOCTL_FS_IOC_GETVERSION = FS_IOC_GETVERSION;
- unsigned IOCTL_FS_IOC_SETFLAGS = FS_IOC_SETFLAGS;
- unsigned IOCTL_FS_IOC_SETVERSION = FS_IOC_SETVERSION;
+ unsigned IOCTL_FS_IOC_GETFLAGS = _IOR('f', 1, long);
+ unsigned IOCTL_FS_IOC_GETVERSION = _IOR('v', 1, long);
+ unsigned IOCTL_FS_IOC_SETFLAGS = _IOW('f', 2, long);
+ unsigned IOCTL_FS_IOC_SETVERSION = _IOW('v', 2, long);
unsigned IOCTL_GIO_CMAP = GIO_CMAP;
unsigned IOCTL_GIO_FONT = GIO_FONT;
unsigned IOCTL_GIO_UNIMAP = GIO_UNIMAP;
COMPILER_CHECK(sizeof(__sanitizer_dirent) <= sizeof(dirent));
CHECK_SIZE_AND_OFFSET(dirent, d_ino);
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
CHECK_SIZE_AND_OFFSET(dirent, d_seekoff);
#elif SANITIZER_FREEBSD
// There is no 'd_off' field on FreeBSD.
#endif
CHECK_SIZE_AND_OFFSET(dirent, d_reclen);
-#if SANITIZER_LINUX && !SANITIZER_ANDROID
+#if SANITIZER_GLIBC
COMPILER_CHECK(sizeof(__sanitizer_dirent64) <= sizeof(dirent64));
CHECK_SIZE_AND_OFFSET(dirent64, d_ino);
CHECK_SIZE_AND_OFFSET(dirent64, d_off);
CHECK_STRUCT_SIZE_AND_OFFSET(sigaction, sa_restorer);
#endif
+#if SANITIZER_HAS_SIGINFO
+COMPILER_CHECK(alignof(siginfo_t) == alignof(__sanitizer_siginfo));
+using __sanitizer_siginfo_t = __sanitizer_siginfo;
+CHECK_TYPE_SIZE(siginfo_t);
+CHECK_SIZE_AND_OFFSET(siginfo_t, si_signo);
+CHECK_SIZE_AND_OFFSET(siginfo_t, si_errno);
+CHECK_SIZE_AND_OFFSET(siginfo_t, si_code);
+#endif
+
#if SANITIZER_LINUX
CHECK_TYPE_SIZE(__sysctl_args);
CHECK_SIZE_AND_OFFSET(__sysctl_args, name);
CHECK_SIZE_AND_OFFSET(passwd, pw_gecos);
#endif
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
CHECK_SIZE_AND_OFFSET(passwd, pw_change);
CHECK_SIZE_AND_OFFSET(passwd, pw_expire);
CHECK_SIZE_AND_OFFSET(passwd, pw_class);
CHECK_SIZE_AND_OFFSET(group, gr_gid);
CHECK_SIZE_AND_OFFSET(group, gr_mem);
-#if HAVE_RPC_XDR_H
+#if HAVE_RPC_XDR_H && !SANITIZER_APPLE
CHECK_TYPE_SIZE(XDR);
CHECK_SIZE_AND_OFFSET(XDR, x_op);
CHECK_SIZE_AND_OFFSET(XDR, x_ops);
COMPILER_CHECK(ARM_VFPREGS_SIZE == ARM_VFPREGS_SIZE_ASAN);
#endif
-#endif // SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_MAC
+#endif // SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_APPLE
#ifndef SANITIZER_PLATFORM_LIMITS_POSIX_H
#define SANITIZER_PLATFORM_LIMITS_POSIX_H
-#if SANITIZER_LINUX || SANITIZER_MAC
+#if SANITIZER_LINUX || SANITIZER_APPLE
#include "sanitizer_internal_defs.h"
#include "sanitizer_platform.h"
+#if SANITIZER_APPLE
+#include <sys/cdefs.h>
+#if !__DARWIN_ONLY_64_BIT_INO_T
+#define SANITIZER_HAS_STAT64 1
+#define SANITIZER_HAS_STATFS64 1
+#else
+#define SANITIZER_HAS_STAT64 0
+#define SANITIZER_HAS_STATFS64 0
+#endif
+#elif SANITIZER_GLIBC || SANITIZER_ANDROID
+#define SANITIZER_HAS_STAT64 1
+#define SANITIZER_HAS_STATFS64 1
+#endif
+
#if defined(__sparc__)
// FIXME: This can't be included from tsan which does not support sparc yet.
#include "sanitizer_glibc_version.h"
namespace __sanitizer {
extern unsigned struct_utsname_sz;
extern unsigned struct_stat_sz;
-#if !SANITIZER_IOS
+#if SANITIZER_HAS_STAT64
extern unsigned struct_stat64_sz;
#endif
extern unsigned struct_rusage_sz;
extern unsigned struct_sigevent_sz;
extern unsigned struct_stack_t_sz;
extern unsigned struct_sched_param_sz;
+#if SANITIZER_HAS_STATFS64
extern unsigned struct_statfs64_sz;
+#endif
extern unsigned struct_regex_sz;
extern unsigned struct_regmatch_sz;
extern unsigned struct_fstab_sz;
extern unsigned struct_statfs_sz;
extern unsigned struct_sockaddr_sz;
-extern unsigned ucontext_t_sz;
-#endif // !SANITIZER_ANDROID
+unsigned ucontext_t_sz(void *uctx);
+# endif // !SANITIZER_ANDROID
-#if SANITIZER_LINUX
+# if SANITIZER_LINUX
-#if defined(__x86_64__)
+# if defined(__x86_64__)
const unsigned struct_kernel_stat_sz = 144;
const unsigned struct_kernel_stat64_sz = 0;
#elif defined(__i386__)
const unsigned struct_kernel_stat_sz = 144;
const unsigned struct_kernel_stat64_sz = 104;
#elif defined(__mips__)
-const unsigned struct_kernel_stat_sz = SANITIZER_ANDROID
- ? FIRST_32_SECOND_64(104, 128)
- : FIRST_32_SECOND_64(160, 216);
+const unsigned struct_kernel_stat_sz =
+ SANITIZER_ANDROID
+ ? FIRST_32_SECOND_64(104, 128)
+ : FIRST_32_SECOND_64((_MIPS_SIM == _ABIN32) ? 176 : 160, 216);
const unsigned struct_kernel_stat64_sz = 104;
#elif defined(__s390__) && !defined(__s390x__)
const unsigned struct_kernel_stat_sz = 64;
#elif SANITIZER_RISCV64
const unsigned struct_kernel_stat_sz = 128;
const unsigned struct_kernel_stat64_sz = 0; // RISCV64 does not use stat64
-#endif
+# elif defined(__hexagon__)
+const unsigned struct_kernel_stat_sz = 128;
+const unsigned struct_kernel_stat64_sz = 0;
+# elif defined(__loongarch__)
+const unsigned struct_kernel_stat_sz = 128;
+const unsigned struct_kernel_stat64_sz = 0;
+# endif
struct __sanitizer_perf_event_attr {
unsigned type;
unsigned size;
#if SANITIZER_LINUX
-#if defined(__powerpc64__) || defined(__s390__)
+#if defined(__powerpc64__) || defined(__s390__) || defined(__loongarch__)
const unsigned struct___old_kernel_stat_sz = 0;
#elif !defined(__sparc__)
const unsigned struct___old_kernel_stat_sz = 32;
};
#endif // !SANITIZER_ANDROID
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
typedef unsigned long __sanitizer_pthread_key_t;
#else
typedef unsigned __sanitizer_pthread_key_t;
char *pw_passwd;
int pw_uid;
int pw_gid;
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
long pw_change;
char *pw_class;
#endif
#endif
char *pw_dir;
char *pw_shell;
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
long pw_expire;
#endif
};
char **gr_mem;
};
-#if defined(__x86_64__) && !defined(_LP64)
+# if (SANITIZER_LINUX && !SANITIZER_GLIBC && !SANITIZER_ANDROID) || \
+ (defined(__x86_64__) && !defined(_LP64)) || defined(__hexagon__)
typedef long long __sanitizer_time_t;
#else
typedef long __sanitizer_time_t;
};
#endif
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
struct __sanitizer_msghdr {
void *msg_name;
unsigned msg_namelen;
};
#endif
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
struct __sanitizer_dirent {
unsigned long long d_ino;
unsigned long long d_seekoff;
unsigned short d_reclen;
// more fields that we don't care about
};
-#elif SANITIZER_ANDROID || defined(__x86_64__)
+# elif (SANITIZER_LINUX && !SANITIZER_GLIBC) || defined(__x86_64__) || \
+ defined(__hexagon__)
struct __sanitizer_dirent {
unsigned long long d_ino;
unsigned long long d_off;
unsigned short d_reclen;
// more fields that we don't care about
};
-#else
+# else
struct __sanitizer_dirent {
uptr d_ino;
uptr d_off;
unsigned short d_reclen;
// more fields that we don't care about
};
-#endif
+# endif
-#if SANITIZER_LINUX && !SANITIZER_ANDROID
+# if SANITIZER_GLIBC
struct __sanitizer_dirent64 {
unsigned long long d_ino;
unsigned long long d_off;
#endif
#if SANITIZER_LINUX
-#if defined(_LP64) || defined(__x86_64__) || defined(__powerpc__) || \
- defined(__mips__)
+# if defined(_LP64) || defined(__x86_64__) || defined(__powerpc__) || \
+ defined(__mips__) || defined(__hexagon__)
typedef unsigned __sanitizer___kernel_uid_t;
typedef unsigned __sanitizer___kernel_gid_t;
#else
# else
typedef unsigned long __sanitizer_sigset_t;
# endif
-#elif SANITIZER_MAC
+#elif SANITIZER_APPLE
typedef unsigned __sanitizer_sigset_t;
#elif SANITIZER_LINUX
struct __sanitizer_sigset_t {
};
#endif
-struct __sanitizer_siginfo {
- // The size is determined by looking at sizeof of real siginfo_t on linux.
- u64 opaque[128 / sizeof(u64)];
+struct __sanitizer_siginfo_pad {
+ // Require uptr, because siginfo_t is always pointer-size aligned on Linux.
+ uptr pad[128 / sizeof(uptr)];
+};
+
+#if SANITIZER_LINUX
+# define SANITIZER_HAS_SIGINFO 1
+union __sanitizer_siginfo {
+ struct {
+ int si_signo;
+# if SANITIZER_MIPS
+ int si_code;
+ int si_errno;
+# else
+ int si_errno;
+ int si_code;
+# endif
+ };
+ __sanitizer_siginfo_pad pad;
};
+#else
+# define SANITIZER_HAS_SIGINFO 0
+typedef __sanitizer_siginfo_pad __sanitizer_siginfo;
+#endif
using __sanitizer_sighandler_ptr = void (*)(int sig);
using __sanitizer_sigactionhandler_ptr = void (*)(int sig,
int p_proto;
};
+struct __sanitizer_netent {
+ char *n_name;
+ char **n_aliases;
+ int n_addrtype;
+ u32 n_net;
+};
+
struct __sanitizer_addrinfo {
int ai_flags;
int ai_family;
int ai_socktype;
int ai_protocol;
-#if SANITIZER_ANDROID || SANITIZER_MAC
+#if SANITIZER_ANDROID || SANITIZER_APPLE
unsigned ai_addrlen;
char *ai_canonname;
void *ai_addr;
short revents;
};
-#if SANITIZER_ANDROID || SANITIZER_MAC
+#if SANITIZER_ANDROID || SANITIZER_APPLE
typedef unsigned __sanitizer_nfds_t;
#else
typedef unsigned long __sanitizer_nfds_t;
extern unsigned path_max;
+# if !SANITIZER_ANDROID
+extern const int wordexp_wrde_dooffs;
+# endif // !SANITIZER_ANDROID
+
struct __sanitizer_wordexp_t {
uptr we_wordc;
char **we_wordv;
#if SANITIZER_LINUX && !SANITIZER_ANDROID && \
(defined(__i386) || defined(__x86_64) || defined(__mips64) || \
defined(__powerpc64__) || defined(__aarch64__) || defined(__arm__) || \
- defined(__s390__) || SANITIZER_RISCV64)
+ defined(__s390__) || defined(__loongarch__) || SANITIZER_RISCV64)
extern unsigned struct_user_regs_struct_sz;
extern unsigned struct_user_fpregs_struct_sz;
extern unsigned struct_user_fpxregs_struct_sz;
extern int shmctl_shm_stat;
#endif
-#if !SANITIZER_MAC && !SANITIZER_FREEBSD
+#if !SANITIZER_APPLE && !SANITIZER_FREEBSD
extern unsigned struct_utmp_sz;
#endif
#if !SANITIZER_ANDROID
union {
void *ifcu_req;
} ifc_ifcu;
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
} __attribute__((packed));
#else
};
extern unsigned struct_ppp_stats_sz;
#endif // (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID
-#if !SANITIZER_ANDROID && !SANITIZER_MAC
+#if !SANITIZER_ANDROID && !SANITIZER_APPLE
extern unsigned struct_sioc_sg_req_sz;
extern unsigned struct_sioc_vif_req_sz;
#endif
extern unsigned IOCTL_BLKROGET;
extern unsigned IOCTL_BLKROSET;
extern unsigned IOCTL_BLKRRPART;
+extern unsigned IOCTL_BLKFRASET;
+extern unsigned IOCTL_BLKFRAGET;
+extern unsigned IOCTL_BLKSECTSET;
+extern unsigned IOCTL_BLKSECTGET;
+extern unsigned IOCTL_BLKSSZGET;
+extern unsigned IOCTL_BLKBSZGET;
+extern unsigned IOCTL_BLKBSZSET;
+extern unsigned IOCTL_BLKGETSIZE64;
extern unsigned IOCTL_CDROMAUDIOBUFSIZ;
extern unsigned IOCTL_CDROMEJECT;
extern unsigned IOCTL_CDROMEJECT_SW;
#define SIGACTION_SYMNAME sigaction
-#endif // SANITIZER_LINUX || SANITIZER_MAC
+#endif // SANITIZER_LINUX || SANITIZER_APPLE
#endif
unsigned struct_sched_param_sz = sizeof(struct sched_param);
unsigned struct_statfs_sz = sizeof(struct statfs);
unsigned struct_sockaddr_sz = sizeof(struct sockaddr);
- unsigned ucontext_t_sz = sizeof(ucontext_t);
+ unsigned ucontext_t_sz(void *ctx) { return sizeof(ucontext_t); }
unsigned struct_timespec_sz = sizeof(struct timespec);
#if SANITIZER_SOLARIS32
unsigned struct_statvfs64_sz = sizeof(struct statvfs64);
unsigned struct_ElfW_Phdr_sz = sizeof(ElfW(Phdr));
int glob_nomatch = GLOB_NOMATCH;
+ const int wordexp_wrde_dooffs = WRDE_DOOFFS;
unsigned path_max = PATH_MAX;
extern unsigned struct_statfs64_sz;
extern unsigned struct_statfs_sz;
extern unsigned struct_sockaddr_sz;
-extern unsigned ucontext_t_sz;
+unsigned ucontext_t_sz(void *ctx);
extern unsigned struct_timespec_sz;
extern unsigned struct_rlimit_sz;
extern int glob_nomatch;
extern int glob_altdirfunc;
+extern const int wordexp_wrde_dooffs;
extern unsigned path_max;
return GetPageSize();
}
+bool ErrorIsOOM(error_t err) { return err == ENOMEM; }
+
void *MmapOrDie(uptr size, const char *mem_type, bool raw_report) {
size = RoundUpTo(size, GetPageSizeCached());
uptr res = MmapNamed(nullptr, size, PROT_READ | PROT_WRITE,
CHECK(IsPowerOfTwo(size));
CHECK(IsPowerOfTwo(alignment));
uptr map_size = size + alignment;
+ // mmap maps entire pages and rounds up map_size needs to be a an integral
+ // number of pages.
+ // We need to be aware of this size for calculating end and for unmapping
+ // fragments before and after the alignment region.
+ map_size = RoundUpTo(map_size, GetPageSizeCached());
uptr map_res = (uptr)MmapOrDieOnFatalError(map_size, mem_type);
if (UNLIKELY(!map_res))
return nullptr;
- uptr map_end = map_res + map_size;
uptr res = map_res;
if (!IsAligned(res, alignment)) {
res = (map_res + alignment - 1) & ~(alignment - 1);
UnmapOrDie((void*)map_res, res - map_res);
}
+ uptr map_end = map_res + map_size;
uptr end = res + size;
- if (end != map_end)
+ end = RoundUpTo(end, GetPageSizeCached());
+ if (end != map_end) {
+ CHECK_LT(end, map_end);
UnmapOrDie((void*)end, map_end - end);
+ }
return (void*)res;
}
return 0 == internal_mprotect((void *)addr, size, PROT_READ);
}
-#if !SANITIZER_MAC
+#if !SANITIZER_APPLE
void MprotectMallocZones(void *addr, int prot) {}
#endif
return true;
}
-#if !SANITIZER_MAC
+#if !SANITIZER_APPLE
void DumpProcessMap() {
MemoryMappingLayout proc_maps(/*cache_enabled*/true);
const sptr kBufSize = 4095;
#include "sanitizer_platform_limits_posix.h"
#include "sanitizer_platform_limits_solaris.h"
-#if !SANITIZER_POSIX
-// Make it hard to accidentally use any of functions declared in this file:
-#error This file should only be included on POSIX
-#endif
+#if SANITIZER_POSIX
namespace __sanitizer {
} // namespace __sanitizer
+#endif // SANITIZER_POSIX
+
#endif // SANITIZER_POSIX_H
#endif
}
+bool CreateDir(const char *pathname) { return mkdir(pathname, 0755) == 0; }
+
bool SupportsColoredOutput(fd_t fd) {
return isatty(fd) != 0;
}
return result;
}
-void PlatformPrepareForSandboxing(__sanitizer_sandbox_arguments *args) {
+void PlatformPrepareForSandboxing(void *args) {
// Some kinds of sandboxes may forbid filesystem access, so we won't be able
// to read the file mappings from /proc/self/maps. Luckily, neither the
// process will be able to load additional libraries, so it's fine to use the
} // extern "C"
int my_pthread_attr_getstack(void *attr, void **addr, uptr *size) {
-#if !SANITIZER_GO && !SANITIZER_MAC
+#if !SANITIZER_GO && !SANITIZER_APPLE
if (&real_pthread_attr_getstack)
return real_pthread_attr_getstack((pthread_attr_t *)attr, addr,
(size_t *)size);
#include <stdio.h>
#include <stdarg.h>
-#if defined(__x86_64__)
-# include <emmintrin.h>
-#endif
-
#if SANITIZER_WINDOWS && defined(_MSC_VER) && _MSC_VER < 1800 && \
!defined(va_copy)
# define va_copy(dst, src) ((dst) = (src))
int VSNPrintf(char *buff, int buff_length,
const char *format, va_list args) {
static const char *kPrintfFormatsHelp =
- "Supported Printf formats: %([0-9]*)?(z|ll)?{d,u,x,X,V}; %p; "
- "%[-]([0-9]*)?(\\.\\*)?s; %c\n";
+ "Supported Printf formats: %([0-9]*)?(z|l|ll)?{d,u,x,X}; %p; "
+ "%[-]([0-9]*)?(\\.\\*)?s; %c\nProvided format: ";
RAW_CHECK(format);
RAW_CHECK(buff_length > 0);
const char *buff_end = &buff[buff_length - 1];
}
bool have_z = (*cur == 'z');
cur += have_z;
- bool have_ll = !have_z && (cur[0] == 'l' && cur[1] == 'l');
+ bool have_l = cur[0] == 'l' && cur[1] != 'l';
+ cur += have_l;
+ bool have_ll = cur[0] == 'l' && cur[1] == 'l';
cur += have_ll * 2;
- const bool have_length = have_z || have_ll;
+ const bool have_length = have_z || have_l || have_ll;
const bool have_flags = have_width || have_length;
// At the moment only %s supports precision and left-justification.
CHECK(!((precision >= 0 || left_justified) && *cur != 's'));
case 'd': {
s64 dval = have_ll ? va_arg(args, s64)
: have_z ? va_arg(args, sptr)
+ : have_l ? va_arg(args, long)
: va_arg(args, int);
result += AppendSignedDecimal(&buff, buff_end, dval, width,
pad_with_zero);
case 'X': {
u64 uval = have_ll ? va_arg(args, u64)
: have_z ? va_arg(args, uptr)
+ : have_l ? va_arg(args, unsigned long)
: va_arg(args, unsigned);
bool uppercase = (*cur == 'X');
result += AppendUnsigned(&buff, buff_end, uval, (*cur == 'u') ? 10 : 16,
width, pad_with_zero, uppercase);
break;
}
- case 'V': {
- for (uptr i = 0; i < 16; i++) {
- unsigned x = va_arg(args, unsigned);
- result += AppendUnsigned(&buff, buff_end, x, 16, 2, true, false);
- }
- break;
- }
case 'p': {
- RAW_CHECK_MSG(!have_flags, kPrintfFormatsHelp);
+ RAW_CHECK_VA(!have_flags, kPrintfFormatsHelp, format);
result += AppendPointer(&buff, buff_end, va_arg(args, uptr));
break;
}
case 's': {
- RAW_CHECK_MSG(!have_length, kPrintfFormatsHelp);
+ RAW_CHECK_VA(!have_length, kPrintfFormatsHelp, format);
// Only left-justified width is supported.
CHECK(!have_width || left_justified);
result += AppendString(&buff, buff_end, left_justified ? -width : width,
break;
}
case 'c': {
- RAW_CHECK_MSG(!have_flags, kPrintfFormatsHelp);
+ RAW_CHECK_VA(!have_flags, kPrintfFormatsHelp, format);
result += AppendChar(&buff, buff_end, va_arg(args, int));
break;
}
case '%' : {
- RAW_CHECK_MSG(!have_flags, kPrintfFormatsHelp);
+ RAW_CHECK_VA(!have_flags, kPrintfFormatsHelp, format);
result += AppendChar(&buff, buff_end, '%');
break;
}
default: {
- RAW_CHECK_MSG(false, kPrintfFormatsHelp);
+ RAW_CHECK_VA(false, kPrintfFormatsHelp, format);
}
}
}
format, args);
}
-FORMAT(1, 2)
void Printf(const char *format, ...) {
va_list args;
va_start(args, format);
}
// Like Printf, but prints the current PID before the output string.
-FORMAT(1, 2)
void Report(const char *format, ...) {
va_list args;
va_start(args, format);
// Returns the number of symbols that should have been written to buffer
// (not including trailing '\0'). Thus, the string is truncated
// iff return value is not less than "length".
-FORMAT(3, 4)
int internal_snprintf(char *buffer, uptr length, const char *format, ...) {
va_list args;
va_start(args, format);
return needed_length;
}
-FORMAT(2, 3)
void InternalScopedString::append(const char *format, ...) {
uptr prev_len = length();
#include "sanitizer_platform.h"
#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD || \
- SANITIZER_MAC || SANITIZER_SOLARIS || \
+ SANITIZER_APPLE || SANITIZER_SOLARIS || \
SANITIZER_FUCHSIA
#include "sanitizer_common.h"
MemoryMappedSegmentData *data_;
};
-class MemoryMappingLayout {
+class MemoryMappingLayoutBase {
+ public:
+ virtual bool Next(MemoryMappedSegment *segment) { UNIMPLEMENTED(); }
+ virtual bool Error() const { UNIMPLEMENTED(); };
+ virtual void Reset() { UNIMPLEMENTED(); }
+
+ protected:
+ ~MemoryMappingLayoutBase() {}
+};
+
+class MemoryMappingLayout final : public MemoryMappingLayoutBase {
public:
explicit MemoryMappingLayout(bool cache_enabled);
~MemoryMappingLayout();
- bool Next(MemoryMappedSegment *segment);
- bool Error() const;
- void Reset();
+ virtual bool Next(MemoryMappedSegment *segment) override;
+ virtual bool Error() const override;
+ virtual void Reset() override;
// In some cases, e.g. when running under a sandbox on Linux, ASan is unable
// to obtain the memory mappings. It should fall back to pre-cached data
// instead of aborting.
namespace __sanitizer {
+#if SANITIZER_FREEBSD
+void GetMemoryProfile(fill_profile_f cb, uptr *stats) {
+ const int Mib[] = {
+ CTL_KERN,
+ KERN_PROC,
+ KERN_PROC_PID,
+ getpid()
+ };
+
+ struct kinfo_proc InfoProc;
+ uptr Len = sizeof(InfoProc);
+ CHECK_EQ(internal_sysctl(Mib, ARRAY_SIZE(Mib), nullptr, (uptr *)&InfoProc, &Len, 0), 0);
+ cb(0, InfoProc.ki_rssize * GetPageSizeCached(), false, stats);
+}
+#endif
+
void ReadProcMaps(ProcSelfMapsBuff *proc_maps) {
const int Mib[] = {
#if SANITIZER_FREEBSD
}
}
-void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size) {
+#if SANITIZER_LINUX || SANITIZER_ANDROID || SANITIZER_SOLARIS || SANITIZER_NETBSD
+void GetMemoryProfile(fill_profile_f cb, uptr *stats) {
char *smaps = nullptr;
uptr smaps_cap = 0;
uptr smaps_len = 0;
if (!ReadFileToBuffer("/proc/self/smaps", &smaps, &smaps_cap, &smaps_len))
return;
+ ParseUnixMemoryProfile(cb, stats, smaps, smaps_len);
+ UnmapOrDie(smaps, smaps_cap);
+}
+
+void ParseUnixMemoryProfile(fill_profile_f cb, uptr *stats, char *smaps,
+ uptr smaps_len) {
uptr start = 0;
bool file = false;
const char *pos = smaps;
- while (pos < smaps + smaps_len) {
+ char *end = smaps + smaps_len;
+ if (smaps_len < 2)
+ return;
+ // The following parsing can crash on almost every line
+ // in the case of malformed/truncated input.
+ // Fixing that is hard b/c e.g. ParseDecimal does not
+ // even accept end of the buffer and assumes well-formed input.
+ // So instead we patch end of the input a bit,
+ // it does not affect well-formed complete inputs.
+ *--end = 0;
+ *--end = '\n';
+ while (pos < end) {
if (IsHex(pos[0])) {
start = ParseHex(&pos);
for (; *pos != '/' && *pos > '\n'; pos++) {}
file = *pos == '/';
} else if (internal_strncmp(pos, "Rss:", 4) == 0) {
- while (!IsDecimal(*pos)) pos++;
+ while (pos < end && !IsDecimal(*pos)) pos++;
uptr rss = ParseDecimal(&pos) * 1024;
- cb(start, rss, file, stats, stats_size);
+ cb(start, rss, file, stats);
}
while (*pos++ != '\n') {}
}
- UnmapOrDie(smaps, smaps_cap);
}
+#endif
} // namespace __sanitizer
//===----------------------------------------------------------------------===//
#include "sanitizer_platform.h"
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
#include "sanitizer_common.h"
#include "sanitizer_placement_new.h"
#include "sanitizer_procmaps.h"
// No-op on Mac for now.
}
+static bool IsDyldHdr(const mach_header *hdr) {
+ return (hdr->magic == MH_MAGIC || hdr->magic == MH_MAGIC_64) &&
+ hdr->filetype == MH_DYLINKER;
+}
+
// _dyld_get_image_header() and related APIs don't report dyld itself.
// We work around this by manually recursing through the memory map
// until we hit a Mach header matching dyld instead. These recurse
// calls are expensive, but the first memory map generation occurs
// early in the process, when dyld is one of the only images loaded,
-// so it will be hit after only a few iterations.
-static mach_header *get_dyld_image_header() {
- unsigned depth = 1;
- vm_size_t size = 0;
+// so it will be hit after only a few iterations. These assumptions don't hold
+// on macOS 13+ anymore (dyld itself has moved into the shared cache).
+static mach_header *GetDyldImageHeaderViaVMRegion() {
vm_address_t address = 0;
- kern_return_t err = KERN_SUCCESS;
- mach_msg_type_number_t count = VM_REGION_SUBMAP_INFO_COUNT_64;
while (true) {
+ vm_size_t size = 0;
+ unsigned depth = 1;
struct vm_region_submap_info_64 info;
- err = vm_region_recurse_64(mach_task_self(), &address, &size, &depth,
- (vm_region_info_t)&info, &count);
+ mach_msg_type_number_t count = VM_REGION_SUBMAP_INFO_COUNT_64;
+ kern_return_t err =
+ vm_region_recurse_64(mach_task_self(), &address, &size, &depth,
+ (vm_region_info_t)&info, &count);
if (err != KERN_SUCCESS) return nullptr;
if (size >= sizeof(mach_header) && info.protection & kProtectionRead) {
mach_header *hdr = (mach_header *)address;
- if ((hdr->magic == MH_MAGIC || hdr->magic == MH_MAGIC_64) &&
- hdr->filetype == MH_DYLINKER) {
+ if (IsDyldHdr(hdr)) {
return hdr;
}
}
}
}
+extern "C" {
+struct dyld_shared_cache_dylib_text_info {
+ uint64_t version; // current version 2
+ // following fields all exist in version 1
+ uint64_t loadAddressUnslid;
+ uint64_t textSegmentSize;
+ uuid_t dylibUuid;
+ const char *path; // pointer invalid at end of iterations
+ // following fields all exist in version 2
+ uint64_t textSegmentOffset; // offset from start of cache
+};
+typedef struct dyld_shared_cache_dylib_text_info
+ dyld_shared_cache_dylib_text_info;
+
+extern bool _dyld_get_shared_cache_uuid(uuid_t uuid);
+extern const void *_dyld_get_shared_cache_range(size_t *length);
+extern int dyld_shared_cache_iterate_text(
+ const uuid_t cacheUuid,
+ void (^callback)(const dyld_shared_cache_dylib_text_info *info));
+} // extern "C"
+
+static mach_header *GetDyldImageHeaderViaSharedCache() {
+ uuid_t uuid;
+ bool hasCache = _dyld_get_shared_cache_uuid(uuid);
+ if (!hasCache)
+ return nullptr;
+
+ size_t cacheLength;
+ __block uptr cacheStart = (uptr)_dyld_get_shared_cache_range(&cacheLength);
+ CHECK(cacheStart && cacheLength);
+
+ __block mach_header *dyldHdr = nullptr;
+ int res = dyld_shared_cache_iterate_text(
+ uuid, ^(const dyld_shared_cache_dylib_text_info *info) {
+ CHECK_GE(info->version, 2);
+ mach_header *hdr =
+ (mach_header *)(cacheStart + info->textSegmentOffset);
+ if (IsDyldHdr(hdr))
+ dyldHdr = hdr;
+ });
+ CHECK_EQ(res, 0);
+
+ return dyldHdr;
+}
+
const mach_header *get_dyld_hdr() {
- if (!dyld_hdr) dyld_hdr = get_dyld_image_header();
+ if (!dyld_hdr) {
+ // On macOS 13+, dyld itself has moved into the shared cache. Looking it up
+ // via vm_region_recurse_64() causes spins/hangs/crashes.
+ if (GetMacosAlignedVersion() >= MacosVersion(13, 0)) {
+ dyld_hdr = GetDyldImageHeaderViaSharedCache();
+ if (!dyld_hdr) {
+ VReport(1,
+ "Failed to lookup the dyld image header in the shared cache on "
+ "macOS 13+ (or no shared cache in use). Falling back to "
+ "lookup via vm_region_recurse_64().\n");
+ dyld_hdr = GetDyldImageHeaderViaVMRegion();
+ }
+ } else {
+ dyld_hdr = GetDyldImageHeaderViaVMRegion();
+ }
+ CHECK(dyld_hdr);
+ }
return dyld_hdr;
}
} // namespace __sanitizer
-#endif // SANITIZER_MAC
+#endif // SANITIZER_APPLE
#undef _FILE_OFFSET_BITS
#include "sanitizer_platform.h"
#if SANITIZER_SOLARIS
-#include "sanitizer_common.h"
-#include "sanitizer_procmaps.h"
+# include <fcntl.h>
+# include <limits.h>
+# include <procfs.h>
-#include <procfs.h>
-#include <limits.h>
+# include "sanitizer_common.h"
+# include "sanitizer_procmaps.h"
namespace __sanitizer {
void ReadProcMaps(ProcSelfMapsBuff *proc_maps) {
- if (!ReadFileToBuffer("/proc/self/xmap", &proc_maps->data,
- &proc_maps->mmaped_size, &proc_maps->len)) {
- proc_maps->data = nullptr;
- proc_maps->mmaped_size = 0;
- proc_maps->len = 0;
- }
+ uptr fd = internal_open("/proc/self/xmap", O_RDONLY);
+ CHECK_NE(fd, -1);
+ uptr Size = internal_filesize(fd);
+ CHECK_GT(Size, 0);
+
+ // Allow for additional entries by following mmap.
+ size_t MmapedSize = Size * 4 / 3;
+ void *VmMap = MmapOrDie(MmapedSize, "ReadProcMaps()");
+ Size = internal_read(fd, VmMap, MmapedSize);
+ CHECK_NE(Size, -1);
+ internal_close(fd);
+ proc_maps->data = (char *)VmMap;
+ proc_maps->mmaped_size = MmapedSize;
+ proc_maps->len = Size;
}
bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {
segment->protection |= kProtectionWrite;
if ((xmapentry->pr_mflags & MA_EXEC) != 0)
segment->protection |= kProtectionExecute;
+ if ((xmapentry->pr_mflags & MA_SHARED) != 0)
+ segment->protection |= kProtectionShared;
if (segment->filename != NULL && segment->filename_size > 0) {
char proc_path[PATH_MAX + 1];
- internal_snprintf(proc_path, sizeof(proc_path), "/proc/self/path/%s",
- xmapentry->pr_mapname);
- internal_readlink(proc_path, segment->filename, segment->filename_size);
+ // Avoid unnecessary readlink on unnamed entires.
+ if (xmapentry->pr_mapname[0] == '\0')
+ segment->filename[0] = '\0';
+ else {
+ internal_snprintf(proc_path, sizeof(proc_path), "/proc/self/path/%s",
+ xmapentry->pr_mapname);
+ ssize_t sz = internal_readlink(proc_path, segment->filename,
+ segment->filename_size - 1);
+
+ // If readlink failed, the map is anonymous.
+ if (sz == -1)
+ segment->filename[0] = '\0';
+ else if ((size_t)sz < segment->filename_size)
+ // readlink doesn't NUL-terminate.
+ segment->filename[sz] = '\0';
+ }
}
data_.current += sizeof(prxmap_t);
Cache cache_;
char pad2_[kCacheLineSize];
- void NOINLINE Recycle(uptr min_size, Callback cb) REQUIRES(recycle_mutex_)
- RELEASE(recycle_mutex_) {
+ void NOINLINE Recycle(uptr min_size, Callback cb)
+ SANITIZER_REQUIRES(recycle_mutex_) SANITIZER_RELEASE(recycle_mutex_) {
Cache tmp;
{
SpinMutexLock l(&cache_mutex_);
// Lower bytes store the address of the next buffer element.
static constexpr int kPageSizeBits = 12;
static constexpr int kSizeShift = 56;
+ static constexpr int kSizeBits = 64 - kSizeShift;
static constexpr uptr kNextMask = (1ULL << kSizeShift) - 1;
uptr GetStorageSize() const { return (long_ >> kSizeShift) << kPageSizeBits; }
+ static uptr SignExtend(uptr x) { return ((sptr)x) << kSizeBits >> kSizeBits; }
+
void Init(void *storage, uptr size) {
CHECK_EQ(sizeof(CompactRingBuffer<T>), sizeof(void *));
CHECK(IsPowerOfTwo(size));
CHECK_LE(size, 128 << kPageSizeBits);
CHECK_EQ(size % 4096, 0);
CHECK_EQ(size % sizeof(T), 0);
- CHECK_EQ((uptr)storage % (size * 2), 0);
- long_ = (uptr)storage | ((size >> kPageSizeBits) << kSizeShift);
+ uptr st = (uptr)storage;
+ CHECK_EQ(st % (size * 2), 0);
+ CHECK_EQ(st, SignExtend(st & kNextMask));
+ long_ = (st & kNextMask) | ((size >> kPageSizeBits) << kSizeShift);
}
void SetNext(const T *next) {
- long_ = (long_ & ~kNextMask) | (uptr)next;
+ long_ = (long_ & ~kNextMask) | ((uptr)next & kNextMask);
}
public:
SetNext((const T *)storage + Idx);
}
- T *Next() const { return (T *)(long_ & kNextMask); }
+ T *Next() const { return (T *)(SignExtend(long_ & kNextMask)); }
void *StartOfStorage() const {
return (void *)((uptr)Next() & ~(GetStorageSize() - 1));
#endif
#ifndef SIGNAL_INTERCEPTOR_SIGACTION_IMPL
-#define SIGNAL_INTERCEPTOR_SIGACTION_IMPL(signum, act, oldact) \
- { return REAL(sigaction_symname)(signum, act, oldact); }
+# define SIGNAL_INTERCEPTOR_SIGACTION_IMPL(signum, act, oldact) \
+ { \
+ if (!REAL(sigaction_symname)) { \
+ Printf( \
+ "Warning: REAL(sigaction_symname) == nullptr. This may happen " \
+ "if you link with ubsan statically. Sigaction will not work.\n"); \
+ return -1; \
+ } \
+ return REAL(sigaction_symname)(signum, act, oldact); \
+ }
#endif
#if SANITIZER_INTERCEPT_BSD_SIGNAL
void FutexWake(atomic_uint32_t *p, u32 count) {}
-BlockingMutex::BlockingMutex() {
- CHECK(sizeof(mutex_t) <= sizeof(opaque_storage_));
- internal_memset(this, 0, sizeof(*this));
- CHECK_EQ(mutex_init((mutex_t *)&opaque_storage_, USYNC_THREAD, NULL), 0);
-}
-
-void BlockingMutex::Lock() {
- CHECK(sizeof(mutex_t) <= sizeof(opaque_storage_));
- CHECK_NE(owner_, (uptr)thr_self());
- CHECK_EQ(mutex_lock((mutex_t *)&opaque_storage_), 0);
- CHECK(!owner_);
- owner_ = (uptr)thr_self();
-}
-
-void BlockingMutex::Unlock() {
- CHECK(owner_ == (uptr)thr_self());
- owner_ = 0;
- CHECK_EQ(mutex_unlock((mutex_t *)&opaque_storage_), 0);
-}
-
-void BlockingMutex::CheckLocked() const { CHECK_EQ((uptr)thr_self(), owner_); }
-
} // namespace __sanitizer
#endif // SANITIZER_SOLARIS
--- /dev/null
+//===-- sanitizer_solaris.h -------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of Sanitizer runtime. It contains Solaris-specific
+// definitions.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_SOLARIS_H
+#define SANITIZER_SOLARIS_H
+
+#include "sanitizer_internal_defs.h"
+
+#if SANITIZER_SOLARIS
+
+#include <link.h>
+
+namespace __sanitizer {
+
+// Beginning of declaration from OpenSolaris/Illumos
+// $SRC/cmd/sgs/include/rtld.h.
+struct Rt_map {
+ Link_map rt_public;
+ const char *rt_pathname;
+ ulong_t rt_padstart;
+ ulong_t rt_padimlen;
+ ulong_t rt_msize;
+ uint_t rt_flags;
+ uint_t rt_flags1;
+ ulong_t rt_tlsmodid;
+};
+
+// Structure matching the Solaris 11.4 struct dl_phdr_info used to determine
+// presence of dlpi_tls_modid field at runtime. Cf. Solaris 11.4
+// dl_iterate_phdr(3C), Example 2.
+struct dl_phdr_info_test {
+ ElfW(Addr) dlpi_addr;
+ const char *dlpi_name;
+ const ElfW(Phdr) * dlpi_phdr;
+ ElfW(Half) dlpi_phnum;
+ u_longlong_t dlpi_adds;
+ u_longlong_t dlpi_subs;
+ size_t dlpi_tls_modid;
+ void *dlpi_tls_data;
+};
+
+} // namespace __sanitizer
+
+#endif // SANITIZER_SOLARIS
+
+#endif // SANITIZER_SOLARIS_H
--- /dev/null
+//===-- sanitizer_stack_store.cpp -------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_stack_store.h"
+
+#include "sanitizer_atomic.h"
+#include "sanitizer_common.h"
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_leb128.h"
+#include "sanitizer_lzw.h"
+#include "sanitizer_placement_new.h"
+#include "sanitizer_stacktrace.h"
+
+namespace __sanitizer {
+
+namespace {
+struct StackTraceHeader {
+ static constexpr u32 kStackSizeBits = 8;
+
+ u8 size;
+ u8 tag;
+ explicit StackTraceHeader(const StackTrace &trace)
+ : size(Min<uptr>(trace.size, (1u << 8) - 1)), tag(trace.tag) {
+ CHECK_EQ(trace.tag, static_cast<uptr>(tag));
+ }
+ explicit StackTraceHeader(uptr h)
+ : size(h & ((1 << kStackSizeBits) - 1)), tag(h >> kStackSizeBits) {}
+
+ uptr ToUptr() const {
+ return static_cast<uptr>(size) | (static_cast<uptr>(tag) << kStackSizeBits);
+ }
+};
+} // namespace
+
+StackStore::Id StackStore::Store(const StackTrace &trace, uptr *pack) {
+ if (!trace.size && !trace.tag)
+ return 0;
+ StackTraceHeader h(trace);
+ uptr idx = 0;
+ *pack = 0;
+ uptr *stack_trace = Alloc(h.size + 1, &idx, pack);
+ *stack_trace = h.ToUptr();
+ internal_memcpy(stack_trace + 1, trace.trace, h.size * sizeof(uptr));
+ *pack += blocks_[GetBlockIdx(idx)].Stored(h.size + 1);
+ return OffsetToId(idx);
+}
+
+StackTrace StackStore::Load(Id id) {
+ if (!id)
+ return {};
+ uptr idx = IdToOffset(id);
+ uptr block_idx = GetBlockIdx(idx);
+ CHECK_LT(block_idx, ARRAY_SIZE(blocks_));
+ const uptr *stack_trace = blocks_[block_idx].GetOrUnpack(this);
+ if (!stack_trace)
+ return {};
+ stack_trace += GetInBlockIdx(idx);
+ StackTraceHeader h(*stack_trace);
+ return StackTrace(stack_trace + 1, h.size, h.tag);
+}
+
+uptr StackStore::Allocated() const {
+ return atomic_load_relaxed(&allocated_) + sizeof(*this);
+}
+
+uptr *StackStore::Alloc(uptr count, uptr *idx, uptr *pack) {
+ for (;;) {
+ // Optimisic lock-free allocation, essentially try to bump the
+ // total_frames_.
+ uptr start = atomic_fetch_add(&total_frames_, count, memory_order_relaxed);
+ uptr block_idx = GetBlockIdx(start);
+ uptr last_idx = GetBlockIdx(start + count - 1);
+ if (LIKELY(block_idx == last_idx)) {
+ // Fits into the a single block.
+ CHECK_LT(block_idx, ARRAY_SIZE(blocks_));
+ *idx = start;
+ return blocks_[block_idx].GetOrCreate(this) + GetInBlockIdx(start);
+ }
+
+ // Retry. We can't use range allocated in two different blocks.
+ CHECK_LE(count, kBlockSizeFrames);
+ uptr in_first = kBlockSizeFrames - GetInBlockIdx(start);
+ // Mark tail/head of these blocks as "stored".to avoid waiting before we can
+ // Pack().
+ *pack += blocks_[block_idx].Stored(in_first);
+ *pack += blocks_[last_idx].Stored(count - in_first);
+ }
+}
+
+void *StackStore::Map(uptr size, const char *mem_type) {
+ atomic_fetch_add(&allocated_, size, memory_order_relaxed);
+ return MmapNoReserveOrDie(size, mem_type);
+}
+
+void StackStore::Unmap(void *addr, uptr size) {
+ atomic_fetch_sub(&allocated_, size, memory_order_relaxed);
+ UnmapOrDie(addr, size);
+}
+
+uptr StackStore::Pack(Compression type) {
+ uptr res = 0;
+ for (BlockInfo &b : blocks_) res += b.Pack(type, this);
+ return res;
+}
+
+void StackStore::LockAll() {
+ for (BlockInfo &b : blocks_) b.Lock();
+}
+
+void StackStore::UnlockAll() {
+ for (BlockInfo &b : blocks_) b.Unlock();
+}
+
+void StackStore::TestOnlyUnmap() {
+ for (BlockInfo &b : blocks_) b.TestOnlyUnmap(this);
+ internal_memset(this, 0, sizeof(*this));
+}
+
+uptr *StackStore::BlockInfo::Get() const {
+ // Idiomatic double-checked locking uses memory_order_acquire here. But
+ // relaxed is fine for us, justification is similar to
+ // TwoLevelMap::GetOrCreate.
+ return reinterpret_cast<uptr *>(atomic_load_relaxed(&data_));
+}
+
+uptr *StackStore::BlockInfo::Create(StackStore *store) {
+ SpinMutexLock l(&mtx_);
+ uptr *ptr = Get();
+ if (!ptr) {
+ ptr = reinterpret_cast<uptr *>(store->Map(kBlockSizeBytes, "StackStore"));
+ atomic_store(&data_, reinterpret_cast<uptr>(ptr), memory_order_release);
+ }
+ return ptr;
+}
+
+uptr *StackStore::BlockInfo::GetOrCreate(StackStore *store) {
+ uptr *ptr = Get();
+ if (LIKELY(ptr))
+ return ptr;
+ return Create(store);
+}
+
+class SLeb128Encoder {
+ public:
+ SLeb128Encoder(u8 *begin, u8 *end) : begin(begin), end(end) {}
+
+ bool operator==(const SLeb128Encoder &other) const {
+ return begin == other.begin;
+ }
+
+ bool operator!=(const SLeb128Encoder &other) const {
+ return begin != other.begin;
+ }
+
+ SLeb128Encoder &operator=(uptr v) {
+ sptr diff = v - previous;
+ begin = EncodeSLEB128(diff, begin, end);
+ previous = v;
+ return *this;
+ }
+ SLeb128Encoder &operator*() { return *this; }
+ SLeb128Encoder &operator++() { return *this; }
+
+ u8 *base() const { return begin; }
+
+ private:
+ u8 *begin;
+ u8 *end;
+ uptr previous = 0;
+};
+
+class SLeb128Decoder {
+ public:
+ SLeb128Decoder(const u8 *begin, const u8 *end) : begin(begin), end(end) {}
+
+ bool operator==(const SLeb128Decoder &other) const {
+ return begin == other.begin;
+ }
+
+ bool operator!=(const SLeb128Decoder &other) const {
+ return begin != other.begin;
+ }
+
+ uptr operator*() {
+ sptr diff;
+ begin = DecodeSLEB128(begin, end, &diff);
+ previous += diff;
+ return previous;
+ }
+ SLeb128Decoder &operator++() { return *this; }
+
+ SLeb128Decoder operator++(int) { return *this; }
+
+ private:
+ const u8 *begin;
+ const u8 *end;
+ uptr previous = 0;
+};
+
+static u8 *CompressDelta(const uptr *from, const uptr *from_end, u8 *to,
+ u8 *to_end) {
+ SLeb128Encoder encoder(to, to_end);
+ for (; from != from_end; ++from, ++encoder) *encoder = *from;
+ return encoder.base();
+}
+
+static uptr *UncompressDelta(const u8 *from, const u8 *from_end, uptr *to,
+ uptr *to_end) {
+ SLeb128Decoder decoder(from, from_end);
+ SLeb128Decoder end(from_end, from_end);
+ for (; decoder != end; ++to, ++decoder) *to = *decoder;
+ CHECK_EQ(to, to_end);
+ return to;
+}
+
+static u8 *CompressLzw(const uptr *from, const uptr *from_end, u8 *to,
+ u8 *to_end) {
+ SLeb128Encoder encoder(to, to_end);
+ encoder = LzwEncode<uptr>(from, from_end, encoder);
+ return encoder.base();
+}
+
+static uptr *UncompressLzw(const u8 *from, const u8 *from_end, uptr *to,
+ uptr *to_end) {
+ SLeb128Decoder decoder(from, from_end);
+ SLeb128Decoder end(from_end, from_end);
+ to = LzwDecode<uptr>(decoder, end, to);
+ CHECK_EQ(to, to_end);
+ return to;
+}
+
+#if defined(_MSC_VER) && !defined(__clang__)
+# pragma warning(push)
+// Disable 'nonstandard extension used: zero-sized array in struct/union'.
+# pragma warning(disable : 4200)
+#endif
+namespace {
+struct PackedHeader {
+ uptr size;
+ StackStore::Compression type;
+ u8 data[];
+};
+} // namespace
+#if defined(_MSC_VER) && !defined(__clang__)
+# pragma warning(pop)
+#endif
+
+uptr *StackStore::BlockInfo::GetOrUnpack(StackStore *store) {
+ SpinMutexLock l(&mtx_);
+ switch (state) {
+ case State::Storing:
+ state = State::Unpacked;
+ FALLTHROUGH;
+ case State::Unpacked:
+ return Get();
+ case State::Packed:
+ break;
+ }
+
+ u8 *ptr = reinterpret_cast<u8 *>(Get());
+ CHECK_NE(nullptr, ptr);
+ const PackedHeader *header = reinterpret_cast<const PackedHeader *>(ptr);
+ CHECK_LE(header->size, kBlockSizeBytes);
+ CHECK_GE(header->size, sizeof(PackedHeader));
+
+ uptr packed_size_aligned = RoundUpTo(header->size, GetPageSizeCached());
+
+ uptr *unpacked =
+ reinterpret_cast<uptr *>(store->Map(kBlockSizeBytes, "StackStoreUnpack"));
+
+ uptr *unpacked_end;
+ switch (header->type) {
+ case Compression::Delta:
+ unpacked_end = UncompressDelta(header->data, ptr + header->size, unpacked,
+ unpacked + kBlockSizeFrames);
+ break;
+ case Compression::LZW:
+ unpacked_end = UncompressLzw(header->data, ptr + header->size, unpacked,
+ unpacked + kBlockSizeFrames);
+ break;
+ default:
+ UNREACHABLE("Unexpected type");
+ break;
+ }
+
+ CHECK_EQ(kBlockSizeFrames, unpacked_end - unpacked);
+
+ MprotectReadOnly(reinterpret_cast<uptr>(unpacked), kBlockSizeBytes);
+ atomic_store(&data_, reinterpret_cast<uptr>(unpacked), memory_order_release);
+ store->Unmap(ptr, packed_size_aligned);
+
+ state = State::Unpacked;
+ return Get();
+}
+
+uptr StackStore::BlockInfo::Pack(Compression type, StackStore *store) {
+ if (type == Compression::None)
+ return 0;
+
+ SpinMutexLock l(&mtx_);
+ switch (state) {
+ case State::Unpacked:
+ case State::Packed:
+ return 0;
+ case State::Storing:
+ break;
+ }
+
+ uptr *ptr = Get();
+ if (!ptr || !Stored(0))
+ return 0;
+
+ u8 *packed =
+ reinterpret_cast<u8 *>(store->Map(kBlockSizeBytes, "StackStorePack"));
+ PackedHeader *header = reinterpret_cast<PackedHeader *>(packed);
+ u8 *alloc_end = packed + kBlockSizeBytes;
+
+ u8 *packed_end = nullptr;
+ switch (type) {
+ case Compression::Delta:
+ packed_end =
+ CompressDelta(ptr, ptr + kBlockSizeFrames, header->data, alloc_end);
+ break;
+ case Compression::LZW:
+ packed_end =
+ CompressLzw(ptr, ptr + kBlockSizeFrames, header->data, alloc_end);
+ break;
+ default:
+ UNREACHABLE("Unexpected type");
+ break;
+ }
+
+ header->type = type;
+ header->size = packed_end - packed;
+
+ VPrintf(1, "Packed block of %zu KiB to %zu KiB\n", kBlockSizeBytes >> 10,
+ header->size >> 10);
+
+ if (kBlockSizeBytes - header->size < kBlockSizeBytes / 8) {
+ VPrintf(1, "Undo and keep block unpacked\n");
+ MprotectReadOnly(reinterpret_cast<uptr>(ptr), kBlockSizeBytes);
+ store->Unmap(packed, kBlockSizeBytes);
+ state = State::Unpacked;
+ return 0;
+ }
+
+ uptr packed_size_aligned = RoundUpTo(header->size, GetPageSizeCached());
+ store->Unmap(packed + packed_size_aligned,
+ kBlockSizeBytes - packed_size_aligned);
+ MprotectReadOnly(reinterpret_cast<uptr>(packed), packed_size_aligned);
+
+ atomic_store(&data_, reinterpret_cast<uptr>(packed), memory_order_release);
+ store->Unmap(ptr, kBlockSizeBytes);
+
+ state = State::Packed;
+ return kBlockSizeBytes - packed_size_aligned;
+}
+
+void StackStore::BlockInfo::TestOnlyUnmap(StackStore *store) {
+ if (uptr *ptr = Get())
+ store->Unmap(ptr, kBlockSizeBytes);
+}
+
+bool StackStore::BlockInfo::Stored(uptr n) {
+ return n + atomic_fetch_add(&stored_, n, memory_order_release) ==
+ kBlockSizeFrames;
+}
+
+bool StackStore::BlockInfo::IsPacked() const {
+ SpinMutexLock l(&mtx_);
+ return state == State::Packed;
+}
+
+} // namespace __sanitizer
--- /dev/null
+//===-- sanitizer_stack_store.h ---------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_STACK_STORE_H
+#define SANITIZER_STACK_STORE_H
+
+#include "sanitizer_atomic.h"
+#include "sanitizer_common.h"
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_mutex.h"
+#include "sanitizer_stacktrace.h"
+
+namespace __sanitizer {
+
+class StackStore {
+ static constexpr uptr kBlockSizeFrames = 0x100000;
+ static constexpr uptr kBlockCount = 0x1000;
+ static constexpr uptr kBlockSizeBytes = kBlockSizeFrames * sizeof(uptr);
+
+ public:
+ enum class Compression : u8 {
+ None = 0,
+ Delta,
+ LZW,
+ };
+
+ constexpr StackStore() = default;
+
+ using Id = u32; // Enough for 2^32 * sizeof(uptr) bytes of traces.
+ static_assert(u64(kBlockCount) * kBlockSizeFrames == 1ull << (sizeof(Id) * 8),
+ "");
+
+ Id Store(const StackTrace &trace,
+ uptr *pack /* number of blocks completed by this call */);
+ StackTrace Load(Id id);
+ uptr Allocated() const;
+
+ // Packs all blocks which don't expect any more writes. A block is going to be
+ // packed once. As soon trace from that block was requested, it will unpack
+ // and stay unpacked after that.
+ // Returns the number of released bytes.
+ uptr Pack(Compression type);
+
+ void LockAll();
+ void UnlockAll();
+
+ void TestOnlyUnmap();
+
+ private:
+ friend class StackStoreTest;
+ static constexpr uptr GetBlockIdx(uptr frame_idx) {
+ return frame_idx / kBlockSizeFrames;
+ }
+
+ static constexpr uptr GetInBlockIdx(uptr frame_idx) {
+ return frame_idx % kBlockSizeFrames;
+ }
+
+ static constexpr uptr IdToOffset(Id id) {
+ CHECK_NE(id, 0);
+ return id - 1; // Avoid zero as id.
+ }
+
+ static constexpr uptr OffsetToId(Id id) {
+ // This makes UINT32_MAX to 0 and it will be retrived as and empty stack.
+ // But this is not a problem as we will not be able to store anything after
+ // that anyway.
+ return id + 1; // Avoid zero as id.
+ }
+
+ uptr *Alloc(uptr count, uptr *idx, uptr *pack);
+
+ void *Map(uptr size, const char *mem_type);
+ void Unmap(void *addr, uptr size);
+
+ // Total number of allocated frames.
+ atomic_uintptr_t total_frames_ = {};
+
+ // Tracks total allocated memory in bytes.
+ atomic_uintptr_t allocated_ = {};
+
+ // Each block will hold pointer to exactly kBlockSizeFrames.
+ class BlockInfo {
+ atomic_uintptr_t data_;
+ // Counter to track store progress to know when we can Pack() the block.
+ atomic_uint32_t stored_;
+ // Protects alloc of new blocks.
+ mutable StaticSpinMutex mtx_;
+
+ enum class State : u8 {
+ Storing = 0,
+ Packed,
+ Unpacked,
+ };
+ State state SANITIZER_GUARDED_BY(mtx_);
+
+ uptr *Create(StackStore *store);
+
+ public:
+ uptr *Get() const;
+ uptr *GetOrCreate(StackStore *store);
+ uptr *GetOrUnpack(StackStore *store);
+ uptr Pack(Compression type, StackStore *store);
+ void TestOnlyUnmap(StackStore *store);
+ bool Stored(uptr n);
+ bool IsPacked() const;
+ void Lock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS { mtx_.Lock(); }
+ void Unlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS { mtx_.Unlock(); }
+ };
+
+ BlockInfo blocks_[kBlockCount] = {};
+};
+
+} // namespace __sanitizer
+
+#endif // SANITIZER_STACK_STORE_H
#include "sanitizer_stackdepot.h"
+#include "sanitizer_atomic.h"
#include "sanitizer_common.h"
#include "sanitizer_hash.h"
+#include "sanitizer_mutex.h"
+#include "sanitizer_stack_store.h"
#include "sanitizer_stackdepotbase.h"
namespace __sanitizer {
struct StackDepotNode {
- StackDepotNode *link;
- u32 id;
- atomic_uint32_t hash_and_use_count; // hash_bits : 12; use_count : 20;
- u32 size;
- u32 tag;
- uptr stack[1]; // [size]
+ using hash_type = u64;
+ hash_type stack_hash;
+ u32 link;
+ StackStore::Id store_id;
static const u32 kTabSizeLog = SANITIZER_ANDROID ? 16 : 20;
- // Lower kTabSizeLog bits are equal for all items in one bucket.
- // We use these bits to store the per-stack use counter.
- static const u32 kUseCountBits = kTabSizeLog;
- static const u32 kMaxUseCount = 1 << kUseCountBits;
- static const u32 kUseCountMask = (1 << kUseCountBits) - 1;
- static const u32 kHashMask = ~kUseCountMask;
typedef StackTrace args_type;
- bool eq(u32 hash, const args_type &args) const {
- u32 hash_bits =
- atomic_load(&hash_and_use_count, memory_order_relaxed) & kHashMask;
- if ((hash & kHashMask) != hash_bits || args.size != size || args.tag != tag)
- return false;
- uptr i = 0;
- for (; i < size; i++) {
- if (stack[i] != args.trace[i]) return false;
- }
- return true;
- }
- static uptr storage_size(const args_type &args) {
- return sizeof(StackDepotNode) + (args.size - 1) * sizeof(uptr);
+ bool eq(hash_type hash, const args_type &args) const {
+ return hash == stack_hash;
}
- static u32 hash(const args_type &args) {
- MurMur2HashBuilder H(args.size * sizeof(uptr));
+ static uptr allocated();
+ static hash_type hash(const args_type &args) {
+ MurMur2Hash64Builder H(args.size * sizeof(uptr));
for (uptr i = 0; i < args.size; i++) H.add(args.trace[i]);
+ H.add(args.tag);
return H.get();
}
static bool is_valid(const args_type &args) {
return args.size > 0 && args.trace;
}
- void store(const args_type &args, u32 hash) {
- atomic_store(&hash_and_use_count, hash & kHashMask, memory_order_relaxed);
- size = args.size;
- tag = args.tag;
- internal_memcpy(stack, args.trace, size * sizeof(uptr));
- }
- args_type load() const {
- return args_type(&stack[0], size, tag);
- }
- StackDepotHandle get_handle() { return StackDepotHandle(this); }
+ void store(u32 id, const args_type &args, hash_type hash);
+ args_type load(u32 id) const;
+ static StackDepotHandle get_handle(u32 id);
typedef StackDepotHandle handle_type;
};
-COMPILER_CHECK(StackDepotNode::kMaxUseCount == (u32)kStackDepotMaxUseCount);
-
-u32 StackDepotHandle::id() { return node_->id; }
-int StackDepotHandle::use_count() {
- return atomic_load(&node_->hash_and_use_count, memory_order_relaxed) &
- StackDepotNode::kUseCountMask;
-}
-void StackDepotHandle::inc_use_count_unsafe() {
- u32 prev =
- atomic_fetch_add(&node_->hash_and_use_count, 1, memory_order_relaxed) &
- StackDepotNode::kUseCountMask;
- CHECK_LT(prev + 1, StackDepotNode::kMaxUseCount);
-}
+static StackStore stackStore;
// FIXME(dvyukov): this single reserved bit is used in TSan.
typedef StackDepotBase<StackDepotNode, 1, StackDepotNode::kTabSizeLog>
StackDepot;
static StackDepot theDepot;
+// Keep mutable data out of frequently access nodes to improve caching
+// efficiency.
+static TwoLevelMap<atomic_uint32_t, StackDepot::kNodesSize1,
+ StackDepot::kNodesSize2>
+ useCounts;
+
+int StackDepotHandle::use_count() const {
+ return atomic_load_relaxed(&useCounts[id_]);
+}
+
+void StackDepotHandle::inc_use_count_unsafe() {
+ atomic_fetch_add(&useCounts[id_], 1, memory_order_relaxed);
+}
+
+uptr StackDepotNode::allocated() {
+ return stackStore.Allocated() + useCounts.MemoryUsage();
+}
+
+static void CompressStackStore() {
+ u64 start = Verbosity() >= 1 ? MonotonicNanoTime() : 0;
+ uptr diff = stackStore.Pack(static_cast<StackStore::Compression>(
+ Abs(common_flags()->compress_stack_depot)));
+ if (!diff)
+ return;
+ if (Verbosity() >= 1) {
+ u64 finish = MonotonicNanoTime();
+ uptr total_before = theDepot.GetStats().allocated + diff;
+ VPrintf(1, "%s: StackDepot released %zu KiB out of %zu KiB in %llu ms\n",
+ SanitizerToolName, diff >> 10, total_before >> 10,
+ (finish - start) / 1000000);
+ }
+}
+
+namespace {
+
+class CompressThread {
+ public:
+ constexpr CompressThread() = default;
+ void NewWorkNotify();
+ void Stop();
+ void LockAndStop() SANITIZER_NO_THREAD_SAFETY_ANALYSIS;
+ void Unlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS;
+
+ private:
+ enum class State {
+ NotStarted = 0,
+ Started,
+ Failed,
+ Stopped,
+ };
+
+ void Run();
+
+ bool WaitForWork() {
+ semaphore_.Wait();
+ return atomic_load(&run_, memory_order_acquire);
+ }
+
+ Semaphore semaphore_ = {};
+ StaticSpinMutex mutex_ = {};
+ State state_ SANITIZER_GUARDED_BY(mutex_) = State::NotStarted;
+ void *thread_ SANITIZER_GUARDED_BY(mutex_) = nullptr;
+ atomic_uint8_t run_ = {};
+};
+
+static CompressThread compress_thread;
+
+void CompressThread::NewWorkNotify() {
+ int compress = common_flags()->compress_stack_depot;
+ if (!compress)
+ return;
+ if (compress > 0 /* for testing or debugging */) {
+ SpinMutexLock l(&mutex_);
+ if (state_ == State::NotStarted) {
+ atomic_store(&run_, 1, memory_order_release);
+ CHECK_EQ(nullptr, thread_);
+ thread_ = internal_start_thread(
+ [](void *arg) -> void * {
+ reinterpret_cast<CompressThread *>(arg)->Run();
+ return nullptr;
+ },
+ this);
+ state_ = thread_ ? State::Started : State::Failed;
+ }
+ if (state_ == State::Started) {
+ semaphore_.Post();
+ return;
+ }
+ }
+ CompressStackStore();
+}
+
+void CompressThread::Run() {
+ VPrintf(1, "%s: StackDepot compression thread started\n", SanitizerToolName);
+ while (WaitForWork()) CompressStackStore();
+ VPrintf(1, "%s: StackDepot compression thread stopped\n", SanitizerToolName);
+}
+
+void CompressThread::Stop() {
+ void *t = nullptr;
+ {
+ SpinMutexLock l(&mutex_);
+ if (state_ != State::Started)
+ return;
+ state_ = State::Stopped;
+ CHECK_NE(nullptr, thread_);
+ t = thread_;
+ thread_ = nullptr;
+ }
+ atomic_store(&run_, 0, memory_order_release);
+ semaphore_.Post();
+ internal_join_thread(t);
+}
+
+void CompressThread::LockAndStop() {
+ mutex_.Lock();
+ if (state_ != State::Started)
+ return;
+ CHECK_NE(nullptr, thread_);
+
+ atomic_store(&run_, 0, memory_order_release);
+ semaphore_.Post();
+ internal_join_thread(thread_);
+ // Allow to restart after Unlock() if needed.
+ state_ = State::NotStarted;
+ thread_ = nullptr;
+}
+
+void CompressThread::Unlock() { mutex_.Unlock(); }
+
+} // namespace
-StackDepotStats *StackDepotGetStats() {
- return theDepot.GetStats();
+void StackDepotNode::store(u32 id, const args_type &args, hash_type hash) {
+ stack_hash = hash;
+ uptr pack = 0;
+ store_id = stackStore.Store(args, &pack);
+ if (LIKELY(!pack))
+ return;
+ compress_thread.NewWorkNotify();
}
-u32 StackDepotPut(StackTrace stack) {
- StackDepotHandle h = theDepot.Put(stack);
- return h.valid() ? h.id() : 0;
+StackDepotNode::args_type StackDepotNode::load(u32 id) const {
+ if (!store_id)
+ return {};
+ return stackStore.Load(store_id);
}
+StackDepotStats StackDepotGetStats() { return theDepot.GetStats(); }
+
+u32 StackDepotPut(StackTrace stack) { return theDepot.Put(stack); }
+
StackDepotHandle StackDepotPut_WithHandle(StackTrace stack) {
- return theDepot.Put(stack);
+ return StackDepotNode::get_handle(theDepot.Put(stack));
}
StackTrace StackDepotGet(u32 id) {
void StackDepotLockAll() {
theDepot.LockAll();
+ compress_thread.LockAndStop();
+ stackStore.LockAll();
}
void StackDepotUnlockAll() {
+ stackStore.UnlockAll();
+ compress_thread.Unlock();
theDepot.UnlockAll();
}
#endif
}
-bool StackDepotReverseMap::IdDescPair::IdComparator(
- const StackDepotReverseMap::IdDescPair &a,
- const StackDepotReverseMap::IdDescPair &b) {
- return a.id < b.id;
-}
+void StackDepotStopBackgroundThread() { compress_thread.Stop(); }
-StackDepotReverseMap::StackDepotReverseMap() {
- map_.reserve(StackDepotGetStats()->n_uniq_ids + 100);
- for (int idx = 0; idx < StackDepot::kTabSize; idx++) {
- atomic_uintptr_t *p = &theDepot.tab[idx];
- uptr v = atomic_load(p, memory_order_consume);
- StackDepotNode *s = (StackDepotNode*)(v & ~1);
- for (; s; s = s->link) {
- IdDescPair pair = {s->id, s};
- map_.push_back(pair);
- }
- }
- Sort(map_.data(), map_.size(), &IdDescPair::IdComparator);
+StackDepotHandle StackDepotNode::get_handle(u32 id) {
+ return StackDepotHandle(&theDepot.nodes[id], id);
}
-StackTrace StackDepotReverseMap::Get(u32 id) {
- if (!map_.size())
- return StackTrace();
- IdDescPair pair = {id, nullptr};
- uptr idx = InternalLowerBound(map_, pair, IdDescPair::IdComparator);
- if (idx > map_.size() || map_[idx].id != id)
- return StackTrace();
- return map_[idx].desc->load();
+void StackDepotTestOnlyUnmap() {
+ theDepot.TestOnlyUnmap();
+ stackStore.TestOnlyUnmap();
}
} // namespace __sanitizer
// StackDepot efficiently stores huge amounts of stack traces.
struct StackDepotNode;
struct StackDepotHandle {
- StackDepotNode *node_;
- StackDepotHandle() : node_(nullptr) {}
- explicit StackDepotHandle(StackDepotNode *node) : node_(node) {}
- bool valid() { return node_; }
- u32 id();
- int use_count();
+ StackDepotNode *node_ = nullptr;
+ u32 id_ = 0;
+ StackDepotHandle(StackDepotNode *node, u32 id) : node_(node), id_(id) {}
+ bool valid() const { return node_; }
+ u32 id() const { return id_; }
+ int use_count() const;
void inc_use_count_unsafe();
};
const int kStackDepotMaxUseCount = 1U << (SANITIZER_ANDROID ? 16 : 20);
-StackDepotStats *StackDepotGetStats();
+StackDepotStats StackDepotGetStats();
u32 StackDepotPut(StackTrace stack);
StackDepotHandle StackDepotPut_WithHandle(StackTrace stack);
// Retrieves a stored stack trace by the id.
void StackDepotLockAll();
void StackDepotUnlockAll();
void StackDepotPrintAll();
+void StackDepotStopBackgroundThread();
-// Instantiating this class creates a snapshot of StackDepot which can be
-// efficiently queried with StackDepotGet(). You can use it concurrently with
-// StackDepot, but the snapshot is only guaranteed to contain those stack traces
-// which were stored before it was instantiated.
-class StackDepotReverseMap {
- public:
- StackDepotReverseMap();
- StackTrace Get(u32 id);
-
- private:
- struct IdDescPair {
- u32 id;
- StackDepotNode *desc;
-
- static bool IdComparator(const IdDescPair &a, const IdDescPair &b);
- };
-
- InternalMmapVector<IdDescPair> map_;
-
- // Disallow evil constructors.
- StackDepotReverseMap(const StackDepotReverseMap&);
- void operator=(const StackDepotReverseMap&);
-};
+void StackDepotTestOnlyUnmap();
} // namespace __sanitizer
#include <stdio.h>
#include "sanitizer_atomic.h"
+#include "sanitizer_flat_map.h"
#include "sanitizer_internal_defs.h"
#include "sanitizer_mutex.h"
-#include "sanitizer_persistent_allocator.h"
namespace __sanitizer {
template <class Node, int kReservedBits, int kTabSizeLog>
class StackDepotBase {
+ static constexpr u32 kIdSizeLog =
+ sizeof(u32) * 8 - Max(kReservedBits, 1 /* At least 1 bit for locking. */);
+ static constexpr u32 kNodesSize1Log = kIdSizeLog / 2;
+ static constexpr u32 kNodesSize2Log = kIdSizeLog - kNodesSize1Log;
+ static constexpr int kTabSize = 1 << kTabSizeLog; // Hash table size.
+ static constexpr u32 kUnlockMask = (1ull << kIdSizeLog) - 1;
+ static constexpr u32 kLockMask = ~kUnlockMask;
+
public:
typedef typename Node::args_type args_type;
typedef typename Node::handle_type handle_type;
+ typedef typename Node::hash_type hash_type;
+
+ static constexpr u64 kNodesSize1 = 1ull << kNodesSize1Log;
+ static constexpr u64 kNodesSize2 = 1ull << kNodesSize2Log;
+
// Maps stack trace to an unique id.
- handle_type Put(args_type args, bool *inserted = nullptr);
+ u32 Put(args_type args, bool *inserted = nullptr);
// Retrieves a stored stack trace by the id.
args_type Get(u32 id);
- StackDepotStats *GetStats() { return &stats; }
+ StackDepotStats GetStats() const {
+ return {
+ atomic_load_relaxed(&n_uniq_ids),
+ nodes.MemoryUsage() + Node::allocated(),
+ };
+ }
void LockAll();
void UnlockAll();
void PrintAll();
- private:
- static Node *find(Node *s, args_type args, u32 hash);
- static Node *lock(atomic_uintptr_t *p);
- static void unlock(atomic_uintptr_t *p, Node *s);
+ void TestOnlyUnmap() {
+ nodes.TestOnlyUnmap();
+ internal_memset(this, 0, sizeof(*this));
+ }
- static const int kTabSize = 1 << kTabSizeLog; // Hash table size.
- static const int kPartBits = 8;
- static const int kPartShift = sizeof(u32) * 8 - kPartBits - kReservedBits;
- static const int kPartCount =
- 1 << kPartBits; // Number of subparts in the table.
- static const int kPartSize = kTabSize / kPartCount;
- static const int kMaxId = 1 << kPartShift;
+ private:
+ friend Node;
+ u32 find(u32 s, args_type args, hash_type hash) const;
+ static u32 lock(atomic_uint32_t *p);
+ static void unlock(atomic_uint32_t *p, u32 s);
+ atomic_uint32_t tab[kTabSize]; // Hash table of Node's.
- atomic_uintptr_t tab[kTabSize]; // Hash table of Node's.
- atomic_uint32_t seq[kPartCount]; // Unique id generators.
+ atomic_uint32_t n_uniq_ids;
- StackDepotStats stats;
+ TwoLevelMap<Node, kNodesSize1, kNodesSize2> nodes;
friend class StackDepotReverseMap;
};
template <class Node, int kReservedBits, int kTabSizeLog>
-Node *StackDepotBase<Node, kReservedBits, kTabSizeLog>::find(Node *s,
- args_type args,
- u32 hash) {
+u32 StackDepotBase<Node, kReservedBits, kTabSizeLog>::find(
+ u32 s, args_type args, hash_type hash) const {
// Searches linked list s for the stack, returns its id.
- for (; s; s = s->link) {
- if (s->eq(hash, args)) {
+ for (; s;) {
+ const Node &node = nodes[s];
+ if (node.eq(hash, args))
return s;
- }
+ s = node.link;
}
- return nullptr;
+ return 0;
}
template <class Node, int kReservedBits, int kTabSizeLog>
-Node *StackDepotBase<Node, kReservedBits, kTabSizeLog>::lock(
- atomic_uintptr_t *p) {
+u32 StackDepotBase<Node, kReservedBits, kTabSizeLog>::lock(atomic_uint32_t *p) {
// Uses the pointer lsb as mutex.
for (int i = 0;; i++) {
- uptr cmp = atomic_load(p, memory_order_relaxed);
- if ((cmp & 1) == 0 &&
- atomic_compare_exchange_weak(p, &cmp, cmp | 1, memory_order_acquire))
- return (Node *)cmp;
+ u32 cmp = atomic_load(p, memory_order_relaxed);
+ if ((cmp & kLockMask) == 0 &&
+ atomic_compare_exchange_weak(p, &cmp, cmp | kLockMask,
+ memory_order_acquire))
+ return cmp;
if (i < 10)
proc_yield(10);
else
template <class Node, int kReservedBits, int kTabSizeLog>
void StackDepotBase<Node, kReservedBits, kTabSizeLog>::unlock(
- atomic_uintptr_t *p, Node *s) {
- DCHECK_EQ((uptr)s & 1, 0);
- atomic_store(p, (uptr)s, memory_order_release);
+ atomic_uint32_t *p, u32 s) {
+ DCHECK_EQ(s & kLockMask, 0);
+ atomic_store(p, s, memory_order_release);
}
template <class Node, int kReservedBits, int kTabSizeLog>
-typename StackDepotBase<Node, kReservedBits, kTabSizeLog>::handle_type
-StackDepotBase<Node, kReservedBits, kTabSizeLog>::Put(args_type args,
- bool *inserted) {
- if (inserted) *inserted = false;
- if (!Node::is_valid(args)) return handle_type();
- uptr h = Node::hash(args);
- atomic_uintptr_t *p = &tab[h % kTabSize];
- uptr v = atomic_load(p, memory_order_consume);
- Node *s = (Node *)(v & ~1);
+u32 StackDepotBase<Node, kReservedBits, kTabSizeLog>::Put(args_type args,
+ bool *inserted) {
+ if (inserted)
+ *inserted = false;
+ if (!LIKELY(Node::is_valid(args)))
+ return 0;
+ hash_type h = Node::hash(args);
+ atomic_uint32_t *p = &tab[h % kTabSize];
+ u32 v = atomic_load(p, memory_order_consume);
+ u32 s = v & kUnlockMask;
// First, try to find the existing stack.
- Node *node = find(s, args, h);
- if (node) return node->get_handle();
+ u32 node = find(s, args, h);
+ if (LIKELY(node))
+ return node;
+
// If failed, lock, retry and insert new.
- Node *s2 = lock(p);
+ u32 s2 = lock(p);
if (s2 != s) {
node = find(s2, args, h);
if (node) {
unlock(p, s2);
- return node->get_handle();
+ return node;
}
}
- uptr part = (h % kTabSize) / kPartSize;
- u32 id = atomic_fetch_add(&seq[part], 1, memory_order_relaxed) + 1;
- stats.n_uniq_ids++;
- CHECK_LT(id, kMaxId);
- id |= part << kPartShift;
- CHECK_NE(id, 0);
- CHECK_EQ(id & (((u32)-1) >> kReservedBits), id);
- uptr memsz = Node::storage_size(args);
- s = (Node *)PersistentAlloc(memsz);
- stats.allocated += memsz;
- s->id = id;
- s->store(args, h);
- s->link = s2;
+ s = atomic_fetch_add(&n_uniq_ids, 1, memory_order_relaxed) + 1;
+ CHECK_EQ(s & kUnlockMask, s);
+ CHECK_EQ(s & (((u32)-1) >> kReservedBits), s);
+ Node &new_node = nodes[s];
+ new_node.store(s, args, h);
+ new_node.link = s2;
unlock(p, s);
if (inserted) *inserted = true;
- return s->get_handle();
+ return s;
}
template <class Node, int kReservedBits, int kTabSizeLog>
typename StackDepotBase<Node, kReservedBits, kTabSizeLog>::args_type
StackDepotBase<Node, kReservedBits, kTabSizeLog>::Get(u32 id) {
- if (id == 0) {
+ if (id == 0)
return args_type();
- }
CHECK_EQ(id & (((u32)-1) >> kReservedBits), id);
- // High kPartBits contain part id, so we need to scan at most kPartSize lists.
- uptr part = id >> kPartShift;
- for (int i = 0; i != kPartSize; i++) {
- uptr idx = part * kPartSize + i;
- CHECK_LT(idx, kTabSize);
- atomic_uintptr_t *p = &tab[idx];
- uptr v = atomic_load(p, memory_order_consume);
- Node *s = (Node *)(v & ~1);
- for (; s; s = s->link) {
- if (s->id == id) {
- return s->load();
- }
- }
- }
- return args_type();
+ if (!nodes.contains(id))
+ return args_type();
+ const Node &node = nodes[id];
+ return node.load(id);
}
template <class Node, int kReservedBits, int kTabSizeLog>
template <class Node, int kReservedBits, int kTabSizeLog>
void StackDepotBase<Node, kReservedBits, kTabSizeLog>::UnlockAll() {
for (int i = 0; i < kTabSize; ++i) {
- atomic_uintptr_t *p = &tab[i];
+ atomic_uint32_t *p = &tab[i];
uptr s = atomic_load(p, memory_order_relaxed);
- unlock(p, (Node *)(s & ~1UL));
+ unlock(p, s & kUnlockMask);
}
}
template <class Node, int kReservedBits, int kTabSizeLog>
void StackDepotBase<Node, kReservedBits, kTabSizeLog>::PrintAll() {
for (int i = 0; i < kTabSize; ++i) {
- atomic_uintptr_t *p = &tab[i];
- lock(p);
- uptr v = atomic_load(p, memory_order_relaxed);
- Node *s = (Node *)(v & ~1UL);
- for (; s; s = s->link) {
- Printf("Stack for id %u:\n", s->id);
- s->load().Print();
+ atomic_uint32_t *p = &tab[i];
+ u32 s = atomic_load(p, memory_order_consume) & kUnlockMask;
+ for (; s;) {
+ const Node &node = nodes[s];
+ Printf("Stack for id %u:\n", s);
+ node.load(s).Print();
+ s = node.link;
}
- unlock(p, s);
}
}
namespace __sanitizer {
uptr StackTrace::GetNextInstructionPc(uptr pc) {
-#if defined(__sparc__) || defined(__mips__)
+#if defined(__aarch64__)
+ return STRIP_PAC_PC((void *)pc) + 4;
+#elif defined(__sparc__) || defined(__mips__)
return pc + 8;
-#elif defined(__powerpc__) || defined(__arm__) || defined(__aarch64__)
- return pc + 4;
#elif SANITIZER_RISCV64
// Current check order is 4 -> 2 -> 6 -> 8
u8 InsnByte = *(u8 *)(pc);
}
// bail-out if could not figure out the instruction size
return 0;
-#else
+#elif SANITIZER_S390 || SANITIZER_I386 || SANITIZER_X32 || SANITIZER_X64
return pc + 1;
+#else
+ return pc + 4;
#endif
}
top_frame_bp = 0;
}
-// Sparc implemention is in its own file.
+// Sparc implementation is in its own file.
#if !defined(__sparc__)
// In GCC on ARM bp points to saved lr, not fp, so we should check the next
uhwptr pc1 = caller_frame[2];
#elif defined(__s390__)
uhwptr pc1 = frame[14];
-#elif defined(__riscv)
+#elif defined(__loongarch__) || defined(__riscv)
// frame[-1] contains the return address
uhwptr pc1 = frame[-1];
#else
trace_buffer[size++] = (uptr) pc1;
}
bottom = (uptr)frame;
-#if defined(__riscv)
+#if defined(__loongarch__) || defined(__riscv)
// frame[-2] contain fp of the previous frame
uptr new_bp = (uptr)frame[-2];
#else
struct BufferedStackTrace;
-static const u32 kStackTraceMax = 256;
+static const u32 kStackTraceMax = 255;
#if SANITIZER_LINUX && defined(__mips__)
# define SANITIZER_CAN_FAST_UNWIND 0
// Fast unwind is the only option on Mac for now; we will need to
// revisit this macro when slow unwind works on Mac, see
// https://github.com/google/sanitizers/issues/137
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
# define SANITIZER_CAN_SLOW_UNWIND 0
#else
# define SANITIZER_CAN_SLOW_UNWIND 1
// so we return (pc-2) in that case in order to be safe.
// For A32 mode we return (pc-4) because all instructions are 32 bit long.
return (pc - 3) & (~1);
-#elif defined(__powerpc__) || defined(__powerpc64__) || defined(__aarch64__)
- // PCs are always 4 byte aligned.
- return pc - 4;
#elif defined(__sparc__) || defined(__mips__)
return pc - 8;
#elif SANITIZER_RISCV64
// It seems difficult to figure out the exact instruction length -
// pc - 2 seems like a safe option for the purposes of stack tracing
return pc - 2;
-#else
+#elif SANITIZER_S390 || SANITIZER_I386 || SANITIZER_X32 || SANITIZER_X64
return pc - 1;
+#else
+ return pc - 4;
#endif
}
// StackTrace::GetCurrentPc() faster.
#if defined(__x86_64__)
# define GET_CURRENT_PC() \
- ({ \
+ (__extension__({ \
uptr pc; \
asm("lea 0(%%rip), %0" : "=r"(pc)); \
pc; \
- })
+ }))
#else
# define GET_CURRENT_PC() StackTrace::GetCurrentPc()
#endif
if (dedup_token_->length())
dedup_token_->append("--");
if (stack->info.function != nullptr)
- dedup_token_->append(stack->info.function);
+ dedup_token_->append("%s", stack->info.function);
}
}
UnwindFast(pc, bp, stack_top, stack_bottom, max_depth);
}
-static int GetModuleAndOffsetForPc(uptr pc, char *module_name,
- uptr module_name_len, uptr *pc_offset) {
+int GetModuleAndOffsetForPc(uptr pc, char *module_name, uptr module_name_len,
+ uptr *pc_offset) {
const char *found_module_name = nullptr;
bool ok = Symbolizer::GetOrInit()->GetModuleNameAndOffsetForPC(
pc, &found_module_name, pc_offset);
}
SANITIZER_INTERFACE_ATTRIBUTE
-int __sanitizer_get_module_and_offset_for_pc(uptr pc, char *module_name,
+int __sanitizer_get_module_and_offset_for_pc(void *pc, char *module_name,
uptr module_name_len,
- uptr *pc_offset) {
- return __sanitizer::GetModuleAndOffsetForPc(pc, module_name, module_name_len,
- pc_offset);
+ void **pc_offset) {
+ return __sanitizer::GetModuleAndOffsetForPc(
+ reinterpret_cast<uptr>(pc), module_name, module_name_len,
+ reinterpret_cast<uptr *>(pc_offset));
}
} // extern "C"
return function;
}
+static void MaybeBuildIdToBuffer(const AddressInfo &info, bool PrefixSpace,
+ InternalScopedString *buffer) {
+ if (info.uuid_size) {
+ if (PrefixSpace)
+ buffer->append(" ");
+ buffer->append("(BuildId: ");
+ for (uptr i = 0; i < info.uuid_size; ++i) {
+ buffer->append("%02x", info.uuid[i]);
+ }
+ buffer->append(")");
+ }
+}
+
static const char kDefaultFormat[] = " #%n %p %F %L";
void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no,
break;
// Frame number and all fields of AddressInfo structure.
case 'n':
- buffer->append("%zu", frame_no);
+ buffer->append("%u", frame_no);
break;
case 'p':
buffer->append("0x%zx", address);
case 'o':
buffer->append("0x%zx", info->module_offset);
break;
+ case 'b':
+ MaybeBuildIdToBuffer(*info, /*PrefixSpace=*/false, buffer);
+ break;
case 'f':
buffer->append("%s", DemangleFunctionName(StripFunctionName(
info->function, strip_func_prefix)));
} else if (info->module) {
RenderModuleLocation(buffer, info->module, info->module_offset,
info->module_arch, strip_path_prefix);
+
+ MaybeBuildIdToBuffer(*info, /*PrefixSpace=*/true, buffer);
} else {
buffer->append("(<unknown module>)");
}
// Always strip the module name for %M.
RenderModuleLocation(buffer, StripModuleName(info->module),
info->module_offset, info->module_arch, "");
+ MaybeBuildIdToBuffer(*info, /*PrefixSpace=*/true, buffer);
} else {
buffer->append("(%p)", (void *)address);
}
break;
default:
- Report("Unsupported specifier in stack frame format: %c (0x%zx)!\n", *p,
- *p);
+ Report("Unsupported specifier in stack frame format: %c (%p)!\n", *p,
+ (void *)p);
Die();
}
}
buffer->append("%s", StripPathPrefix(DI->file, strip_path_prefix));
break;
case 'l':
- buffer->append("%d", DI->line);
+ buffer->append("%zu", DI->line);
break;
case 'g':
buffer->append("%s", DI->name);
break;
default:
- Report("Unsupported specifier in stack frame format: %c (0x%zx)!\n", *p,
- *p);
+ Report("Unsupported specifier in stack frame format: %c (%p)!\n", *p,
+ (void *)p);
Die();
}
}
// This file is shared between AddressSanitizer and ThreadSanitizer
// run-time libraries.
//
-// Implemention of fast stack unwinding for Sparc.
+// Implementation of fast stack unwinding for Sparc.
//===----------------------------------------------------------------------===//
#if defined(__sparc__)
#if SANITIZER_LINUX && \
(defined(__x86_64__) || defined(__mips__) || defined(__aarch64__) || \
defined(__powerpc64__) || defined(__s390__) || defined(__i386__) || \
- defined(__arm__) || SANITIZER_RISCV64)
+ defined(__arm__) || SANITIZER_RISCV64 || SANITIZER_LOONGARCH64)
#include "sanitizer_stoptheworld.h"
#include <sys/types.h> // for pid_t
#include <sys/uio.h> // for iovec
#include <elf.h> // for NT_PRSTATUS
-#if (defined(__aarch64__) || SANITIZER_RISCV64) && !SANITIZER_ANDROID
+#if (defined(__aarch64__) || SANITIZER_RISCV64 || SANITIZER_LOONGARCH64) && \
+ !SANITIZER_ANDROID
// GLIBC 2.20+ sys/user does not include asm/ptrace.h
# include <asm/ptrace.h>
#endif
void *callback_argument;
// The tracer thread waits on this mutex while the parent finishes its
// preparations.
- BlockingMutex mutex;
+ Mutex mutex;
// Tracer thread signals its completion by setting done.
atomic_uintptr_t done;
uptr parent_pid;
static constexpr uptr kExtraRegs[] = {0};
#define ARCH_IOVEC_FOR_GETREGSET
+#elif defined(__loongarch__)
+typedef struct user_pt_regs regs_struct;
+#define REG_SP regs[3]
+static constexpr uptr kExtraRegs[] = {0};
+#define ARCH_IOVEC_FOR_GETREGSET
+
#elif SANITIZER_RISCV64
typedef struct user_regs_struct regs_struct;
// sys/ucontext.h already defines REG_SP as 2. Undefine it first.
#endif // SANITIZER_LINUX && (defined(__x86_64__) || defined(__mips__)
// || defined(__aarch64__) || defined(__powerpc64__)
// || defined(__s390__) || defined(__i386__) || defined(__arm__)
+ // || SANITIZER_LOONGARCH64
#include "sanitizer_platform.h"
-#if SANITIZER_MAC && (defined(__x86_64__) || defined(__aarch64__) || \
+#if SANITIZER_APPLE && (defined(__x86_64__) || defined(__aarch64__) || \
defined(__i386))
#include <mach/mach.h>
class SuspendedThreadsListMac final : public SuspendedThreadsList {
public:
- SuspendedThreadsListMac() : threads_(1024) {}
+ SuspendedThreadsListMac() = default;
tid_t GetThreadID(uptr index) const override;
thread_t GetThread(uptr index) const;
#if defined(__x86_64__)
typedef x86_thread_state64_t regs_struct;
+#define regs_flavor x86_THREAD_STATE64
#define SP_REG __rsp
#elif defined(__aarch64__)
typedef arm_thread_state64_t regs_struct;
+#define regs_flavor ARM_THREAD_STATE64
# if __DARWIN_UNIX03
# define SP_REG __sp
#elif defined(__i386)
typedef x86_thread_state32_t regs_struct;
+#define regs_flavor x86_THREAD_STATE32
#define SP_REG __esp
thread_t thread = GetThread(index);
regs_struct regs;
int err;
- mach_msg_type_number_t reg_count = MACHINE_THREAD_STATE_COUNT;
- err = thread_get_state(thread, MACHINE_THREAD_STATE, (thread_state_t)®s,
+ mach_msg_type_number_t reg_count = sizeof(regs) / sizeof(natural_t);
+ err = thread_get_state(thread, regs_flavor, (thread_state_t)®s,
®_count);
if (err != KERN_SUCCESS) {
VReport(1, "Error - unable to get registers for a thread\n");
} // namespace __sanitizer
-#endif // SANITIZER_MAC && (defined(__x86_64__) || defined(__aarch64__)) ||
+#endif // SANITIZER_APPLE && (defined(__x86_64__) || defined(__aarch64__)) ||
// defined(__i386))
struct TracerThreadArgument {
StopTheWorldCallback callback;
void *callback_argument;
- BlockingMutex mutex;
+ Mutex mutex;
atomic_uintptr_t done;
uptr parent_pid;
};
--- /dev/null
+//===-- sanitizer_stoptheworld_win.cpp ------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// See sanitizer_stoptheworld.h for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+
+#if SANITIZER_WINDOWS
+
+# define WIN32_LEAN_AND_MEAN
+# include <windows.h>
+// windows.h needs to be included before tlhelp32.h
+# include <tlhelp32.h>
+
+# include "sanitizer_stoptheworld.h"
+
+namespace __sanitizer {
+
+namespace {
+
+struct SuspendedThreadsListWindows final : public SuspendedThreadsList {
+ InternalMmapVector<HANDLE> threadHandles;
+ InternalMmapVector<DWORD> threadIds;
+
+ SuspendedThreadsListWindows() {
+ threadIds.reserve(1024);
+ threadHandles.reserve(1024);
+ }
+
+ PtraceRegistersStatus GetRegistersAndSP(uptr index,
+ InternalMmapVector<uptr> *buffer,
+ uptr *sp) const override;
+
+ tid_t GetThreadID(uptr index) const override;
+ uptr ThreadCount() const override;
+};
+
+// Stack Pointer register names on different architectures
+# if SANITIZER_X64
+# define SP_REG Rsp
+# elif SANITIZER_I386
+# define SP_REG Esp
+# elif SANITIZER_ARM | SANITIZER_ARM64
+# define SP_REG Sp
+# else
+# error Architecture not supported!
+# endif
+
+PtraceRegistersStatus SuspendedThreadsListWindows::GetRegistersAndSP(
+ uptr index, InternalMmapVector<uptr> *buffer, uptr *sp) const {
+ CHECK_LT(index, threadHandles.size());
+
+ buffer->resize(RoundUpTo(sizeof(CONTEXT), sizeof(uptr)) / sizeof(uptr));
+ CONTEXT *thread_context = reinterpret_cast<CONTEXT *>(buffer->data());
+ thread_context->ContextFlags = CONTEXT_ALL;
+ CHECK(GetThreadContext(threadHandles[index], thread_context));
+ *sp = thread_context->SP_REG;
+
+ return REGISTERS_AVAILABLE;
+}
+
+tid_t SuspendedThreadsListWindows::GetThreadID(uptr index) const {
+ CHECK_LT(index, threadIds.size());
+ return threadIds[index];
+}
+
+uptr SuspendedThreadsListWindows::ThreadCount() const {
+ return threadIds.size();
+}
+
+struct RunThreadArgs {
+ StopTheWorldCallback callback;
+ void *argument;
+};
+
+DWORD WINAPI RunThread(void *argument) {
+ RunThreadArgs *run_args = (RunThreadArgs *)argument;
+
+ const DWORD this_thread = GetCurrentThreadId();
+ const DWORD this_process = GetCurrentProcessId();
+
+ SuspendedThreadsListWindows suspended_threads_list;
+ bool new_thread_found;
+
+ do {
+ // Take a snapshot of all Threads
+ const HANDLE threads = CreateToolhelp32Snapshot(TH32CS_SNAPTHREAD, 0);
+ CHECK(threads != INVALID_HANDLE_VALUE);
+
+ THREADENTRY32 thread_entry;
+ thread_entry.dwSize = sizeof(thread_entry);
+ new_thread_found = false;
+
+ if (!Thread32First(threads, &thread_entry))
+ break;
+
+ do {
+ if (thread_entry.th32ThreadID == this_thread ||
+ thread_entry.th32OwnerProcessID != this_process)
+ continue;
+
+ bool suspended_thread = false;
+ for (const auto thread_id : suspended_threads_list.threadIds) {
+ if (thread_id == thread_entry.th32ThreadID) {
+ suspended_thread = true;
+ break;
+ }
+ }
+
+ // Skip the Thread if it was already suspended
+ if (suspended_thread)
+ continue;
+
+ const HANDLE thread =
+ OpenThread(THREAD_ALL_ACCESS, FALSE, thread_entry.th32ThreadID);
+ CHECK(thread);
+
+ if (SuspendThread(thread) == (DWORD)-1) {
+ DWORD last_error = GetLastError();
+
+ VPrintf(1, "Could not suspend thread %lu (error %lu)",
+ thread_entry.th32ThreadID, last_error);
+ continue;
+ }
+
+ suspended_threads_list.threadIds.push_back(thread_entry.th32ThreadID);
+ suspended_threads_list.threadHandles.push_back(thread);
+ new_thread_found = true;
+ } while (Thread32Next(threads, &thread_entry));
+
+ CloseHandle(threads);
+
+ // Between the call to `CreateToolhelp32Snapshot` and suspending the
+ // relevant Threads, new Threads could have potentially been created. So
+ // continue to find and suspend new Threads until we don't find any.
+ } while (new_thread_found);
+
+ // Now all Threads of this Process except of this Thread should be suspended.
+ // Execute the callback function.
+ run_args->callback(suspended_threads_list, run_args->argument);
+
+ // Resume all Threads
+ for (const auto suspended_thread_handle :
+ suspended_threads_list.threadHandles) {
+ CHECK_NE(ResumeThread(suspended_thread_handle), -1);
+ CloseHandle(suspended_thread_handle);
+ }
+
+ return 0;
+}
+
+} // namespace
+
+void StopTheWorld(StopTheWorldCallback callback, void *argument) {
+ struct RunThreadArgs arg = {callback, argument};
+ DWORD trace_thread_id;
+
+ auto trace_thread =
+ CreateThread(nullptr, 0, RunThread, &arg, 0, &trace_thread_id);
+ CHECK(trace_thread);
+
+ WaitForSingleObject(trace_thread, INFINITE);
+ CloseHandle(trace_thread);
+}
+
+} // namespace __sanitizer
+
+#endif // SANITIZER_WINDOWS
//===----------------------------------------------------------------------===//
#include "sanitizer_allocator_internal.h"
-#include "sanitizer_platform.h"
+#include "sanitizer_common.h"
#include "sanitizer_internal_defs.h"
#include "sanitizer_libc.h"
#include "sanitizer_placement_new.h"
+#include "sanitizer_platform.h"
#include "sanitizer_symbolizer_internal.h"
namespace __sanitizer {
InternalFree(file);
internal_memset(this, 0, sizeof(AddressInfo));
function_offset = kUnknown;
+ uuid_size = 0;
}
void AddressInfo::FillModuleInfo(const char *mod_name, uptr mod_offset,
module = internal_strdup(mod_name);
module_offset = mod_offset;
module_arch = mod_arch;
+ uuid_size = 0;
+}
+
+void AddressInfo::FillModuleInfo(const LoadedModule &mod) {
+ module = internal_strdup(mod.full_name());
+ module_offset = address - mod.base_address();
+ module_arch = mod.arch();
+ if (mod.uuid_size())
+ internal_memcpy(uuid, mod.uuid(), mod.uuid_size());
+ uuid_size = mod.uuid_size();
}
SymbolizedStack::SymbolizedStack() : next(nullptr), info() {}
sym_->end_hook_();
}
-void Symbolizer::LateInitializeTools() {
- for (auto &tool : tools_) {
- tool.LateInitialize();
- }
-}
-
} // namespace __sanitizer
char *module;
uptr module_offset;
ModuleArch module_arch;
+ u8 uuid[kModuleUUIDSize];
+ uptr uuid_size;
static const uptr kUnknown = ~(uptr)0;
char *function;
// Deletes all strings and resets all fields.
void Clear();
void FillModuleInfo(const char *mod_name, uptr mod_offset, ModuleArch arch);
+ void FillModuleInfo(const LoadedModule &mod);
+ uptr module_base() const { return address - module_offset; }
};
// Linked list of symbolized frames (each frame is described by AddressInfo).
// its method should be protected by |mu_|.
class ModuleNameOwner {
public:
- explicit ModuleNameOwner(BlockingMutex *synchronized_by)
+ explicit ModuleNameOwner(Mutex *synchronized_by)
: last_match_(nullptr), mu_(synchronized_by) {
storage_.reserve(kInitialCapacity);
}
InternalMmapVector<const char*> storage_;
const char *last_match_;
- BlockingMutex *mu_;
+ Mutex *mu_;
} module_names_;
/// Platform-specific function for creating a Symbolizer object.
// Mutex locked from public methods of |Symbolizer|, so that the internals
// (including individual symbolizer tools and platform-specific methods) are
// always synchronized.
- BlockingMutex mu_;
+ Mutex mu_;
IntrusiveList<SymbolizerTool> tools_;
private:
const Symbolizer *sym_;
};
-
- // Calls `LateInitialize()` on all items in `tools_`.
- void LateInitializeTools();
};
#ifdef SANITIZER_WINDOWS
// Parsing helpers, 'str' is searched for delimiter(s) and a string or uptr
// is extracted. When extracting a string, a newly allocated (using
-// InternalAlloc) and null-terminataed buffer is returned. They return a pointer
+// InternalAlloc) and null-terminated buffer is returned. They return a pointer
// to the next characted after the found delimiter.
const char *ExtractToken(const char *str, const char *delims, char **result);
const char *ExtractInt(const char *str, const char *delims, int *result);
return nullptr;
}
- // Called during the LateInitialize phase of Sanitizer initialization.
- // Usually this is a safe place to call code that might need to use user
- // memory allocators.
- virtual void LateInitialize() {}
-
protected:
~SymbolizerTool() {}
};
~SymbolizerProcess() {}
/// The maximum number of arguments required to invoke a tool process.
- static const unsigned kArgVMax = 6;
+ static const unsigned kArgVMax = 16;
// Customizable by subclasses.
virtual bool StartSymbolizerSubprocess();
- virtual bool ReadFromSymbolizer(char *buffer, uptr max_length);
+ virtual bool ReadFromSymbolizer();
// Return the environment to run the symbolizer in.
virtual char **GetEnvP() { return GetEnviron(); }
+ InternalMmapVector<char> &GetBuff() { return buffer_; }
private:
virtual bool ReachedEndOfOutput(const char *buffer, uptr length) const {
fd_t input_fd_;
fd_t output_fd_;
- static const uptr kBufferSize = 16 * 1024;
- char buffer_[kBufferSize];
+ InternalMmapVector<char> buffer_;
static const uptr kMaxTimesRestarted = 5;
static const int kSymbolizerStartupTimeMillis = 10;
}
SymbolizedStack *Symbolizer::SymbolizePC(uptr addr) {
- BlockingMutexLock l(&mu_);
- const char *module_name = nullptr;
- uptr module_offset;
- ModuleArch arch;
+ Lock l(&mu_);
SymbolizedStack *res = SymbolizedStack::New(addr);
- if (!FindModuleNameAndOffsetForAddress(addr, &module_name, &module_offset,
- &arch))
+ auto *mod = FindModuleForAddress(addr);
+ if (!mod)
return res;
// Always fill data about module name and offset.
- res->info.FillModuleInfo(module_name, module_offset, arch);
+ res->info.FillModuleInfo(*mod);
for (auto &tool : tools_) {
SymbolizerScope sym_scope(this);
if (tool.SymbolizePC(addr, res)) {
}
bool Symbolizer::SymbolizeData(uptr addr, DataInfo *info) {
- BlockingMutexLock l(&mu_);
+ Lock l(&mu_);
const char *module_name = nullptr;
uptr module_offset;
ModuleArch arch;
}
bool Symbolizer::SymbolizeFrame(uptr addr, FrameInfo *info) {
- BlockingMutexLock l(&mu_);
+ Lock l(&mu_);
const char *module_name = nullptr;
if (!FindModuleNameAndOffsetForAddress(
addr, &module_name, &info->module_offset, &info->module_arch))
bool Symbolizer::GetModuleNameAndOffsetForPC(uptr pc, const char **module_name,
uptr *module_address) {
- BlockingMutexLock l(&mu_);
+ Lock l(&mu_);
const char *internal_module_name = nullptr;
ModuleArch arch;
if (!FindModuleNameAndOffsetForAddress(pc, &internal_module_name,
}
void Symbolizer::Flush() {
- BlockingMutexLock l(&mu_);
+ Lock l(&mu_);
for (auto &tool : tools_) {
SymbolizerScope sym_scope(this);
tool.Flush();
}
const char *Symbolizer::Demangle(const char *name) {
- BlockingMutexLock l(&mu_);
+ Lock l(&mu_);
for (auto &tool : tools_) {
SymbolizerScope sym_scope(this);
if (const char *demangled = tool.Demangle(name))
class LLVMSymbolizerProcess final : public SymbolizerProcess {
public:
explicit LLVMSymbolizerProcess(const char *path)
- : SymbolizerProcess(path, /*use_posix_spawn=*/SANITIZER_MAC) {}
+ : SymbolizerProcess(path, /*use_posix_spawn=*/SANITIZER_APPLE) {}
private:
bool ReachedEndOfOutput(const char *buffer, uptr length) const override {
const char* const kSymbolizerArch = "--default-arch=x86_64";
#elif defined(__i386__)
const char* const kSymbolizerArch = "--default-arch=i386";
+#elif SANITIZER_LOONGARCH64
+ const char *const kSymbolizerArch = "--default-arch=loongarch64";
#elif SANITIZER_RISCV64
const char *const kSymbolizerArch = "--default-arch=riscv64";
#elif defined(__aarch64__)
const char* const kSymbolizerArch = "--default-arch=unknown";
#endif
- const char *const inline_flag = common_flags()->symbolize_inline_frames
- ? "--inlines"
- : "--no-inlines";
+ const char *const demangle_flag =
+ common_flags()->demangle ? "--demangle" : "--no-demangle";
+ const char *const inline_flag =
+ common_flags()->symbolize_inline_frames ? "--inlines" : "--no-inlines";
int i = 0;
argv[i++] = path_to_binary;
+ argv[i++] = demangle_flag;
argv[i++] = inline_flag;
argv[i++] = kSymbolizerArch;
argv[i++] = nullptr;
+ CHECK_LE(i, kArgVMax);
}
};
}
}
-// Parses a two-line string in the following format:
+// Parses a two- or three-line string in the following format:
// <symbol_name>
// <start_address> <size>
-// Used by LLVMSymbolizer and InternalSymbolizer.
+// <filename>:<column>
+// Used by LLVMSymbolizer and InternalSymbolizer. LLVMSymbolizer added support
+// for symbolizing the third line in D123538, but we support the older two-line
+// information as well.
void ParseSymbolizeDataOutput(const char *str, DataInfo *info) {
str = ExtractToken(str, "\n", &info->name);
str = ExtractUptr(str, " ", &info->start);
str = ExtractUptr(str, "\n", &info->size);
+ // Note: If the third line isn't present, these calls will set info.{file,
+ // line} to empty strings.
+ str = ExtractToken(str, ":", &info->file);
+ str = ExtractUptr(str, "\n", &info->line);
}
static void ParseSymbolizeFrameOutput(const char *str,
return nullptr;
if (!WriteToSymbolizer(command, internal_strlen(command)))
return nullptr;
- if (!ReadFromSymbolizer(buffer_, kBufferSize))
- return nullptr;
- return buffer_;
+ if (!ReadFromSymbolizer())
+ return nullptr;
+ return buffer_.data();
}
bool SymbolizerProcess::Restart() {
return StartSymbolizerSubprocess();
}
-bool SymbolizerProcess::ReadFromSymbolizer(char *buffer, uptr max_length) {
- if (max_length == 0)
- return true;
- uptr read_len = 0;
- while (true) {
+bool SymbolizerProcess::ReadFromSymbolizer() {
+ buffer_.clear();
+ constexpr uptr max_length = 1024;
+ bool ret = true;
+ do {
uptr just_read = 0;
- bool success = ReadFromFile(input_fd_, buffer + read_len,
- max_length - read_len - 1, &just_read);
+ uptr size_before = buffer_.size();
+ buffer_.resize(size_before + max_length);
+ buffer_.resize(buffer_.capacity());
+ bool ret = ReadFromFile(input_fd_, &buffer_[size_before],
+ buffer_.size() - size_before, &just_read);
+
+ if (!ret)
+ just_read = 0;
+
+ buffer_.resize(size_before + just_read);
+
// We can't read 0 bytes, as we don't expect external symbolizer to close
// its stdout.
- if (!success || just_read == 0) {
+ if (just_read == 0) {
Report("WARNING: Can't read from symbolizer at fd %d\n", input_fd_);
- return false;
- }
- read_len += just_read;
- if (ReachedEndOfOutput(buffer, read_len))
- break;
- if (read_len + 1 == max_length) {
- Report("WARNING: Symbolizer buffer too small\n");
- read_len = 0;
+ ret = false;
break;
}
- }
- buffer[read_len] = '\0';
- return true;
+ } while (!ReachedEndOfOutput(buffer_.data(), buffer_.size()));
+ buffer_.push_back('\0');
+ return ret;
}
bool SymbolizerProcess::WriteToSymbolizer(const char *buffer, uptr length) {
//===----------------------------------------------------------------------===//
#include "sanitizer_platform.h"
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
#include "sanitizer_allocator_internal.h"
#include "sanitizer_mac.h"
#include <dlfcn.h>
#include <errno.h>
-#include <mach/mach.h>
#include <stdlib.h>
#include <sys/wait.h>
#include <unistd.h>
return true;
}
-#define K_ATOS_ENV_VAR "__check_mach_ports_lookup"
-
-// This cannot live in `AtosSymbolizerProcess` because instances of that object
-// are allocated by the internal allocator which under ASan is poisoned with
-// kAsanInternalHeapMagic.
-static char kAtosMachPortEnvEntry[] = K_ATOS_ENV_VAR "=000000000000000";
-
class AtosSymbolizerProcess final : public SymbolizerProcess {
public:
explicit AtosSymbolizerProcess(const char *path)
pid_str_[0] = '\0';
}
- void LateInitialize() {
- if (SANITIZER_IOSSIM) {
- // `putenv()` may call malloc/realloc so it is only safe to do this
- // during LateInitialize() or later (i.e. we can't do this in the
- // constructor). We also can't do this in `StartSymbolizerSubprocess()`
- // because in TSan we switch allocators when we're symbolizing.
- // We use `putenv()` rather than `setenv()` so that we can later directly
- // write into the storage without LibC getting involved to change what the
- // variable is set to
- int result = putenv(kAtosMachPortEnvEntry);
- CHECK_EQ(result, 0);
- }
- }
-
private:
bool StartSymbolizerSubprocess() override {
- // Configure sandbox before starting atos process.
-
// Put the string command line argument in the object so that it outlives
// the call to GetArgV.
- internal_snprintf(pid_str_, sizeof(pid_str_), "%d", internal_getpid());
-
- if (SANITIZER_IOSSIM) {
- // `atos` in the simulator is restricted in its ability to retrieve the
- // task port for the target process (us) so we need to do extra work
- // to pass our task port to it.
- mach_port_t ports[]{mach_task_self()};
- kern_return_t ret =
- mach_ports_register(mach_task_self(), ports, /*count=*/1);
- CHECK_EQ(ret, KERN_SUCCESS);
-
- // Set environment variable that signals to `atos` that it should look
- // for our task port. We can't call `setenv()` here because it might call
- // malloc/realloc. To avoid that we instead update the
- // `mach_port_env_var_entry_` variable with our current PID.
- uptr count = internal_snprintf(kAtosMachPortEnvEntry,
- sizeof(kAtosMachPortEnvEntry),
- K_ATOS_ENV_VAR "=%s", pid_str_);
- CHECK_GE(count, sizeof(K_ATOS_ENV_VAR) + internal_strlen(pid_str_));
- // Document our assumption but without calling `getenv()` in normal
- // builds.
- DCHECK(getenv(K_ATOS_ENV_VAR));
- DCHECK_EQ(internal_strcmp(getenv(K_ATOS_ENV_VAR), pid_str_), 0);
- }
+ internal_snprintf(pid_str_, sizeof(pid_str_), "%d", (int)internal_getpid());
+ // Configure sandbox before starting atos process.
return SymbolizerProcess::StartSymbolizerSubprocess();
}
argv[i++] = "-d";
}
argv[i++] = nullptr;
+ CHECK_LE(i, kArgVMax);
}
char pid_str_[16];
- // Space for `\0` in `K_ATOS_ENV_VAR` is reused for `=`.
- static_assert(sizeof(kAtosMachPortEnvEntry) ==
- (sizeof(K_ATOS_ENV_VAR) + sizeof(pid_str_)),
- "sizes should match");
};
#undef K_ATOS_ENV_VAR
return true;
}
-void AtosSymbolizer::LateInitialize() { process_->LateInitialize(); }
-
} // namespace __sanitizer
-#endif // SANITIZER_MAC
+#endif // SANITIZER_APPLE
#define SANITIZER_SYMBOLIZER_MAC_H
#include "sanitizer_platform.h"
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
#include "sanitizer_symbolizer_internal.h"
bool SymbolizePC(uptr addr, SymbolizedStack *stack) override;
bool SymbolizeData(uptr addr, DataInfo *info) override;
- void LateInitialize() override;
private:
AtosSymbolizerProcess *process_;
} // namespace __sanitizer
-#endif // SANITIZER_MAC
+#endif // SANITIZER_APPLE
#endif // SANITIZER_SYMBOLIZER_MAC_H
return new (symbolizer_allocator_) Symbolizer({});
}
-void Symbolizer::LateInitialize() {
- Symbolizer::GetOrInit()->LateInitializeTools();
-}
+void Symbolizer::LateInitialize() { Symbolizer::GetOrInit(); }
void StartReportDeadlySignal() {}
void ReportDeadlySignal(const SignalContext &sig, u32 tid,
// symbolication.
static void InitializeSwiftDemangler() {
swift_demangle_f = (swift_demangle_ft)dlsym(RTLD_DEFAULT, "swift_demangle");
- (void)dlerror(); // Cleanup error message in case of failure
}
// Attempts to demangle a Swift name. The demangler will return nullptr if a
}
if (use_posix_spawn_) {
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
fd_t fd = internal_spawn(argv, const_cast<const char **>(GetEnvP()), &pid);
if (fd == kInvalidFd) {
Report("WARNING: failed to spawn external symbolizer (errno: %d)\n",
input_fd_ = fd;
output_fd_ = fd;
-#else // SANITIZER_MAC
+#else // SANITIZER_APPLE
UNIMPLEMENTED();
-#endif // SANITIZER_MAC
+#endif // SANITIZER_APPLE
} else {
fd_t infd[2] = {}, outfd[2] = {};
if (!CreateTwoHighNumberedPipes(infd, outfd)) {
const char *(&argv)[kArgVMax]) const override {
int i = 0;
argv[i++] = path_to_binary;
- argv[i++] = "-iCfe";
+ if (common_flags()->demangle)
+ argv[i++] = "-C";
+ if (common_flags()->symbolize_inline_frames)
+ argv[i++] = "-i";
+ argv[i++] = "-fe";
argv[i++] = module_name_;
argv[i++] = nullptr;
+ CHECK_LE(i, kArgVMax);
}
bool ReachedEndOfOutput(const char *buffer, uptr length) const override;
- bool ReadFromSymbolizer(char *buffer, uptr max_length) override {
- if (!SymbolizerProcess::ReadFromSymbolizer(buffer, max_length))
+ bool ReadFromSymbolizer() override {
+ if (!SymbolizerProcess::ReadFromSymbolizer())
return false;
- // The returned buffer is empty when output is valid, but exceeds
- // max_length.
- if (*buffer == '\0')
- return true;
+ auto &buff = GetBuff();
// We should cut out output_terminator_ at the end of given buffer,
// appended by addr2line to mark the end of its meaningful output.
// We cannot scan buffer from it's beginning, because it is legal for it
// to start with output_terminator_ in case given offset is invalid. So,
// scanning from second character.
- char *garbage = internal_strstr(buffer + 1, output_terminator_);
+ char *garbage = internal_strstr(buff.data() + 1, output_terminator_);
// This should never be NULL since buffer must end up with
// output_terminator_.
CHECK(garbage);
+
// Trim the buffer.
- garbage[0] = '\0';
+ uintptr_t new_size = garbage - buff.data();
+ GetBuff().resize(new_size);
+ GetBuff().push_back('\0');
return true;
}
FIRST_32_SECOND_64(UINT32_MAX, UINT64_MAX);
};
-#if SANITIZER_SUPPORTS_WEAK_HOOKS
+# if SANITIZER_SUPPORTS_WEAK_HOOKS
extern "C" {
SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE bool
__sanitizer_symbolize_code(const char *ModuleName, u64 ModuleOffset,
- char *Buffer, int MaxLength,
- bool SymbolizeInlineFrames);
-SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-bool __sanitizer_symbolize_data(const char *ModuleName, u64 ModuleOffset,
- char *Buffer, int MaxLength);
-SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-void __sanitizer_symbolize_flush();
-SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-int __sanitizer_symbolize_demangle(const char *Name, char *Buffer,
- int MaxLength);
+ char *Buffer, int MaxLength);
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE bool
+__sanitizer_symbolize_data(const char *ModuleName, u64 ModuleOffset,
+ char *Buffer, int MaxLength);
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+__sanitizer_symbolize_flush();
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE int
+__sanitizer_symbolize_demangle(const char *Name, char *Buffer, int MaxLength);
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE bool
+__sanitizer_symbolize_set_demangle(bool Demangle);
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE bool
+__sanitizer_symbolize_set_inline_frames(bool InlineFrames);
} // extern "C"
class InternalSymbolizer final : public SymbolizerTool {
public:
static InternalSymbolizer *get(LowLevelAllocator *alloc) {
- if (__sanitizer_symbolize_code != 0 &&
- __sanitizer_symbolize_data != 0) {
- return new(*alloc) InternalSymbolizer();
- }
+ if (__sanitizer_symbolize_set_demangle)
+ CHECK(__sanitizer_symbolize_set_demangle(common_flags()->demangle));
+ if (__sanitizer_symbolize_set_inline_frames)
+ CHECK(__sanitizer_symbolize_set_inline_frames(
+ common_flags()->symbolize_inline_frames));
+ if (__sanitizer_symbolize_code && __sanitizer_symbolize_data)
+ return new (*alloc) InternalSymbolizer();
return 0;
}
bool SymbolizePC(uptr addr, SymbolizedStack *stack) override {
bool result = __sanitizer_symbolize_code(
- stack->info.module, stack->info.module_offset, buffer_, kBufferSize,
- common_flags()->symbolize_inline_frames);
- if (result) ParseSymbolizePCOutput(buffer_, stack);
+ stack->info.module, stack->info.module_offset, buffer_, kBufferSize);
+ if (result)
+ ParseSymbolizePCOutput(buffer_, stack);
return result;
}
if (__sanitizer_symbolize_demangle) {
for (uptr res_length = 1024;
res_length <= InternalSizeClassMap::kMaxSize;) {
- char *res_buff = static_cast<char*>(InternalAlloc(res_length));
+ char *res_buff = static_cast<char *>(InternalAlloc(res_length));
uptr req_length =
__sanitizer_symbolize_demangle(name, res_buff, res_length);
if (req_length > res_length) {
}
private:
- InternalSymbolizer() { }
+ InternalSymbolizer() {}
static const int kBufferSize = 16 * 1024;
char buffer_[kBufferSize];
};
-#else // SANITIZER_SUPPORTS_WEAK_HOOKS
+# else // SANITIZER_SUPPORTS_WEAK_HOOKS
class InternalSymbolizer final : public SymbolizerTool {
public:
static InternalSymbolizer *get(LowLevelAllocator *alloc) { return 0; }
};
-#endif // SANITIZER_SUPPORTS_WEAK_HOOKS
+# endif // SANITIZER_SUPPORTS_WEAK_HOOKS
const char *Symbolizer::PlatformDemangle(const char *name) {
return DemangleSwiftAndCXX(name);
VReport(2, "Using llvm-symbolizer at user-specified path: %s\n", path);
return new(*allocator) LLVMSymbolizer(path, allocator);
} else if (!internal_strcmp(binary_name, "atos")) {
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
VReport(2, "Using atos at user-specified path: %s\n", path);
return new(*allocator) AtosSymbolizer(path, allocator);
-#else // SANITIZER_MAC
+#else // SANITIZER_APPLE
Report("ERROR: Using `atos` is only supported on Darwin.\n");
Die();
-#endif // SANITIZER_MAC
+#endif // SANITIZER_APPLE
} else if (!internal_strcmp(binary_name, "addr2line")) {
VReport(2, "Using addr2line at user-specified path: %s\n", path);
return new(*allocator) Addr2LinePool(path, allocator);
// Otherwise symbolizer program is unknown, let's search $PATH
CHECK(path == nullptr);
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
if (const char *found_path = FindPathToBinary("atos")) {
VReport(2, "Using atos found at: %s\n", found_path);
return new(*allocator) AtosSymbolizer(found_path, allocator);
}
-#endif // SANITIZER_MAC
+#endif // SANITIZER_APPLE
if (const char *found_path = FindPathToBinary("llvm-symbolizer")) {
VReport(2, "Using llvm-symbolizer found at: %s\n", found_path);
return new(*allocator) LLVMSymbolizer(found_path, allocator);
list->push_back(tool);
}
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
VReport(2, "Using dladdr symbolizer.\n");
list->push_back(new(*allocator) DlAddrSymbolizer());
-#endif // SANITIZER_MAC
+#endif // SANITIZER_APPLE
}
Symbolizer *Symbolizer::PlatformInit() {
}
void Symbolizer::LateInitialize() {
- Symbolizer::GetOrInit()->LateInitializeTools();
+ Symbolizer::GetOrInit();
InitializeSwiftDemangler();
}
#endif
}
-void ReportMmapWriteExec(int prot) {
+void ReportMmapWriteExec(int prot, int flags) {
#if SANITIZER_POSIX && (!SANITIZER_GO && !SANITIZER_ANDROID)
- if ((prot & (PROT_WRITE | PROT_EXEC)) != (PROT_WRITE | PROT_EXEC))
+ int pflags = (PROT_WRITE | PROT_EXEC);
+ if ((prot & pflags) != pflags)
return;
+# if SANITIZER_APPLE && defined(MAP_JIT)
+ if ((flags & MAP_JIT) == MAP_JIT)
+ return;
+# endif
+
ScopedErrorReportLock l;
SanitizerCommonDecorator d;
Report("Hint: pc points to the zero page.\n");
if (sig.is_memory_access) {
const char *access_type =
- sig.write_flag == SignalContext::WRITE
+ sig.write_flag == SignalContext::Write
? "WRITE"
- : (sig.write_flag == SignalContext::READ ? "READ" : "UNKNOWN");
+ : (sig.write_flag == SignalContext::Read ? "READ" : "UNKNOWN");
Report("The signal is caused by a %s memory access.\n", access_type);
if (!sig.is_true_faulting_addr)
Report("Hint: this fault was caused by a dereference of a high value "
// Check that tool command lines are simple and that complete escaping is
// unnecessary.
CHECK(!internal_strchr(arg, '"') && "quotes in args unsupported");
- CHECK(!internal_strstr(arg, "\\\\") &&
- "double backslashes in args unsupported");
CHECK(arglen > 0 && arg[arglen - 1] != '\\' &&
"args ending in backslash and empty args unsupported");
command_line.append("\"%s\" ", arg);
}
void Symbolizer::LateInitialize() {
- Symbolizer::GetOrInit()->LateInitializeTools();
+ Symbolizer::GetOrInit();
}
} // namespace __sanitizer
// NetBSD uses libc calls directly
#if !SANITIZER_NETBSD
-#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_SOLARIS
+#if SANITIZER_FREEBSD || SANITIZER_APPLE || SANITIZER_SOLARIS
# define SYSCALL(name) SYS_ ## name
#else
# define SYSCALL(name) __NR_ ## name
#endif
-#if defined(__x86_64__) && (SANITIZER_FREEBSD || SANITIZER_MAC)
+#if (defined(__x86_64__) && (SANITIZER_FREEBSD || SANITIZER_APPLE)) || \
+ (defined(__aarch64__) && SANITIZER_FREEBSD)
# define internal_syscall __syscall
# else
# define internal_syscall syscall
--- /dev/null
+//===-- sanitizer_syscall_linux_hexagon.inc ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implementations of internal_syscall and internal_iserror for Linux/hexagon.
+//
+//===----------------------------------------------------------------------===//
+
+#define SYSCALL(name) __NR_##name
+
+#define __internal_syscall_LL_E(x) \
+ ((union { \
+ long long ll; \
+ long l[2]; \
+ }){.ll = x}) \
+ .l[0], \
+ ((union { \
+ long long ll; \
+ long l[2]; \
+ }){.ll = x}) \
+ .l[1]
+#define __internal_syscall_LL_O(x) 0, __SYSCALL_LL_E((x))
+
+#define __asm_syscall(...) \
+ do { \
+ __asm__ __volatile__("trap0(#1)" : "=r"(r0) : __VA_ARGS__ : "memory"); \
+ return r0; \
+ } while (0)
+
+#define __internal_syscall0(n) (__internal_syscall)(n)
+
+static uptr __internal_syscall(long n) {
+ register u32 r6 __asm__("r6") = n;
+ register u32 r0 __asm__("r0");
+ __asm_syscall("r"(r6));
+}
+
+#define __internal_syscall1(n, a1) (__internal_syscall)(n, (long)(a1))
+
+static uptr __internal_syscall(long n, long a) {
+ register u32 r6 __asm__("r6") = n;
+ register u32 r0 __asm__("r0") = a;
+ __asm_syscall("r"(r6), "0"(r0));
+}
+
+#define __internal_syscall2(n, a1, a2) \
+ (__internal_syscall)(n, (long)(a1), (long)(a2))
+
+static uptr __internal_syscall(long n, long a, long b) {
+ register u32 r6 __asm__("r6") = n;
+ register u32 r0 __asm__("r0") = a;
+ register u32 r1 __asm__("r1") = b;
+ __asm_syscall("r"(r6), "0"(r0), "r"(r1));
+}
+
+#define __internal_syscall3(n, a1, a2, a3) \
+ (__internal_syscall)(n, (long)(a1), (long)(a2), (long)(a3))
+
+static uptr __internal_syscall(long n, long a, long b, long c) {
+ register u32 r6 __asm__("r6") = n;
+ register u32 r0 __asm__("r0") = a;
+ register u32 r1 __asm__("r1") = b;
+ register u32 r2 __asm__("r2") = c;
+ __asm_syscall("r"(r6), "0"(r0), "r"(r1), "r"(r2));
+}
+
+#define __internal_syscall4(n, a1, a2, a3, a4) \
+ (__internal_syscall)(n, (long)(a1), (long)(a2), (long)(a3), (long)(a4))
+
+static uptr __internal_syscall(long n, long a, long b, long c, long d) {
+ register u32 r6 __asm__("r6") = n;
+ register u32 r0 __asm__("r0") = a;
+ register u32 r1 __asm__("r1") = b;
+ register u32 r2 __asm__("r2") = c;
+ register u32 r3 __asm__("r3") = d;
+ __asm_syscall("r"(r6), "0"(r0), "r"(r1), "r"(r2), "r"(r3));
+}
+
+#define __internal_syscall5(n, a1, a2, a3, a4, a5) \
+ (__internal_syscall)(n, (long)(a1), (long)(a2), (long)(a3), (long)(a4), \
+ (long)(a5))
+
+static uptr __internal_syscall(long n, long a, long b, long c, long d, long e) {
+ register u32 r6 __asm__("r6") = n;
+ register u32 r0 __asm__("r0") = a;
+ register u32 r1 __asm__("r1") = b;
+ register u32 r2 __asm__("r2") = c;
+ register u32 r3 __asm__("r3") = d;
+ register u32 r4 __asm__("r4") = e;
+ __asm_syscall("r"(r6), "0"(r0), "r"(r1), "r"(r2), "r"(r3), "r"(r4));
+}
+
+#define __internal_syscall6(n, a1, a2, a3, a4, a5, a6) \
+ (__internal_syscall)(n, (long)(a1), (long)(a2), (long)(a3), (long)(a4), \
+ (long)(a5), (long)(a6))
+
+static uptr __internal_syscall(long n, long a, long b, long c, long d, long e,
+ long f) {
+ register u32 r6 __asm__("r6") = n;
+ register u32 r0 __asm__("r0") = a;
+ register u32 r1 __asm__("r1") = b;
+ register u32 r2 __asm__("r2") = c;
+ register u32 r3 __asm__("r3") = d;
+ register u32 r4 __asm__("r4") = e;
+ register u32 r5 __asm__("r5") = f;
+ __asm_syscall("r"(r6), "0"(r0), "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5));
+}
+
+#define __SYSCALL_NARGS_X(a1, a2, a3, a4, a5, a6, a7, a8, n, ...) n
+#define __SYSCALL_NARGS(...) \
+ __SYSCALL_NARGS_X(__VA_ARGS__, 7, 6, 5, 4, 3, 2, 1, 0, )
+#define __SYSCALL_CONCAT_X(a, b) a##b
+#define __SYSCALL_CONCAT(a, b) __SYSCALL_CONCAT_X(a, b)
+#define __SYSCALL_DISP(b, ...) \
+ __SYSCALL_CONCAT(b, __SYSCALL_NARGS(__VA_ARGS__))(__VA_ARGS__)
+
+#define internal_syscall(...) __SYSCALL_DISP(__internal_syscall, __VA_ARGS__)
+
+// Helper function used to avoid clobbering of errno.
+bool internal_iserror(uptr retval, int *rverrno) {
+ if (retval >= (uptr)-4095) {
+ if (rverrno)
+ *rverrno = -retval;
+ return true;
+ }
+ return false;
+}
--- /dev/null
+//===-- sanitizer_syscall_linux_loongarch64.inc -----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implementations of internal_syscall and internal_iserror for
+// Linux/loongarch64.
+//
+//===----------------------------------------------------------------------===//
+
+// About local register variables:
+// https://gcc.gnu.org/onlinedocs/gcc/Local-Register-Variables.html#Local-Register-Variables
+//
+// Kernel ABI:
+// https://lore.kernel.org/loongarch/1f353678-3398-e30b-1c87-6edb278f74db@xen0n.name/T/#m1613bc86c2d7bf5f6da92bd62984302bfd699a2f
+// syscall number is placed in a7
+// parameters, if present, are placed in a0-a6
+// upon return:
+// the return value is placed in a0
+// t0-t8 should be considered clobbered
+// all other registers are preserved
+#define SYSCALL(name) __NR_##name
+
+#define INTERNAL_SYSCALL_CLOBBERS \
+ "memory", "$t0", "$t1", "$t2", "$t3", "$t4", "$t5", "$t6", "$t7", "$t8"
+
+static uptr __internal_syscall(u64 nr) {
+ register u64 a7 asm("$a7") = nr;
+ register u64 a0 asm("$a0");
+ __asm__ volatile("syscall 0\n\t"
+ : "=r"(a0)
+ : "r"(a7)
+ : INTERNAL_SYSCALL_CLOBBERS);
+ return a0;
+}
+#define __internal_syscall0(n) (__internal_syscall)(n)
+
+static uptr __internal_syscall(u64 nr, u64 arg1) {
+ register u64 a7 asm("$a7") = nr;
+ register u64 a0 asm("$a0") = arg1;
+ __asm__ volatile("syscall 0\n\t"
+ : "+r"(a0)
+ : "r"(a7)
+ : INTERNAL_SYSCALL_CLOBBERS);
+ return a0;
+}
+#define __internal_syscall1(n, a1) (__internal_syscall)(n, (u64)(a1))
+
+static uptr __internal_syscall(u64 nr, u64 arg1, long arg2) {
+ register u64 a7 asm("$a7") = nr;
+ register u64 a0 asm("$a0") = arg1;
+ register u64 a1 asm("$a1") = arg2;
+ __asm__ volatile("syscall 0\n\t"
+ : "+r"(a0)
+ : "r"(a7), "r"(a1)
+ : INTERNAL_SYSCALL_CLOBBERS);
+ return a0;
+}
+#define __internal_syscall2(n, a1, a2) \
+ (__internal_syscall)(n, (u64)(a1), (long)(a2))
+
+static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3) {
+ register u64 a7 asm("$a7") = nr;
+ register u64 a0 asm("$a0") = arg1;
+ register u64 a1 asm("$a1") = arg2;
+ register u64 a2 asm("$a2") = arg3;
+ __asm__ volatile("syscall 0\n\t"
+ : "+r"(a0)
+ : "r"(a7), "r"(a1), "r"(a2)
+ : INTERNAL_SYSCALL_CLOBBERS);
+ return a0;
+}
+#define __internal_syscall3(n, a1, a2, a3) \
+ (__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3))
+
+static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3,
+ u64 arg4) {
+ register u64 a7 asm("$a7") = nr;
+ register u64 a0 asm("$a0") = arg1;
+ register u64 a1 asm("$a1") = arg2;
+ register u64 a2 asm("$a2") = arg3;
+ register u64 a3 asm("$a3") = arg4;
+ __asm__ volatile("syscall 0\n\t"
+ : "+r"(a0)
+ : "r"(a7), "r"(a1), "r"(a2), "r"(a3)
+ : INTERNAL_SYSCALL_CLOBBERS);
+ return a0;
+}
+#define __internal_syscall4(n, a1, a2, a3, a4) \
+ (__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3), (long)(a4))
+
+static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3, u64 arg4,
+ long arg5) {
+ register u64 a7 asm("$a7") = nr;
+ register u64 a0 asm("$a0") = arg1;
+ register u64 a1 asm("$a1") = arg2;
+ register u64 a2 asm("$a2") = arg3;
+ register u64 a3 asm("$a3") = arg4;
+ register u64 a4 asm("$a4") = arg5;
+ __asm__ volatile("syscall 0\n\t"
+ : "+r"(a0)
+ : "r"(a7), "r"(a1), "r"(a2), "r"(a3), "r"(a4)
+ : INTERNAL_SYSCALL_CLOBBERS);
+ return a0;
+}
+#define __internal_syscall5(n, a1, a2, a3, a4, a5) \
+ (__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3), (long)(a4), \
+ (u64)(a5))
+
+static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3, u64 arg4,
+ long arg5, long arg6) {
+ register u64 a7 asm("$a7") = nr;
+ register u64 a0 asm("$a0") = arg1;
+ register u64 a1 asm("$a1") = arg2;
+ register u64 a2 asm("$a2") = arg3;
+ register u64 a3 asm("$a3") = arg4;
+ register u64 a4 asm("$a4") = arg5;
+ register u64 a5 asm("$a5") = arg6;
+ __asm__ volatile("syscall 0\n\t"
+ : "+r"(a0)
+ : "r"(a7), "r"(a1), "r"(a2), "r"(a3), "r"(a4), "r"(a5)
+ : INTERNAL_SYSCALL_CLOBBERS);
+ return a0;
+}
+#define __internal_syscall6(n, a1, a2, a3, a4, a5, a6) \
+ (__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3), (long)(a4), \
+ (u64)(a5), (long)(a6))
+
+static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3, u64 arg4,
+ long arg5, long arg6, long arg7) {
+ register u64 a7 asm("$a7") = nr;
+ register u64 a0 asm("$a0") = arg1;
+ register u64 a1 asm("$a1") = arg2;
+ register u64 a2 asm("$a2") = arg3;
+ register u64 a3 asm("$a3") = arg4;
+ register u64 a4 asm("$a4") = arg5;
+ register u64 a5 asm("$a5") = arg6;
+ register u64 a6 asm("$a6") = arg7;
+ __asm__ volatile("syscall 0\n\t"
+ : "+r"(a0)
+ : "r"(a7), "r"(a1), "r"(a2), "r"(a3), "r"(a4), "r"(a5),
+ "r"(a6)
+ : INTERNAL_SYSCALL_CLOBBERS);
+ return a0;
+}
+#define __internal_syscall7(n, a1, a2, a3, a4, a5, a6, a7) \
+ (__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3), (long)(a4), \
+ (u64)(a5), (long)(a6), (long)(a7))
+
+#define __SYSCALL_NARGS_X(a1, a2, a3, a4, a5, a6, a7, a8, n, ...) n
+#define __SYSCALL_NARGS(...) \
+ __SYSCALL_NARGS_X(__VA_ARGS__, 7, 6, 5, 4, 3, 2, 1, 0, )
+#define __SYSCALL_CONCAT_X(a, b) a##b
+#define __SYSCALL_CONCAT(a, b) __SYSCALL_CONCAT_X(a, b)
+#define __SYSCALL_DISP(b, ...) \
+ __SYSCALL_CONCAT(b, __SYSCALL_NARGS(__VA_ARGS__))(__VA_ARGS__)
+
+#define internal_syscall(...) __SYSCALL_DISP(__internal_syscall, __VA_ARGS__)
+
+// Helper function used to avoid clobbering of errno.
+bool internal_iserror(uptr retval, int *internal_errno) {
+ if (retval >= (uptr)-4095) {
+ if (internal_errno)
+ *internal_errno = -retval;
+ return true;
+ }
+ return false;
+}
POST_SYSCALL(getcontext)(long long res, void *ucp_) { /* Nothing to do */ }
PRE_SYSCALL(setcontext)(void *ucp_) {
if (ucp_) {
- PRE_READ(ucp_, ucontext_t_sz);
+ PRE_READ(ucp_, ucontext_t_sz(ucp_));
}
}
POST_SYSCALL(setcontext)(long long res, void *ucp_) {}
PRE_SYSCALL(_lwp_create)(void *ucp_, long long flags_, void *new_lwp_) {
if (ucp_) {
- PRE_READ(ucp_, ucontext_t_sz);
+ PRE_READ(ucp_, ucontext_t_sz(ucp_));
}
}
POST_SYSCALL(_lwp_create)
#include "sanitizer_thread_registry.h"
+#include "sanitizer_placement_new.h"
+
namespace __sanitizer {
ThreadContextBase::ThreadContextBase(u32 tid)
max_threads_(max_threads),
thread_quarantine_size_(thread_quarantine_size),
max_reuse_(max_reuse),
- mtx_(),
+ mtx_(MutexThreadRegistry),
total_threads_(0),
alive_threads_(0),
max_alive_threads_(0),
void ThreadRegistry::GetNumberOfThreads(uptr *total, uptr *running,
uptr *alive) {
- BlockingMutexLock l(&mtx_);
+ ThreadRegistryLock l(this);
if (total)
*total = threads_.size();
if (running) *running = running_threads_;
}
uptr ThreadRegistry::GetMaxAliveThreads() {
- BlockingMutexLock l(&mtx_);
+ ThreadRegistryLock l(this);
return max_alive_threads_;
}
u32 ThreadRegistry::CreateThread(uptr user_id, bool detached, u32 parent_tid,
void *arg) {
- BlockingMutexLock l(&mtx_);
+ ThreadRegistryLock l(this);
u32 tid = kInvalidTid;
ThreadContextBase *tctx = QuarantinePop();
if (tctx) {
max_alive_threads_++;
CHECK_EQ(alive_threads_, max_alive_threads_);
}
+ if (user_id) {
+ // Ensure that user_id is unique. If it's not the case we are screwed.
+ // Ignoring this situation may lead to very hard to debug false
+ // positives later (e.g. if we join a wrong thread).
+ CHECK(live_.try_emplace(user_id, tid).second);
+ }
tctx->SetCreated(user_id, total_threads_++, detached,
parent_tid, arg);
return tid;
}
u32 ThreadRegistry::FindThread(FindThreadCallback cb, void *arg) {
- BlockingMutexLock l(&mtx_);
+ ThreadRegistryLock l(this);
for (u32 tid = 0; tid < threads_.size(); tid++) {
ThreadContextBase *tctx = threads_[tid];
if (tctx != 0 && cb(tctx, arg))
}
void ThreadRegistry::SetThreadName(u32 tid, const char *name) {
- BlockingMutexLock l(&mtx_);
+ ThreadRegistryLock l(this);
ThreadContextBase *tctx = threads_[tid];
CHECK_NE(tctx, 0);
CHECK_EQ(SANITIZER_FUCHSIA ? ThreadStatusCreated : ThreadStatusRunning,
}
void ThreadRegistry::SetThreadNameByUserId(uptr user_id, const char *name) {
- BlockingMutexLock l(&mtx_);
- for (u32 tid = 0; tid < threads_.size(); tid++) {
- ThreadContextBase *tctx = threads_[tid];
- if (tctx != 0 && tctx->user_id == user_id &&
- tctx->status != ThreadStatusInvalid) {
- tctx->SetName(name);
- return;
- }
- }
+ ThreadRegistryLock l(this);
+ if (const auto *tid = live_.find(user_id))
+ threads_[tid->second]->SetName(name);
}
void ThreadRegistry::DetachThread(u32 tid, void *arg) {
- BlockingMutexLock l(&mtx_);
+ ThreadRegistryLock l(this);
ThreadContextBase *tctx = threads_[tid];
CHECK_NE(tctx, 0);
if (tctx->status == ThreadStatusInvalid) {
}
tctx->OnDetached(arg);
if (tctx->status == ThreadStatusFinished) {
+ if (tctx->user_id)
+ live_.erase(tctx->user_id);
tctx->SetDead();
QuarantinePush(tctx);
} else {
bool destroyed = false;
do {
{
- BlockingMutexLock l(&mtx_);
+ ThreadRegistryLock l(this);
ThreadContextBase *tctx = threads_[tid];
CHECK_NE(tctx, 0);
if (tctx->status == ThreadStatusInvalid) {
return;
}
if ((destroyed = tctx->GetDestroyed())) {
+ if (tctx->user_id)
+ live_.erase(tctx->user_id);
tctx->SetJoined(arg);
QuarantinePush(tctx);
}
// thread before trying to create it, and then failed to actually
// create it, and so never called StartThread.
ThreadStatus ThreadRegistry::FinishThread(u32 tid) {
- BlockingMutexLock l(&mtx_);
+ ThreadRegistryLock l(this);
CHECK_GT(alive_threads_, 0);
alive_threads_--;
ThreadContextBase *tctx = threads_[tid];
}
tctx->SetFinished();
if (dead) {
+ if (tctx->user_id)
+ live_.erase(tctx->user_id);
tctx->SetDead();
QuarantinePush(tctx);
}
void ThreadRegistry::StartThread(u32 tid, tid_t os_id, ThreadType thread_type,
void *arg) {
- BlockingMutexLock l(&mtx_);
+ ThreadRegistryLock l(this);
running_threads_++;
ThreadContextBase *tctx = threads_[tid];
CHECK_NE(tctx, 0);
return tctx;
}
+u32 ThreadRegistry::ConsumeThreadUserId(uptr user_id) {
+ ThreadRegistryLock l(this);
+ u32 tid;
+ auto *t = live_.find(user_id);
+ CHECK(t);
+ tid = t->second;
+ live_.erase(t);
+ auto *tctx = threads_[tid];
+ CHECK_EQ(tctx->user_id, user_id);
+ tctx->user_id = 0;
+ return tid;
+}
+
void ThreadRegistry::SetThreadUserId(u32 tid, uptr user_id) {
- BlockingMutexLock l(&mtx_);
+ ThreadRegistryLock l(this);
ThreadContextBase *tctx = threads_[tid];
CHECK_NE(tctx, 0);
CHECK_NE(tctx->status, ThreadStatusInvalid);
CHECK_NE(tctx->status, ThreadStatusDead);
CHECK_EQ(tctx->user_id, 0);
tctx->user_id = user_id;
+ CHECK(live_.try_emplace(user_id, tctx->tid).second);
+}
+
+u32 ThreadRegistry::OnFork(u32 tid) {
+ ThreadRegistryLock l(this);
+ // We only purge user_id (pthread_t) of live threads because
+ // they cause CHECK failures if new threads with matching pthread_t
+ // created after fork.
+ // Potentially we could purge more info (ThreadContextBase themselves),
+ // but it's hard to test and easy to introduce new issues by doing this.
+ for (auto *tctx : threads_) {
+ if (tctx->tid == tid || !tctx->user_id)
+ continue;
+ CHECK(live_.erase(tctx->user_id));
+ tctx->user_id = 0;
+ }
+ return alive_threads_;
}
} // namespace __sanitizer
#define SANITIZER_THREAD_REGISTRY_H
#include "sanitizer_common.h"
+#include "sanitizer_dense_map.h"
#include "sanitizer_list.h"
#include "sanitizer_mutex.h"
typedef ThreadContextBase* (*ThreadContextFactory)(u32 tid);
-class MUTEX ThreadRegistry {
+class SANITIZER_MUTEX ThreadRegistry {
public:
ThreadRegistry(ThreadContextFactory factory);
ThreadRegistry(ThreadContextFactory factory, u32 max_threads,
uptr *alive = nullptr);
uptr GetMaxAliveThreads();
- void Lock() ACQUIRE() { mtx_.Lock(); }
- void CheckLocked() const CHECK_LOCKED() { mtx_.CheckLocked(); }
- void Unlock() RELEASE() { mtx_.Unlock(); }
+ void Lock() SANITIZER_ACQUIRE() { mtx_.Lock(); }
+ void CheckLocked() const SANITIZER_CHECK_LOCKED() { mtx_.CheckLocked(); }
+ void Unlock() SANITIZER_RELEASE() { mtx_.Unlock(); }
// Should be guarded by ThreadRegistryLock.
ThreadContextBase *GetThreadLocked(u32 tid) {
return threads_.empty() ? nullptr : threads_[tid];
}
+ u32 NumThreadsLocked() const { return threads_.size(); }
+
u32 CreateThread(uptr user_id, bool detached, u32 parent_tid, void *arg);
typedef void (*ThreadCallback)(ThreadContextBase *tctx, void *arg);
// Finishes thread and returns previous status.
ThreadStatus FinishThread(u32 tid);
void StartThread(u32 tid, tid_t os_id, ThreadType thread_type, void *arg);
+ u32 ConsumeThreadUserId(uptr user_id);
void SetThreadUserId(u32 tid, uptr user_id);
+ // OnFork must be called in the child process after fork to purge old
+ // threads that don't exist anymore (except for the current thread tid).
+ // Returns number of alive threads before fork.
+ u32 OnFork(u32 tid);
+
private:
const ThreadContextFactory context_factory_;
const u32 max_threads_;
const u32 thread_quarantine_size_;
const u32 max_reuse_;
- BlockingMutex mtx_;
+ Mutex mtx_;
u64 total_threads_; // Total number of created threads. May be greater than
// max_threads_ if contexts were reused.
InternalMmapVector<ThreadContextBase *> threads_;
IntrusiveList<ThreadContextBase> dead_threads_;
IntrusiveList<ThreadContextBase> invalid_threads_;
+ DenseMap<uptr, Tid> live_;
void QuarantinePush(ThreadContextBase *tctx);
ThreadContextBase *QuarantinePop();
#define SANITIZER_THREAD_SAFETY_H
#if defined(__clang__)
-# define THREAD_ANNOTATION(x) __attribute__((x))
+# define SANITIZER_THREAD_ANNOTATION(x) __attribute__((x))
#else
-# define THREAD_ANNOTATION(x)
+# define SANITIZER_THREAD_ANNOTATION(x)
#endif
-#define MUTEX THREAD_ANNOTATION(capability("mutex"))
-#define SCOPED_LOCK THREAD_ANNOTATION(scoped_lockable)
-#define GUARDED_BY(x) THREAD_ANNOTATION(guarded_by(x))
-#define PT_GUARDED_BY(x) THREAD_ANNOTATION(pt_guarded_by(x))
-#define REQUIRES(...) THREAD_ANNOTATION(requires_capability(__VA_ARGS__))
-#define REQUIRES_SHARED(...) \
- THREAD_ANNOTATION(requires_shared_capability(__VA_ARGS__))
-#define ACQUIRE(...) THREAD_ANNOTATION(acquire_capability(__VA_ARGS__))
-#define ACQUIRE_SHARED(...) \
- THREAD_ANNOTATION(acquire_shared_capability(__VA_ARGS__))
-#define TRY_ACQUIRE(...) THREAD_ANNOTATION(try_acquire_capability(__VA_ARGS__))
-#define RELEASE(...) THREAD_ANNOTATION(release_capability(__VA_ARGS__))
-#define RELEASE_SHARED(...) \
- THREAD_ANNOTATION(release_shared_capability(__VA_ARGS__))
-#define EXCLUDES(...) THREAD_ANNOTATION(locks_excluded(__VA_ARGS__))
-#define CHECK_LOCKED(...) THREAD_ANNOTATION(assert_capability(__VA_ARGS__))
-#define NO_THREAD_SAFETY_ANALYSIS THREAD_ANNOTATION(no_thread_safety_analysis)
+#define SANITIZER_MUTEX SANITIZER_THREAD_ANNOTATION(capability("mutex"))
+#define SANITIZER_SCOPED_LOCK SANITIZER_THREAD_ANNOTATION(scoped_lockable)
+#define SANITIZER_GUARDED_BY(x) SANITIZER_THREAD_ANNOTATION(guarded_by(x))
+#define SANITIZER_PT_GUARDED_BY(x) SANITIZER_THREAD_ANNOTATION(pt_guarded_by(x))
+#define SANITIZER_REQUIRES(...) \
+ SANITIZER_THREAD_ANNOTATION(requires_capability(__VA_ARGS__))
+#define SANITIZER_REQUIRES_SHARED(...) \
+ SANITIZER_THREAD_ANNOTATION(requires_shared_capability(__VA_ARGS__))
+#define SANITIZER_ACQUIRE(...) \
+ SANITIZER_THREAD_ANNOTATION(acquire_capability(__VA_ARGS__))
+#define SANITIZER_ACQUIRE_SHARED(...) \
+ SANITIZER_THREAD_ANNOTATION(acquire_shared_capability(__VA_ARGS__))
+#define SANITIZER_TRY_ACQUIRE(...) \
+ SANITIZER_THREAD_ANNOTATION(try_acquire_capability(__VA_ARGS__))
+#define SANITIZER_RELEASE(...) \
+ SANITIZER_THREAD_ANNOTATION(release_capability(__VA_ARGS__))
+#define SANITIZER_RELEASE_SHARED(...) \
+ SANITIZER_THREAD_ANNOTATION(release_shared_capability(__VA_ARGS__))
+#define SANITIZER_EXCLUDES(...) \
+ SANITIZER_THREAD_ANNOTATION(locks_excluded(__VA_ARGS__))
+#define SANITIZER_CHECK_LOCKED(...) \
+ SANITIZER_THREAD_ANNOTATION(assert_capability(__VA_ARGS__))
+#define SANITIZER_NO_THREAD_SAFETY_ANALYSIS \
+ SANITIZER_THREAD_ANNOTATION(no_thread_safety_analysis)
#endif
static const uptr kDestroyedThread = -1;
static void DTLS_Deallocate(DTLS::DTVBlock *block) {
- VReport(2, "__tls_get_addr: DTLS_Deallocate %p %zd\n", block);
+ VReport(2, "__tls_get_addr: DTLS_Deallocate %p\n", (void *)block);
UnmapOrDie(block, sizeof(DTLS::DTVBlock));
atomic_fetch_sub(&number_of_live_dtls, 1, memory_order_relaxed);
}
}
uptr num_live_dtls =
atomic_fetch_add(&number_of_live_dtls, 1, memory_order_relaxed);
- VReport(2, "__tls_get_addr: DTLS_NextBlock %p %zd\n", &dtls, num_live_dtls);
+ VReport(2, "__tls_get_addr: DTLS_NextBlock %p %zd\n", (void *)&dtls,
+ num_live_dtls);
return new_dtv;
}
static DTLS::DTV *DTLS_Find(uptr id) {
- VReport(2, "__tls_get_addr: DTLS_Find %p %zd\n", &dtls, id);
+ VReport(2, "__tls_get_addr: DTLS_Find %p %zd\n", (void *)&dtls, id);
static constexpr uptr kPerBlock = ARRAY_SIZE(DTLS::DTVBlock::dtvs);
DTLS::DTVBlock *cur = DTLS_NextBlock(&dtls.dtv_block);
if (!cur)
void DTLS_Destroy() {
if (!common_flags()->intercept_tls_get_addr) return;
- VReport(2, "__tls_get_addr: DTLS_Destroy %p\n", &dtls);
+ VReport(2, "__tls_get_addr: DTLS_Destroy %p\n", (void *)&dtls);
DTLS::DTVBlock *block = (DTLS::DTVBlock *)atomic_exchange(
&dtls.dtv_block, kDestroyedThread, memory_order_release);
while (block) {
return 0;
uptr tls_size = 0;
uptr tls_beg = reinterpret_cast<uptr>(res) - arg->offset - kDtvOffset;
- VReport(2, "__tls_get_addr: %p {%p,%p} => %p; tls_beg: %p; sp: %p "
- "num_live_dtls %zd\n",
- arg, arg->dso_id, arg->offset, res, tls_beg, &tls_beg,
+ VReport(2,
+ "__tls_get_addr: %p {0x%zx,0x%zx} => %p; tls_beg: 0x%zx; sp: %p "
+ "num_live_dtls %zd\n",
+ (void *)arg, arg->dso_id, arg->offset, res, tls_beg, (void *)&tls_beg,
atomic_load(&number_of_live_dtls, memory_order_relaxed));
if (dtls.last_memalign_ptr == tls_beg) {
tls_size = dtls.last_memalign_size;
- VReport(2, "__tls_get_addr: glibc <=2.18 suspected; tls={%p,%p}\n",
- tls_beg, tls_size);
+ VReport(2, "__tls_get_addr: glibc <=2.18 suspected; tls={0x%zx,0x%zx}\n",
+ tls_beg, tls_size);
} else if (tls_beg >= static_tls_begin && tls_beg < static_tls_end) {
// This is the static TLS block which was initialized / unpoisoned at thread
// creation.
- VReport(2, "__tls_get_addr: static tls: %p\n", tls_beg);
+ VReport(2, "__tls_get_addr: static tls: 0x%zx\n", tls_beg);
tls_size = 0;
} else if ((tls_beg % 4096) == sizeof(Glibc_2_19_tls_header)) {
// We may want to check gnu_get_libc_version().
Glibc_2_19_tls_header *header = (Glibc_2_19_tls_header *)tls_beg - 1;
tls_size = header->size;
tls_beg = header->start;
- VReport(2, "__tls_get_addr: glibc >=2.19 suspected; tls={%p %p}\n",
- tls_beg, tls_size);
+ VReport(2, "__tls_get_addr: glibc >=2.19 suspected; tls={0x%zx 0x%zx}\n",
+ tls_beg, tls_size);
} else {
VReport(2, "__tls_get_addr: Can't guess glibc version\n");
// This may happen inside the DTOR of main thread, so just ignore it.
void DTLS_on_libc_memalign(void *ptr, uptr size) {
if (!common_flags()->intercept_tls_get_addr) return;
- VReport(2, "DTLS_on_libc_memalign: %p %p\n", ptr, size);
+ VReport(2, "DTLS_on_libc_memalign: %p 0x%zx\n", ptr, size);
dtls.last_memalign_ptr = reinterpret_cast<uptr>(ptr);
dtls.last_memalign_size = size;
}
#ifndef SANITIZER_TYPE_TRAITS_H
#define SANITIZER_TYPE_TRAITS_H
+#include "sanitizer_common/sanitizer_internal_defs.h"
+
namespace __sanitizer {
struct true_type {
using type = F;
};
+template <class T>
+struct remove_reference {
+ using type = T;
+};
+template <class T>
+struct remove_reference<T&> {
+ using type = T;
+};
+template <class T>
+struct remove_reference<T&&> {
+ using type = T;
+};
+
+template <class T>
+WARN_UNUSED_RESULT inline typename remove_reference<T>::type&& move(T&& t) {
+ return static_cast<typename remove_reference<T>::type&&>(t);
+}
+
+template <class T>
+WARN_UNUSED_RESULT inline constexpr T&& forward(
+ typename remove_reference<T>::type& t) {
+ return static_cast<T&&>(t);
+}
+
+template <class T>
+WARN_UNUSED_RESULT inline constexpr T&& forward(
+ typename remove_reference<T>::type&& t) {
+ return static_cast<T&&>(t);
+}
+
+template <class T, T v>
+struct integral_constant {
+ static constexpr const T value = v;
+ typedef T value_type;
+ typedef integral_constant type;
+ constexpr operator value_type() const { return value; }
+ constexpr value_type operator()() const { return value; }
+};
+
+#ifndef __has_builtin
+# define __has_builtin(x) 0
+#endif
+
+#if __has_builtin(__is_trivially_destructible)
+
+template <class T>
+struct is_trivially_destructible
+ : public integral_constant<bool, __is_trivially_destructible(T)> {};
+
+#elif __has_builtin(__has_trivial_destructor)
+
+template <class T>
+struct is_trivially_destructible
+ : public integral_constant<bool, __has_trivial_destructor(T)> {};
+
+#else
+
+template <class T>
+struct is_trivially_destructible
+ : public integral_constant<bool, /* less efficient fallback */ false> {};
+
+#endif
+
+#if __has_builtin(__is_trivially_copyable)
+
+template <class T>
+struct is_trivially_copyable
+ : public integral_constant<bool, __is_trivially_copyable(T)> {};
+
+#else
+
+template <class T>
+struct is_trivially_copyable
+ : public integral_constant<bool, /* less efficient fallback */ false> {};
+
+#endif
+
} // namespace __sanitizer
#endif
#endif
uptr Unwind_GetIP(struct _Unwind_Context *ctx) {
-#if defined(__arm__) && !SANITIZER_MAC
+#if defined(__arm__) && !SANITIZER_APPLE
uptr val;
_Unwind_VRS_Result res = _Unwind_VRS_Get(ctx, _UVRSC_CORE,
15 /* r15 = PC */, _UVRSD_UINT32, &val);
InitializeDbgHelpIfNeeded();
size = 0;
-#if defined(_WIN64)
+# if SANITIZER_WINDOWS64
+# if SANITIZER_ARM64
+ int machine_type = IMAGE_FILE_MACHINE_ARM64;
+ stack_frame.AddrPC.Offset = ctx.Pc;
+ stack_frame.AddrFrame.Offset = ctx.Fp;
+ stack_frame.AddrStack.Offset = ctx.Sp;
+# else
int machine_type = IMAGE_FILE_MACHINE_AMD64;
stack_frame.AddrPC.Offset = ctx.Rip;
stack_frame.AddrFrame.Offset = ctx.Rbp;
stack_frame.AddrStack.Offset = ctx.Rsp;
-#else
+# endif
+# else
int machine_type = IMAGE_FILE_MACHINE_I386;
stack_frame.AddrPC.Offset = ctx.Eip;
stack_frame.AddrFrame.Offset = ctx.Ebp;
stack_frame.AddrStack.Offset = ctx.Esp;
-#endif
+# endif
stack_frame.AddrPC.Mode = AddrModeFlat;
stack_frame.AddrFrame.Mode = AddrModeFlat;
stack_frame.AddrStack.Mode = AddrModeFlat;
while (StackWalk64(machine_type, GetCurrentProcess(), GetCurrentThread(),
- &stack_frame, &ctx, NULL, SymFunctionTableAccess64,
- SymGetModuleBase64, NULL) &&
- size < Min(max_depth, kStackTraceMax)) {
+ &stack_frame, &ctx, NULL, SymFunctionTableAccess64,
+ SymGetModuleBase64, NULL) &&
+ size < Min(max_depth, kStackTraceMax)) {
trace_buffer[size++] = (uptr)stack_frame.AddrPC.Offset;
}
}
-#ifdef __clang__
-#pragma clang diagnostic pop
-#endif
-#endif // #if !SANITIZER_GO
+# ifdef __clang__
+# pragma clang diagnostic pop
+# endif
+# endif // #if !SANITIZER_GO
#endif // SANITIZER_WINDOWS
}
EnsureSize(size);
if (old_size < size) {
- for (uptr i = old_size; i < size; i++)
- internal_memset(&begin_[i], 0, sizeof(begin_[i]));
+ internal_memset(&begin_[old_size], 0,
+ sizeof(begin_[old_size]) * (size - old_size));
}
}
return ::GetFileAttributesA(filename) != INVALID_FILE_ATTRIBUTES;
}
+bool DirExists(const char *path) {
+ auto attr = ::GetFileAttributesA(path);
+ return (attr != INVALID_FILE_ATTRIBUTES) && (attr & FILE_ATTRIBUTE_DIRECTORY);
+}
+
uptr internal_getpid() {
return GetProcessId(GetCurrentProcess());
}
}
#endif // #if !SANITIZER_GO
+bool ErrorIsOOM(error_t err) {
+ // TODO: This should check which `err`s correspond to OOM.
+ return false;
+}
+
void *MmapOrDie(uptr size, const char *mem_type, bool raw_report) {
void *rv = VirtualAlloc(0, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
if (rv == 0)
return (void *)mapped_addr;
}
+// ZeroMmapFixedRegion zero's out a region of memory previously returned from a
+// call to one of the MmapFixed* helpers. On non-windows systems this would be
+// done with another mmap, but on windows remapping is not an option.
+// VirtualFree(DECOMMIT)+VirtualAlloc(RECOMMIT) would also be a way to zero the
+// memory, but we can't do this atomically, so instead we fall back to using
+// internal_memset.
+bool ZeroMmapFixedRegion(uptr fixed_addr, uptr size) {
+ internal_memset((void*) fixed_addr, 0, size);
+ return true;
+}
+
bool MmapFixedNoReserve(uptr fixed_addr, uptr size, const char *name) {
// FIXME: is this really "NoReserve"? On Win32 this does not matter much,
// but on Win64 it does.
return VirtualProtect((LPVOID)addr, size, PAGE_NOACCESS, &old_protection);
}
+bool MprotectReadOnly(uptr addr, uptr size) {
+ DWORD old_protection;
+ return VirtualProtect((LPVOID)addr, size, PAGE_READONLY, &old_protection);
+}
+
void ReleaseMemoryPagesToOS(uptr beg, uptr end) {
uptr beg_aligned = RoundDownTo(beg, GetPageSizeCached()),
end_aligned = RoundDownTo(end, GetPageSizeCached());
UNIMPLEMENTED();
}
-void PlatformPrepareForSandboxing(__sanitizer_sandbox_arguments *args) {}
+void PlatformPrepareForSandboxing(void *args) {}
bool StackSizeIsUnlimited() {
UNIMPLEMENTED();
internal__exit(3);
}
+bool CreateDir(const char *pathname) {
+ return CreateDirectoryA(pathname, nullptr) != 0;
+}
+
#if !SANITIZER_GO
// Read the file to extract the ImageBase field from the PE header. If ASLR is
// disabled and this virtual address is available, the loader will typically
WakeByAddressAll(p);
}
-// ---------------------- BlockingMutex ---------------- {{{1
-
-BlockingMutex::BlockingMutex() {
- CHECK(sizeof(SRWLOCK) <= sizeof(opaque_storage_));
- internal_memset(this, 0, sizeof(*this));
-}
-
-void BlockingMutex::Lock() {
- AcquireSRWLockExclusive((PSRWLOCK)opaque_storage_);
- CHECK_EQ(owner_, 0);
- owner_ = GetThreadSelf();
-}
-
-void BlockingMutex::Unlock() {
- CheckLocked();
- owner_ = 0;
- ReleaseSRWLockExclusive((PSRWLOCK)opaque_storage_);
-}
-
-void BlockingMutex::CheckLocked() const { CHECK_EQ(owner_, GetThreadSelf()); }
-
uptr GetTlsSize() {
return 0;
}
CONTEXT *context_record = (CONTEXT *)context;
pc = (uptr)exception_record->ExceptionAddress;
-#ifdef _WIN64
+# if SANITIZER_WINDOWS64
+# if SANITIZER_ARM64
+ bp = (uptr)context_record->Fp;
+ sp = (uptr)context_record->Sp;
+# else
bp = (uptr)context_record->Rbp;
sp = (uptr)context_record->Rsp;
-#else
+# endif
+# else
bp = (uptr)context_record->Ebp;
sp = (uptr)context_record->Esp;
-#endif
+# endif
}
uptr SignalContext::GetAddress() const {
// The write flag is only available for access violation exceptions.
if (exception_record->ExceptionCode != EXCEPTION_ACCESS_VIOLATION)
- return SignalContext::UNKNOWN;
+ return SignalContext::Unknown;
// The contents of this array are documented at
// https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-exception_record
// second element is the faulting address.
switch (exception_record->ExceptionInformation[0]) {
case 0:
- return SignalContext::READ;
+ return SignalContext::Read;
case 1:
- return SignalContext::WRITE;
+ return SignalContext::Write;
case 8:
- return SignalContext::UNKNOWN;
+ return SignalContext::Unknown;
}
- return SignalContext::UNKNOWN;
+ return SignalContext::Unknown;
}
void SignalContext::DumpAllRegisters(void *context) {
// Do nothing.
}
-void MaybeReexec() {
- // No need to re-exec on Windows.
-}
-
void CheckASLR() {
// Do nothing
}
int WaitForProcess(pid_t pid) { return -1; }
// FIXME implement on this platform.
-void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size) { }
+void GetMemoryProfile(fill_profile_f cb, uptr *stats) {}
void CheckNoDeepBind(const char *filename, int flag) {
// Do nothing.
//
//===----------------------------------------------------------------------===//
+#include <inttypes.h>
#include <stdio.h>
#include <string>
#include "llvm/DebugInfo/Symbolize/DIPrinter.h"
#include "llvm/DebugInfo/Symbolize/Symbolize.h"
+static llvm::symbolize::LLVMSymbolizer *Symbolizer = nullptr;
+static bool Demangle = true;
+static bool InlineFrames = true;
+
static llvm::symbolize::LLVMSymbolizer *getDefaultSymbolizer() {
- static llvm::symbolize::LLVMSymbolizer *DefaultSymbolizer =
- new llvm::symbolize::LLVMSymbolizer();
- return DefaultSymbolizer;
+ if (Symbolizer)
+ return Symbolizer;
+ llvm::symbolize::LLVMSymbolizer::Options Opts;
+ Opts.Demangle = Demangle;
+ Symbolizer = new llvm::symbolize::LLVMSymbolizer(Opts);
+ return Symbolizer;
}
static llvm::symbolize::PrinterConfig getDefaultPrinterConfig() {
}
namespace __sanitizer {
-int internal_snprintf(char *buffer, unsigned long length, const char *format,
+int internal_snprintf(char *buffer, uintptr_t length, const char *format,
...);
} // namespace __sanitizer
typedef uint64_t u64;
bool __sanitizer_symbolize_code(const char *ModuleName, uint64_t ModuleOffset,
- char *Buffer, int MaxLength,
- bool SymbolizeInlineFrames) {
+ char *Buffer, int MaxLength) {
std::string Result;
{
llvm::raw_string_ostream OS(Result);
// TODO: it is neccessary to set proper SectionIndex here.
// object::SectionedAddress::UndefSection works for only absolute addresses.
- if (SymbolizeInlineFrames) {
+ if (InlineFrames) {
auto ResOrErr = getDefaultSymbolizer()->symbolizeInlinedCode(
ModuleName,
{ModuleOffset, llvm::object::SectionedAddress::UndefSection});
Result.c_str()) < MaxLength;
}
-void __sanitizer_symbolize_flush() { getDefaultSymbolizer()->flush(); }
+void __sanitizer_symbolize_flush() {
+ if (Symbolizer)
+ Symbolizer->flush();
+}
int __sanitizer_symbolize_demangle(const char *Name, char *Buffer,
int MaxLength) {
: 0;
}
+bool __sanitizer_symbolize_set_demangle(bool Value) {
+ // Must be called before LLVMSymbolizer created.
+ if (Symbolizer)
+ return false;
+ Demangle = Value;
+ return true;
+}
+
+bool __sanitizer_symbolize_set_inline_frames(bool Value) {
+ InlineFrames = Value;
+ return true;
+}
+
// Override __cxa_atexit and ignore callbacks.
// This prevents crashes in a configuration when the symbolizer
// is built into sanitizer runtime and consequently into the test process.
#include <dlfcn.h>
#include <errno.h>
#include <fcntl.h>
+#include <inttypes.h>
#include <stdarg.h>
#include <stdio.h>
#include <unistd.h>
unsigned long internal_lstat(const char *path, void *buf);
unsigned long internal_fstat(int fd, void *buf);
size_t internal_strlen(const char *s);
-unsigned long internal_mmap(void *addr, unsigned long length, int prot,
- int flags, int fd, unsigned long long offset);
+unsigned long internal_mmap(void *addr, uintptr_t length, int prot, int flags,
+ int fd, unsigned long long offset);
void *internal_memcpy(void *dest, const void *src, unsigned long n);
// Used to propagate errno.
-bool internal_iserror(unsigned long retval, int *rverrno = 0);
+bool internal_iserror(uintptr_t retval, int *rverrno = 0);
} // namespace __sanitizer
namespace {
void *mmap(void *addr, size_t length, int prot, int flags, int fd,
off_t offset) {
- unsigned long res = __sanitizer::internal_mmap(
- addr, (unsigned long)length, prot, flags, fd, (unsigned long long)offset);
+ unsigned long res =
+ __sanitizer::internal_mmap(addr, length, prot, flags, fd, offset);
RETURN_OR_SET_ERRNO(void *, res);
}
-#!/bin/bash -eu
+#!/usr/bin/env bash
#
# Run as: CLANG=bin/clang ZLIB_SRC=src/zlib \
# build_symbolizer.sh runtime_build/lib/clang/4.0.0/lib/linux/
COMPILER_RT_SRC=$(readlink -f ${SCRIPT_DIR}/../../../..)
LLVM_SRC=${LLVM_SRC:-${COMPILER_RT_SRC}/../llvm}
LLVM_SRC=$(readlink -f $LLVM_SRC)
-if [[ ! -d "${LLVM_SRC}/../llvm" ]] ; then
- LLVM_SRC=$(readlink -f ${COMPILER_RT_SRC}/../../../llvm)
-fi
-LIBCXX_SRC=$(readlink -f ${COMPILER_RT_SRC}/../libcxx)
-LIBCXXABI_SRC=$(readlink -f ${COMPILER_RT_SRC}/../libcxxabi)
-
-if [[ ! -d "${LLVM_SRC}/../llvm" ||
- ! -d "${LIBCXX_SRC}" ||
- ! -d "${LIBCXXABI_SRC}" ]]; then
- echo "Missing or incomplete LLVM_SRC"
- exit 1
-fi
if [[ "$ZLIB_SRC" == "" ||
! -x "${ZLIB_SRC}/configure" ||
fi
ZLIB_SRC=$(readlink -f $ZLIB_SRC)
-J="${J:-50}"
-
CLANG="${CLANG:-`which clang`}"
CLANG_DIR=$(readlink -f $(dirname "$CLANG"))
CXX=$CLANG_DIR/clang++
TBLGEN=$CLANG_DIR/llvm-tblgen
OPT=$CLANG_DIR/opt
-export AR=$CLANG_DIR/llvm-ar
-export LINK=$CLANG_DIR/llvm-link
+AR=$CLANG_DIR/llvm-ar
+LINK=$CLANG_DIR/llvm-link
for F in $CC $CXX $TBLGEN $LINK $OPT $AR; do
if [[ ! -x "$F" ]]; then
SYMBOLIZER_BUILD=${BUILD_DIR}/symbolizer
FLAGS=${FLAGS:-}
-FLAGS="$FLAGS -fPIC -flto -Os -g0 -DNDEBUG"
+TARGET_TRIPLE=$($CC -print-target-triple $FLAGS)
+if [[ "$FLAGS" =~ "-m32" ]] ; then
+ # Avoid new wrappers.
+ FLAGS+=" -U_FILE_OFFSET_BITS"
+fi
+FLAGS+=" -fPIC -flto -Oz -g0 -DNDEBUG -target $TARGET_TRIPLE -Wno-unused-command-line-argument"
+LINKFLAGS="-fuse-ld=lld -target $TARGET_TRIPLE"
# Build zlib.
mkdir -p ${ZLIB_BUILD}
cd ${ZLIB_BUILD}
cp -r ${ZLIB_SRC}/* .
-CC=$CC CFLAGS="$FLAGS" RANLIB=/bin/true ./configure --static
-make -j${J} libz.a
+AR="${AR}" CC="${CC}" CFLAGS="$FLAGS -Wno-deprecated-non-prototype" RANLIB=/bin/true ./configure --static
+make -j libz.a
# Build and install libcxxabi and libcxx.
if [[ ! -d ${LIBCXX_BUILD} ]]; then
mkdir -p ${LIBCXX_BUILD}
cd ${LIBCXX_BUILD}
LIBCXX_FLAGS="${FLAGS} -Wno-macro-redefined"
- PROJECTS=
- if [[ ! -d $LLVM_SRC/projects/libcxxabi ]] ; then
- PROJECTS="-DLLVM_ENABLE_PROJECTS='libcxx;libcxxabi'"
- fi
cmake -GNinja \
- ${PROJECTS} \
+ -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi" \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_C_COMPILER=$CC \
-DCMAKE_CXX_COMPILER=$CXX \
-DCMAKE_CXX_FLAGS_RELEASE="${LIBCXX_FLAGS}" \
-DLIBCXXABI_ENABLE_ASSERTIONS=OFF \
-DLIBCXXABI_ENABLE_EXCEPTIONS=OFF \
- -DLIBCXXABI_ENABLE_SHARED=OFF \
-DLIBCXX_ENABLE_ASSERTIONS=OFF \
-DLIBCXX_ENABLE_EXCEPTIONS=OFF \
-DLIBCXX_ENABLE_RTTI=OFF \
- -DLIBCXX_ENABLE_SHARED=OFF \
- $LLVM_SRC
+ -DCMAKE_SHARED_LINKER_FLAGS="$LINKFLAGS" \
+ $LLVM_SRC/../runtimes
fi
cd ${LIBCXX_BUILD}
ninja cxx cxxabi
FLAGS="${FLAGS} -fno-rtti -fno-exceptions"
-LLVM_FLAGS="${FLAGS} -nostdinc++ -I${ZLIB_BUILD} -I${LIBCXX_BUILD}/include/c++/v1 -Wno-error=global-constructors"
+LLVM_CFLAGS="${FLAGS} -Wno-global-constructors"
+LLVM_CXXFLAGS="${LLVM_CFLAGS} -nostdinc++ -I${ZLIB_BUILD} -isystem ${LIBCXX_BUILD}/include -isystem ${LIBCXX_BUILD}/include/c++/v1"
# Build LLVM.
if [[ ! -d ${LLVM_BUILD} ]]; then
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_C_COMPILER=$CC \
-DCMAKE_CXX_COMPILER=$CXX \
- -DCMAKE_C_FLAGS_RELEASE="${LLVM_FLAGS}" \
- -DCMAKE_CXX_FLAGS_RELEASE="${LLVM_FLAGS}" \
+ -DCMAKE_C_FLAGS="${LLVM_CFLAGS}" \
+ -DCMAKE_CXX_FLAGS="${LLVM_CXXFLAGS}" \
+ -DCMAKE_EXE_LINKER_FLAGS="$LINKFLAGS -stdlib=libc++ -L${LIBCXX_BUILD}/lib" \
-DLLVM_TABLEGEN=$TBLGEN \
-DLLVM_ENABLE_ZLIB=ON \
-DLLVM_ENABLE_TERMINFO=OFF \
$LLVM_SRC
fi
cd ${LLVM_BUILD}
-ninja LLVMSymbolize LLVMObject LLVMBinaryFormat LLVMDebugInfoDWARF LLVMSupport LLVMDebugInfoPDB LLVMMC LLVMDemangle LLVMTextAPI
+ninja LLVMSymbolize LLVMObject LLVMBinaryFormat LLVMDebugInfoDWARF LLVMSupport LLVMDebugInfoPDB LLVMDebuginfod LLVMMC LLVMDemangle LLVMTextAPI LLVMTargetParser
cd ${BUILD_DIR}
rm -rf ${SYMBOLIZER_BUILD}
cd ${SYMBOLIZER_BUILD}
echo "Compiling..."
-SYMBOLIZER_FLAGS="$LLVM_FLAGS -I${LLVM_SRC}/include -I${LLVM_BUILD}/include -std=c++14"
+SYMBOLIZER_FLAGS="$LLVM_CXXFLAGS -I${LLVM_SRC}/include -I${LLVM_BUILD}/include -std=c++17"
$CXX $SYMBOLIZER_FLAGS ${SRC_DIR}/sanitizer_symbolize.cpp ${SRC_DIR}/sanitizer_wrappers.cpp -c
$AR rc symbolizer.a sanitizer_symbolize.o sanitizer_wrappers.o
-SYMBOLIZER_API_LIST=__sanitizer_symbolize_code,__sanitizer_symbolize_data,__sanitizer_symbolize_flush,__sanitizer_symbolize_demangle
+SYMBOLIZER_API_LIST=__sanitizer_symbolize_code
+SYMBOLIZER_API_LIST+=,__sanitizer_symbolize_data
+SYMBOLIZER_API_LIST+=,__sanitizer_symbolize_flush
+SYMBOLIZER_API_LIST+=,__sanitizer_symbolize_demangle
+SYMBOLIZER_API_LIST+=,__sanitizer_symbolize_set_demangle
+SYMBOLIZER_API_LIST+=,__sanitizer_symbolize_set_inline_frames
+
+LIBCXX_ARCHIVE_DIR=$(dirname $(find $LIBCXX_BUILD -name libc++.a | head -n1))
# Merge all the object files together and copy the resulting library back.
-$SCRIPT_DIR/ar_to_bc.sh $LIBCXX_BUILD/lib/libc++.a \
- $LIBCXX_BUILD/lib/libc++abi.a \
- $LLVM_BUILD/lib/libLLVMSymbolize.a \
- $LLVM_BUILD/lib/libLLVMObject.a \
- $LLVM_BUILD/lib/libLLVMBinaryFormat.a \
- $LLVM_BUILD/lib/libLLVMDebugInfoDWARF.a \
- $LLVM_BUILD/lib/libLLVMSupport.a \
- $LLVM_BUILD/lib/libLLVMDebugInfoPDB.a \
- $LLVM_BUILD/lib/libLLVMDebugInfoMSF.a \
- $LLVM_BUILD/lib/libLLVMDebugInfoCodeView.a \
- $LLVM_BUILD/lib/libLLVMDemangle.a \
- $LLVM_BUILD/lib/libLLVMMC.a \
- $LLVM_BUILD/lib/libLLVMTextAPI.a \
- $ZLIB_BUILD/libz.a \
- symbolizer.a \
- all.bc
+$LINK $LIBCXX_ARCHIVE_DIR/libc++.a \
+ $LIBCXX_ARCHIVE_DIR/libc++abi.a \
+ $LLVM_BUILD/lib/libLLVMSymbolize.a \
+ $LLVM_BUILD/lib/libLLVMObject.a \
+ $LLVM_BUILD/lib/libLLVMBinaryFormat.a \
+ $LLVM_BUILD/lib/libLLVMDebugInfoDWARF.a \
+ $LLVM_BUILD/lib/libLLVMSupport.a \
+ $LLVM_BUILD/lib/libLLVMDebugInfoPDB.a \
+ $LLVM_BUILD/lib/libLLVMDebugInfoMSF.a \
+ $LLVM_BUILD/lib/libLLVMDebugInfoCodeView.a \
+ $LLVM_BUILD/lib/libLLVMDebuginfod.a \
+ $LLVM_BUILD/lib/libLLVMDemangle.a \
+ $LLVM_BUILD/lib/libLLVMMC.a \
+ $LLVM_BUILD/lib/libLLVMTextAPI.a \
+ $LLVM_BUILD/lib/libLLVMTargetParser.a \
+ $ZLIB_BUILD/libz.a \
+ symbolizer.a \
+ -ignore-non-bitcode -o all.bc
echo "Optimizing..."
-$OPT -internalize -internalize-public-api-list=${SYMBOLIZER_API_LIST} all.bc -o opt.bc
+$OPT -passes=internalize -internalize-public-api-list=${SYMBOLIZER_API_LIST} all.bc -o opt.bc
$CC $FLAGS -fno-lto -c opt.bc -o symbolizer.o
echo "Checking undefined symbols..."
_GLOBAL_OFFSET_TABLE_ U
+_ZN11__sanitizer13internal_mmapEPvjiiiy U
_ZN11__sanitizer13internal_mmapEPvmiiiy U
_ZN11__sanitizer13internal_openEPKcij U
_ZN11__sanitizer13internal_statEPKcPv U
_ZN11__sanitizer14internal_fstatEiPv U
_ZN11__sanitizer14internal_lstatEPKcPv U
_ZN11__sanitizer15internal_strlenEPKc U
+_ZN11__sanitizer16internal_iserrorEjPi U
_ZN11__sanitizer16internal_iserrorEmPi U
+_ZN11__sanitizer17internal_snprintfEPcjPKcz U
_ZN11__sanitizer17internal_snprintfEPcmPKcz U
__ctype_b_loc U
__ctype_get_mb_cur_max U
__sanitizer_symbolize_data T
__sanitizer_symbolize_demangle T
__sanitizer_symbolize_flush T
+__sanitizer_symbolize_set_demangle T
+__sanitizer_symbolize_set_inline_frames T
__strdup U
__udivdi3 U
__umoddi3 U
_exit U
abort U
access U
+aligned_alloc U
bcmp U
calloc U
catclose U
catopen U
ceil U
ceilf U
-clock_gettime U
cfgetospeed U
+clock_gettime U
dl_iterate_phdr U
dlsym U
dup U
getenv U
getpagesize U
getpid U
+getpwuid U
getrlimit U
gettimeofday U
+getuid U
ioctl U
isalnum U
isalpha U
isatty U
islower U
-isspace U
isprint U
+isspace U
isupper U
isxdigit U
log10 U
lseek U
lseek64 U
+madvise U
malloc U
mbrlen U
mbrtowc U
munmap U
newlocale U
perror U
+posix_madvise U
+posix_memalign U
posix_spawn U
posix_spawn_file_actions_adddup2 U
posix_spawn_file_actions_addopen U
posix_spawn_file_actions_destroy U
posix_spawn_file_actions_init U
qsort U
+raise U
rand U
readlink U
realloc U
remove U
+rename U
setrlimit U
setvbuf U
+sigaction U
+sigaltstack U
+sigemptyset U
sigfillset U
sigprocmask U
snprintf U
strtoll_l U
strtoull_l U
syscall U
+sysconf U
tcgetattr U
+tolower U
+toupper U
uname U
ungetc U
unlink U
clang_compiler_add_cxx_check()
# FIXME: use SANITIZER_COMMON_SUPPORTED_ARCH here
-filter_available_targets(SANITIZER_UNITTEST_SUPPORTED_ARCH x86_64 i386 mips64 mips64el riscv64)
+filter_available_targets(SANITIZER_UNITTEST_SUPPORTED_ARCH x86_64 i386 mips64 mips64el riscv64 sparcv9 sparc)
if(APPLE)
+ list(APPEND SANITIZER_UNITTEST_SUPPORTED_ARCH arm64)
darwin_filter_host_archs(SANITIZER_UNITTEST_SUPPORTED_ARCH SANITIZER_UNITTEST_SUPPORTED_ARCH)
endif()
set(SANITIZER_UNITTESTS
+ sanitizer_addrhashmap_test.cpp
sanitizer_allocator_test.cpp
sanitizer_atomic_test.cpp
sanitizer_bitvector_test.cpp
sanitizer_chained_origin_depot_test.cpp
sanitizer_common_test.cpp
sanitizer_deadlock_detector_test.cpp
+ sanitizer_dense_map_test.cpp
sanitizer_flags_test.cpp
+ sanitizer_flat_map_test.cpp
sanitizer_format_interceptor_test.cpp
+ sanitizer_hash_test.cpp
sanitizer_ioctl_test.cpp
+ sanitizer_leb128_test.cpp
sanitizer_libc_test.cpp
sanitizer_linux_test.cpp
sanitizer_list_test.cpp
+ sanitizer_lzw_test.cpp
sanitizer_mac_test.cpp
sanitizer_mutex_test.cpp
sanitizer_nolibc_test.cpp
sanitizer_procmaps_test.cpp
sanitizer_ring_buffer_test.cpp
sanitizer_quarantine_test.cpp
+ sanitizer_stack_store_test.cpp
sanitizer_stackdepot_test.cpp
sanitizer_stacktrace_printer_test.cpp
sanitizer_stacktrace_test.cpp
${COMPILER_RT_UNITTEST_CFLAGS}
${COMPILER_RT_GTEST_CFLAGS}
${COMPILER_RT_GMOCK_CFLAGS}
+ ${SANITIZER_TEST_CXX_CFLAGS}
-I${COMPILER_RT_SOURCE_DIR}/include
-I${COMPILER_RT_SOURCE_DIR}/lib
-I${COMPILER_RT_SOURCE_DIR}/lib/sanitizer_common
-Wno-gnu-zero-variadic-macro-arguments
)
-set(SANITIZER_TEST_LINK_FLAGS_COMMON ${COMPILER_RT_UNITTEST_LINK_FLAGS})
+set(SANITIZER_TEST_LINK_FLAGS_COMMON
+ ${COMPILER_RT_UNITTEST_LINK_FLAGS}
+ ${COMPILER_RT_UNWINDER_LINK_LIBS}
+ ${SANITIZER_TEST_CXX_LIBRARIES})
# -gline-tables-only must be enough for these tests, so use it if possible.
if(COMPILER_RT_TEST_COMPILER_ID MATCHES "Clang")
add_weak_symbols("sanitizer_common" WEAK_SYMBOL_LINK_FLAGS)
list(APPEND SANITIZER_TEST_LINK_FLAGS_COMMON ${WEAK_SYMBOL_LINK_FLAGS})
+ # For c++17 sanitizer_allocator_test requires language features introduced in macos 10.13
+ list(APPEND SANITIZER_TEST_CFLAGS_COMMON "-mmacosx-version-min=10.13")
endif()
# MSVC linker is allocating 1M for the stack by default, which is not
list(APPEND extra_flags "-D_FILE_OFFSET_BITS=64")
endif()
get_sanitizer_common_lib_for_arch(${arch} SANITIZER_COMMON_LIB)
+ set(TARGET_LINK_FLAGS)
+ get_target_link_flags_for_arch(${arch} TARGET_LINK_FLAGS)
set(SANITIZER_TEST_OBJECTS)
generate_compiler_rt_tests(SANITIZER_TEST_OBJECTS SanitizerUnitTests
RUNTIME "${SANITIZER_COMMON_LIB}"
SOURCES ${SANITIZER_UNITTESTS} ${COMPILER_RT_GTEST_SOURCE} ${COMPILER_RT_GMOCK_SOURCE}
COMPILE_DEPS ${SANITIZER_TEST_HEADERS}
- DEPS gtest
+ DEPS llvm_gtest
CFLAGS ${SANITIZER_TEST_CFLAGS_COMMON} ${extra_flags}
- LINK_FLAGS ${SANITIZER_TEST_LINK_FLAGS_COMMON} ${extra_flags})
+ LINK_FLAGS ${SANITIZER_TEST_LINK_FLAGS_COMMON} ${TARGET_LINK_FLAGS} ${extra_flags})
if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux" AND "${arch}" STREQUAL "x86_64")
# Test that the libc-independent part of sanitizer_common is indeed
--- /dev/null
+//===-- sanitizer_addrhashmap_test.cpp ------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include "sanitizer_common/sanitizer_addrhashmap.h"
+
+#include <unordered_map>
+
+#include "gtest/gtest.h"
+
+namespace __sanitizer {
+
+struct Value {
+ int payload;
+ inline bool operator==(const Value& rhs) const {
+ return payload == rhs.payload;
+ }
+};
+
+using MapTy = AddrHashMap<Value, 11>;
+using HandleTy = MapTy::Handle;
+using RefMapTy = std::unordered_map<uptr, Value>;
+
+static void ExistsInReferenceMap(const uptr key, const Value& val, void* arg) {
+ RefMapTy* ref = reinterpret_cast<RefMapTy*>(arg);
+ const RefMapTy::iterator iter = ref->find(key);
+ ASSERT_NE(iter, ref->end());
+ EXPECT_EQ(iter->second, val);
+ ref->erase(iter);
+}
+
+TEST(AddrHashMap, Basic) {
+ // Use a reference implementation to compare with.
+ RefMapTy reference_map{
+ {0x1000, {1}},
+ {0x2000, {2}},
+ {0x3000, {3}},
+ };
+
+ MapTy m;
+
+ for (const auto& key_val : reference_map) {
+ const uptr key = key_val.first;
+ const Value val = key_val.second;
+
+ // Insert all the elements.
+ {
+ HandleTy h(&m, key);
+ ASSERT_TRUE(h.created());
+ h->payload = val.payload;
+ }
+ }
+
+ // Now check that all the elements are present.
+ m.ForEach(ExistsInReferenceMap, &reference_map);
+ EXPECT_TRUE(reference_map.empty());
+}
+
+} // namespace __sanitizer
#endif // SANITIZER_CAN_USE_ALLOCATOR64
-TEST(SanitizerCommon, TwoLevelByteMap) {
- const u64 kSize1 = 1 << 6, kSize2 = 1 << 12;
- const u64 n = kSize1 * kSize2;
- TwoLevelByteMap<kSize1, kSize2> m;
- m.Init();
- for (u64 i = 0; i < n; i += 7) {
- m.set(i, (i % 100) + 1);
- }
- for (u64 j = 0; j < n; j++) {
- if (j % 7)
- EXPECT_EQ(m[j], 0);
- else
- EXPECT_EQ(m[j], (j % 100) + 1);
- }
-
- m.TestOnlyUnmap();
-}
-
-template <typename AddressSpaceView>
-using TestByteMapASVT =
- TwoLevelByteMap<1 << 12, 1 << 13, AddressSpaceView, TestMapUnmapCallback>;
-using TestByteMap = TestByteMapASVT<LocalAddressSpaceView>;
-
-struct TestByteMapParam {
- TestByteMap *m;
- size_t shard;
- size_t num_shards;
-};
-
-void *TwoLevelByteMapUserThread(void *param) {
- TestByteMapParam *p = (TestByteMapParam*)param;
- for (size_t i = p->shard; i < p->m->size(); i += p->num_shards) {
- size_t val = (i % 100) + 1;
- p->m->set(i, val);
- EXPECT_EQ((*p->m)[i], val);
- }
- return 0;
-}
-
-TEST(SanitizerCommon, ThreadedTwoLevelByteMap) {
- TestByteMap m;
- m.Init();
- TestMapUnmapCallback::map_count = 0;
- TestMapUnmapCallback::unmap_count = 0;
- static const int kNumThreads = 4;
- pthread_t t[kNumThreads];
- TestByteMapParam p[kNumThreads];
- for (int i = 0; i < kNumThreads; i++) {
- p[i].m = &m;
- p[i].shard = i;
- p[i].num_shards = kNumThreads;
- PTHREAD_CREATE(&t[i], 0, TwoLevelByteMapUserThread, &p[i]);
- }
- for (int i = 0; i < kNumThreads; i++) {
- PTHREAD_JOIN(t[i], 0);
- }
- EXPECT_EQ((uptr)TestMapUnmapCallback::map_count, m.size1());
- EXPECT_EQ((uptr)TestMapUnmapCallback::unmap_count, 0UL);
- m.TestOnlyUnmap();
- EXPECT_EQ((uptr)TestMapUnmapCallback::map_count, m.size1());
- EXPECT_EQ((uptr)TestMapUnmapCallback::unmap_count, m.size1());
-}
-
TEST(SanitizerCommon, LowLevelAllocatorShouldRoundUpSizeOnAlloc) {
// When allocating a memory block slightly bigger than a memory page and
// LowLevelAllocator calls MmapOrDie for the internal buffer, it should round
#if defined(_WIN64)
fprintf(stderr, "%llu ", *it);
#else
- fprintf(stderr, "%lu ", *it);
+ fprintf(stderr, "%zu ", *it);
#endif
}
fprintf(stderr, "\n");
}
TEST(SanitizerCommon, ChainedOriginDepotStats) {
- StackDepotStats stats0 = *chainedOriginDepot.GetStats();
+ chainedOriginDepot.TestOnlyUnmap();
+ StackDepotStats stats0 = chainedOriginDepot.GetStats();
u32 new_id;
EXPECT_TRUE(chainedOriginDepot.Put(33, 34, &new_id));
- StackDepotStats stats1 = *chainedOriginDepot.GetStats();
+ StackDepotStats stats1 = chainedOriginDepot.GetStats();
EXPECT_EQ(stats1.n_uniq_ids, stats0.n_uniq_ids + 1);
EXPECT_GT(stats1.allocated, stats0.allocated);
EXPECT_FALSE(chainedOriginDepot.Put(33, 34, &new_id));
- StackDepotStats stats2 = *chainedOriginDepot.GetStats();
+ StackDepotStats stats2 = chainedOriginDepot.GetStats();
EXPECT_EQ(stats2.n_uniq_ids, stats1.n_uniq_ids);
EXPECT_EQ(stats2.allocated, stats1.allocated);
- EXPECT_TRUE(chainedOriginDepot.Put(35, 36, &new_id));
- StackDepotStats stats3 = *chainedOriginDepot.GetStats();
- EXPECT_EQ(stats3.n_uniq_ids, stats2.n_uniq_ids + 1);
- EXPECT_GT(stats3.allocated, stats2.allocated);
+ for (int i = 0; i < 100000; ++i) {
+ ASSERT_TRUE(chainedOriginDepot.Put(35, i, &new_id));
+ StackDepotStats stats3 = chainedOriginDepot.GetStats();
+ ASSERT_EQ(stats3.n_uniq_ids, stats2.n_uniq_ids + 1 + i);
+ }
+ EXPECT_GT(chainedOriginDepot.GetStats().allocated, stats2.allocated);
}
} // namespace __sanitizer
//===----------------------------------------------------------------------===//
#include <algorithm>
+// This ensures that including both internal sanitizer_common headers
+// and the interface headers does not lead to compilation failures.
+// Both may be included in unit tests, where googletest transitively
+// pulls in sanitizer interface headers.
+// The headers are specifically included using relative paths,
+// because a compiler may use a different mismatching version
+// of sanitizer headers.
+#include "../../../include/sanitizer/asan_interface.h"
+#include "../../../include/sanitizer/msan_interface.h"
+#include "../../../include/sanitizer/tsan_interface.h"
+#include "gtest/gtest.h"
#include "sanitizer_common/sanitizer_allocator_internal.h"
#include "sanitizer_common/sanitizer_common.h"
#include "sanitizer_common/sanitizer_file.h"
#include "sanitizer_common/sanitizer_flags.h"
#include "sanitizer_common/sanitizer_libc.h"
#include "sanitizer_common/sanitizer_platform.h"
-
#include "sanitizer_pthread_wrappers.h"
-#include "gtest/gtest.h"
-
namespace __sanitizer {
static bool IsSorted(const uptr *array, uptr n) {
}
}
+TEST(SanitizerCommon, Mprotect) {
+ uptr PageSize = GetPageSizeCached();
+ u8 *mem = reinterpret_cast<u8 *>(MmapOrDie(PageSize, "MprotectTest"));
+ for (u8 *p = mem; p < mem + PageSize; ++p) ++(*p);
+
+ MprotectReadOnly(reinterpret_cast<uptr>(mem), PageSize);
+ for (u8 *p = mem; p < mem + PageSize; ++p) EXPECT_EQ(1u, *p);
+ EXPECT_DEATH(++mem[0], "");
+ EXPECT_DEATH(++mem[PageSize / 2], "");
+ EXPECT_DEATH(++mem[PageSize - 1], "");
+
+ MprotectNoAccess(reinterpret_cast<uptr>(mem), PageSize);
+ volatile u8 t;
+ (void)t;
+ EXPECT_DEATH(t = mem[0], "");
+ EXPECT_DEATH(t = mem[PageSize / 2], "");
+ EXPECT_DEATH(t = mem[PageSize - 1], "");
+}
+
TEST(SanitizerCommon, InternalMmapVectorRoundUpCapacity) {
InternalMmapVector<uptr> v;
v.reserve(1);
for (int i = 0; i < 1000; ++i) {
std::string append(i, 'a' + i % 26);
expected += append;
- str.append(append.c_str());
+ str.append("%s", append.c_str());
EXPECT_EQ(expected, str.data());
}
}
}
}
-#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_IOS
+#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_APPLE || SANITIZER_IOS
TEST(SanitizerCommon, GetRandom) {
u8 buffer_1[32], buffer_2[32];
for (bool blocking : { false, true }) {
--- /dev/null
+//===- sanitizer_dense_map_test.cpp -----------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_dense_map.h"
+
+#include <initializer_list>
+#include <map>
+#include <set>
+
+#include "gtest/gtest.h"
+
+using namespace __sanitizer;
+
+namespace {
+
+// Helps to keep some tests.
+template <typename KeyT, typename ValueT,
+ typename KeyInfoT = DenseMapInfo<KeyT>>
+class TestDenseMap : public DenseMap<KeyT, ValueT, KeyInfoT> {
+ using BaseT = DenseMap<KeyT, ValueT, KeyInfoT>;
+
+ public:
+ using BaseT::BaseT;
+
+ TestDenseMap(std::initializer_list<typename BaseT::value_type> Vals)
+ : BaseT(Vals.size()) {
+ for (const auto &V : Vals) this->BaseT::insert(V);
+ }
+
+ template <typename I>
+ TestDenseMap(I B, I E) : BaseT(std::distance(B, E)) {
+ for (; B != E; ++B) this->BaseT::insert(*B);
+ }
+};
+
+template <typename... T>
+using DenseMap = TestDenseMap<T...>;
+
+uint32_t getTestKey(int i, uint32_t *) { return i; }
+uint32_t getTestValue(int i, uint32_t *) { return 42 + i; }
+
+uint32_t *getTestKey(int i, uint32_t **) {
+ static uint32_t dummy_arr1[8192];
+ assert(i < 8192 && "Only support 8192 dummy keys.");
+ return &dummy_arr1[i];
+}
+uint32_t *getTestValue(int i, uint32_t **) {
+ static uint32_t dummy_arr1[8192];
+ assert(i < 8192 && "Only support 8192 dummy keys.");
+ return &dummy_arr1[i];
+}
+
+/// A test class that tries to check that construction and destruction
+/// occur correctly.
+class CtorTester {
+ static std::set<CtorTester *> Constructed;
+ int Value;
+
+ public:
+ explicit CtorTester(int Value = 0) : Value(Value) {
+ EXPECT_TRUE(Constructed.insert(this).second);
+ }
+ CtorTester(uint32_t Value) : Value(Value) {
+ EXPECT_TRUE(Constructed.insert(this).second);
+ }
+ CtorTester(const CtorTester &Arg) : Value(Arg.Value) {
+ EXPECT_TRUE(Constructed.insert(this).second);
+ }
+ CtorTester &operator=(const CtorTester &) = default;
+ ~CtorTester() { EXPECT_EQ(1u, Constructed.erase(this)); }
+ operator uint32_t() const { return Value; }
+
+ int getValue() const { return Value; }
+ bool operator==(const CtorTester &RHS) const { return Value == RHS.Value; }
+};
+
+std::set<CtorTester *> CtorTester::Constructed;
+
+struct CtorTesterMapInfo {
+ static inline CtorTester getEmptyKey() { return CtorTester(-1); }
+ static inline CtorTester getTombstoneKey() { return CtorTester(-2); }
+ static unsigned getHashValue(const CtorTester &Val) {
+ return Val.getValue() * 37u;
+ }
+ static bool isEqual(const CtorTester &LHS, const CtorTester &RHS) {
+ return LHS == RHS;
+ }
+};
+
+CtorTester getTestKey(int i, CtorTester *) { return CtorTester(i); }
+CtorTester getTestValue(int i, CtorTester *) { return CtorTester(42 + i); }
+
+// Test fixture, with helper functions implemented by forwarding to global
+// function overloads selected by component types of the type parameter. This
+// allows all of the map implementations to be tested with shared
+// implementations of helper routines.
+template <typename T>
+class DenseMapTest : public ::testing::Test {
+ protected:
+ T Map;
+
+ static typename T::key_type *const dummy_key_ptr;
+ static typename T::mapped_type *const dummy_value_ptr;
+
+ typename T::key_type getKey(int i = 0) {
+ return getTestKey(i, dummy_key_ptr);
+ }
+ typename T::mapped_type getValue(int i = 0) {
+ return getTestValue(i, dummy_value_ptr);
+ }
+};
+
+template <typename T>
+typename T::key_type *const DenseMapTest<T>::dummy_key_ptr = nullptr;
+template <typename T>
+typename T::mapped_type *const DenseMapTest<T>::dummy_value_ptr = nullptr;
+
+// Register these types for testing.
+typedef ::testing::Types<DenseMap<uint32_t, uint32_t>,
+ DenseMap<uint32_t *, uint32_t *>,
+ DenseMap<CtorTester, CtorTester, CtorTesterMapInfo>>
+ DenseMapTestTypes;
+TYPED_TEST_SUITE(DenseMapTest, DenseMapTestTypes, );
+
+// Empty map tests
+TYPED_TEST(DenseMapTest, EmptyIntMapTest) {
+ // Size tests
+ EXPECT_EQ(0u, this->Map.size());
+ EXPECT_TRUE(this->Map.empty());
+
+ // Lookup tests
+ EXPECT_FALSE(this->Map.count(this->getKey()));
+ EXPECT_EQ(nullptr, this->Map.find(this->getKey()));
+ EXPECT_EQ(typename TypeParam::mapped_type(),
+ this->Map.lookup(this->getKey()));
+}
+
+// Constant map tests
+TYPED_TEST(DenseMapTest, ConstEmptyMapTest) {
+ const TypeParam &ConstMap = this->Map;
+ EXPECT_EQ(0u, ConstMap.size());
+ EXPECT_TRUE(ConstMap.empty());
+}
+
+// A map with a single entry
+TYPED_TEST(DenseMapTest, SingleEntryMapTest) {
+ this->Map[this->getKey()] = this->getValue();
+
+ // Size tests
+ EXPECT_EQ(1u, this->Map.size());
+ EXPECT_FALSE(this->Map.empty());
+
+ // Lookup tests
+ EXPECT_TRUE(this->Map.count(this->getKey()));
+ EXPECT_NE(nullptr, this->Map.find(this->getKey()));
+ EXPECT_EQ(this->getValue(), this->Map.lookup(this->getKey()));
+ EXPECT_EQ(this->getValue(), this->Map[this->getKey()]);
+}
+
+// Test clear() method
+TYPED_TEST(DenseMapTest, ClearTest) {
+ this->Map[this->getKey()] = this->getValue();
+ this->Map.clear();
+
+ EXPECT_EQ(0u, this->Map.size());
+ EXPECT_TRUE(this->Map.empty());
+}
+
+// Test erase(iterator) method
+TYPED_TEST(DenseMapTest, EraseTest) {
+ this->Map[this->getKey()] = this->getValue();
+ this->Map.erase(this->Map.find(this->getKey()));
+
+ EXPECT_EQ(0u, this->Map.size());
+ EXPECT_TRUE(this->Map.empty());
+}
+
+// Test erase(value) method
+TYPED_TEST(DenseMapTest, EraseTest2) {
+ this->Map[this->getKey()] = this->getValue();
+ this->Map.erase(this->getKey());
+
+ EXPECT_EQ(0u, this->Map.size());
+ EXPECT_TRUE(this->Map.empty());
+}
+
+// Test insert() method
+TYPED_TEST(DenseMapTest, InsertTest) {
+ this->Map.insert(
+ typename TypeParam::value_type(this->getKey(), this->getValue()));
+ EXPECT_EQ(1u, this->Map.size());
+ EXPECT_EQ(this->getValue(), this->Map[this->getKey()]);
+}
+
+// Test copy constructor method
+TYPED_TEST(DenseMapTest, CopyConstructorTest) {
+ this->Map[this->getKey()] = this->getValue();
+ TypeParam copyMap(this->Map);
+
+ EXPECT_EQ(1u, copyMap.size());
+ EXPECT_EQ(this->getValue(), copyMap[this->getKey()]);
+}
+
+// Test copy constructor method where SmallDenseMap isn't small.
+TYPED_TEST(DenseMapTest, CopyConstructorNotSmallTest) {
+ for (int Key = 0; Key < 5; ++Key)
+ this->Map[this->getKey(Key)] = this->getValue(Key);
+ TypeParam copyMap(this->Map);
+
+ EXPECT_EQ(5u, copyMap.size());
+ for (int Key = 0; Key < 5; ++Key)
+ EXPECT_EQ(this->getValue(Key), copyMap[this->getKey(Key)]);
+}
+
+// Test copying from a default-constructed map.
+TYPED_TEST(DenseMapTest, CopyConstructorFromDefaultTest) {
+ TypeParam copyMap(this->Map);
+
+ EXPECT_TRUE(copyMap.empty());
+}
+
+// Test copying from an empty map where SmallDenseMap isn't small.
+TYPED_TEST(DenseMapTest, CopyConstructorFromEmptyTest) {
+ for (int Key = 0; Key < 5; ++Key)
+ this->Map[this->getKey(Key)] = this->getValue(Key);
+ this->Map.clear();
+ TypeParam copyMap(this->Map);
+
+ EXPECT_TRUE(copyMap.empty());
+}
+
+// Test assignment operator method
+TYPED_TEST(DenseMapTest, AssignmentTest) {
+ this->Map[this->getKey()] = this->getValue();
+ TypeParam copyMap = this->Map;
+
+ EXPECT_EQ(1u, copyMap.size());
+ EXPECT_EQ(this->getValue(), copyMap[this->getKey()]);
+
+ // test self-assignment.
+ copyMap = static_cast<TypeParam &>(copyMap);
+ EXPECT_EQ(1u, copyMap.size());
+ EXPECT_EQ(this->getValue(), copyMap[this->getKey()]);
+}
+
+TYPED_TEST(DenseMapTest, AssignmentTestNotSmall) {
+ for (int Key = 0; Key < 5; ++Key)
+ this->Map[this->getKey(Key)] = this->getValue(Key);
+ TypeParam copyMap = this->Map;
+
+ EXPECT_EQ(5u, copyMap.size());
+ for (int Key = 0; Key < 5; ++Key)
+ EXPECT_EQ(this->getValue(Key), copyMap[this->getKey(Key)]);
+
+ // test self-assignment.
+ copyMap = static_cast<TypeParam &>(copyMap);
+ EXPECT_EQ(5u, copyMap.size());
+ for (int Key = 0; Key < 5; ++Key)
+ EXPECT_EQ(this->getValue(Key), copyMap[this->getKey(Key)]);
+}
+
+// Test swap method
+TYPED_TEST(DenseMapTest, SwapTest) {
+ this->Map[this->getKey()] = this->getValue();
+ TypeParam otherMap;
+
+ this->Map.swap(otherMap);
+ EXPECT_EQ(0u, this->Map.size());
+ EXPECT_TRUE(this->Map.empty());
+ EXPECT_EQ(1u, otherMap.size());
+ EXPECT_EQ(this->getValue(), otherMap[this->getKey()]);
+
+ this->Map.swap(otherMap);
+ EXPECT_EQ(0u, otherMap.size());
+ EXPECT_TRUE(otherMap.empty());
+ EXPECT_EQ(1u, this->Map.size());
+ EXPECT_EQ(this->getValue(), this->Map[this->getKey()]);
+
+ // Make this more interesting by inserting 100 numbers into the map.
+ for (int i = 0; i < 100; ++i) this->Map[this->getKey(i)] = this->getValue(i);
+
+ this->Map.swap(otherMap);
+ EXPECT_EQ(0u, this->Map.size());
+ EXPECT_TRUE(this->Map.empty());
+ EXPECT_EQ(100u, otherMap.size());
+ for (int i = 0; i < 100; ++i)
+ EXPECT_EQ(this->getValue(i), otherMap[this->getKey(i)]);
+
+ this->Map.swap(otherMap);
+ EXPECT_EQ(0u, otherMap.size());
+ EXPECT_TRUE(otherMap.empty());
+ EXPECT_EQ(100u, this->Map.size());
+ for (int i = 0; i < 100; ++i)
+ EXPECT_EQ(this->getValue(i), this->Map[this->getKey(i)]);
+}
+
+// A more complex iteration test
+TYPED_TEST(DenseMapTest, IterationTest) {
+ int visited[100];
+ std::map<typename TypeParam::key_type, unsigned> visitedIndex;
+
+ // Insert 100 numbers into the map
+ for (int i = 0; i < 100; ++i) {
+ visited[i] = 0;
+ visitedIndex[this->getKey(i)] = i;
+
+ this->Map[this->getKey(i)] = this->getValue(i);
+ }
+
+ // Iterate over all numbers and mark each one found.
+ this->Map.forEach([&](const typename TypeParam::value_type &kv) {
+ ++visited[visitedIndex[kv.first]];
+ return true;
+ });
+
+ // Ensure every number was visited.
+ for (int i = 0; i < 100; ++i) ASSERT_EQ(1, visited[i]);
+}
+
+namespace {
+// Simple class that counts how many moves and copy happens when growing a map
+struct CountCopyAndMove {
+ static int Move;
+ static int Copy;
+ CountCopyAndMove() {}
+
+ CountCopyAndMove(const CountCopyAndMove &) { Copy++; }
+ CountCopyAndMove &operator=(const CountCopyAndMove &) {
+ Copy++;
+ return *this;
+ }
+ CountCopyAndMove(CountCopyAndMove &&) { Move++; }
+ CountCopyAndMove &operator=(const CountCopyAndMove &&) {
+ Move++;
+ return *this;
+ }
+};
+int CountCopyAndMove::Copy = 0;
+int CountCopyAndMove::Move = 0;
+
+} // anonymous namespace
+
+// Test initializer list construction.
+TEST(DenseMapCustomTest, InitializerList) {
+ DenseMap<int, int> M({{0, 0}, {0, 1}, {1, 2}});
+ EXPECT_EQ(2u, M.size());
+ EXPECT_EQ(1u, M.count(0));
+ EXPECT_EQ(0, M[0]);
+ EXPECT_EQ(1u, M.count(1));
+ EXPECT_EQ(2, M[1]);
+}
+
+// Test initializer list construction.
+TEST(DenseMapCustomTest, EqualityComparison) {
+ DenseMap<int, int> M1({{0, 0}, {1, 2}});
+ DenseMap<int, int> M2({{0, 0}, {1, 2}});
+ DenseMap<int, int> M3({{0, 0}, {1, 3}});
+
+ EXPECT_EQ(M1, M2);
+ EXPECT_NE(M1, M3);
+}
+
+const int ExpectedInitialBucketCount = GetPageSizeCached() / /* sizeof(KV) */ 8;
+
+// Test for the default minimum size of a DenseMap
+TEST(DenseMapCustomTest, DefaultMinReservedSizeTest) {
+ // Formula from DenseMap::getMinBucketToReserveForEntries()
+ const int ExpectedMaxInitialEntries = ExpectedInitialBucketCount * 3 / 4 - 1;
+
+ DenseMap<int, CountCopyAndMove> Map;
+ // Will allocate 64 buckets
+ Map.reserve(1);
+ unsigned MemorySize = Map.getMemorySize();
+ CountCopyAndMove::Copy = 0;
+ CountCopyAndMove::Move = 0;
+ for (int i = 0; i < ExpectedMaxInitialEntries; ++i) {
+ detail::DenseMapPair<int, CountCopyAndMove> KV;
+ KV.first = i;
+ Map.insert(move(KV));
+ }
+ // Check that we didn't grow
+ EXPECT_EQ(MemorySize, Map.getMemorySize());
+ // Check that move was called the expected number of times
+ EXPECT_EQ(ExpectedMaxInitialEntries, CountCopyAndMove::Move);
+ // Check that no copy occurred
+ EXPECT_EQ(0, CountCopyAndMove::Copy);
+
+ // Adding one extra element should grow the map
+ detail::DenseMapPair<int, CountCopyAndMove> KV;
+ KV.first = ExpectedMaxInitialEntries;
+ Map.insert(move(KV));
+ // Check that we grew
+ EXPECT_NE(MemorySize, Map.getMemorySize());
+ // Check that move was called the expected number of times
+ // This relies on move-construction elision, and cannot be reliably tested.
+ // EXPECT_EQ(ExpectedMaxInitialEntries + 2, CountCopyAndMove::Move);
+ // Check that no copy occurred
+ EXPECT_EQ(0, CountCopyAndMove::Copy);
+}
+
+// Make sure creating the map with an initial size of N actually gives us enough
+// buckets to insert N items without increasing allocation size.
+TEST(DenseMapCustomTest, InitialSizeTest) {
+ // Test a few different size, 341 is *not* a random choice: we need a value
+ // that is 2/3 of a power of two to stress the grow() condition, and the power
+ // of two has to be at least 512 because of minimum size allocation in the
+ // DenseMap (see DefaultMinReservedSizeTest).
+ for (auto Size : {1, 2, 48, 66, 341, ExpectedInitialBucketCount + 1}) {
+ DenseMap<int, CountCopyAndMove> Map(Size);
+ unsigned MemorySize = Map.getMemorySize();
+ CountCopyAndMove::Copy = 0;
+ CountCopyAndMove::Move = 0;
+ for (int i = 0; i < Size; ++i) {
+ detail::DenseMapPair<int, CountCopyAndMove> KV;
+ KV.first = i;
+ Map.insert(move(KV));
+ }
+ // Check that we didn't grow
+ EXPECT_EQ(MemorySize, Map.getMemorySize());
+ // Check that move was called the expected number of times
+ EXPECT_EQ(Size, CountCopyAndMove::Move);
+ // Check that no copy occurred
+ EXPECT_EQ(0, CountCopyAndMove::Copy);
+ }
+}
+
+// Make sure creating the map with a iterator range does not trigger grow()
+TEST(DenseMapCustomTest, InitFromIterator) {
+ std::vector<detail::DenseMapPair<int, CountCopyAndMove>> Values;
+ // The size is a random value greater than 64 (hardcoded DenseMap min init)
+ const int Count = 65;
+ for (int i = 0; i < Count; i++) Values.emplace_back(i, CountCopyAndMove());
+
+ CountCopyAndMove::Move = 0;
+ CountCopyAndMove::Copy = 0;
+ DenseMap<int, CountCopyAndMove> Map(Values.begin(), Values.end());
+ // Check that no move occurred
+ EXPECT_EQ(0, CountCopyAndMove::Move);
+ // Check that copy was called the expected number of times
+ EXPECT_EQ(Count, CountCopyAndMove::Copy);
+}
+
+// Make sure reserve actually gives us enough buckets to insert N items
+// without increasing allocation size.
+TEST(DenseMapCustomTest, ReserveTest) {
+ // Test a few different size, 341 is *not* a random choice: we need a value
+ // that is 2/3 of a power of two to stress the grow() condition, and the power
+ // of two has to be at least 512 because of minimum size allocation in the
+ // DenseMap (see DefaultMinReservedSizeTest).
+ for (auto Size : {1, 2, 48, 66, 341, ExpectedInitialBucketCount + 1}) {
+ DenseMap<int, CountCopyAndMove> Map;
+ Map.reserve(Size);
+ unsigned MemorySize = Map.getMemorySize();
+ CountCopyAndMove::Copy = 0;
+ CountCopyAndMove::Move = 0;
+ for (int i = 0; i < Size; ++i) {
+ detail::DenseMapPair<int, CountCopyAndMove> KV;
+ KV.first = i;
+ Map.insert(move(KV));
+ }
+ // Check that we didn't grow
+ EXPECT_EQ(MemorySize, Map.getMemorySize());
+ // Check that move was called the expected number of times
+ EXPECT_EQ(Size, CountCopyAndMove::Move);
+ // Check that no copy occurred
+ EXPECT_EQ(0, CountCopyAndMove::Copy);
+ }
+}
+
+// Key traits that allows lookup with either an unsigned or char* key;
+// In the latter case, "a" == 0, "b" == 1 and so on.
+struct TestDenseMapInfo {
+ static inline unsigned getEmptyKey() { return ~0; }
+ static inline unsigned getTombstoneKey() { return ~0U - 1; }
+ static unsigned getHashValue(const unsigned &Val) { return Val * 37U; }
+ static unsigned getHashValue(const char *Val) {
+ return (unsigned)(Val[0] - 'a') * 37U;
+ }
+ static bool isEqual(const unsigned &LHS, const unsigned &RHS) {
+ return LHS == RHS;
+ }
+ static bool isEqual(const char *LHS, const unsigned &RHS) {
+ return (unsigned)(LHS[0] - 'a') == RHS;
+ }
+};
+
+// find_as() tests
+TEST(DenseMapCustomTest, FindAsTest) {
+ DenseMap<unsigned, unsigned, TestDenseMapInfo> map;
+ map[0] = 1;
+ map[1] = 2;
+ map[2] = 3;
+
+ // Size tests
+ EXPECT_EQ(3u, map.size());
+
+ // Normal lookup tests
+ EXPECT_EQ(1u, map.count(1));
+ EXPECT_EQ(1u, map.find(0)->second);
+ EXPECT_EQ(2u, map.find(1)->second);
+ EXPECT_EQ(3u, map.find(2)->second);
+ EXPECT_EQ(nullptr, map.find(3));
+
+ // find_as() tests
+ EXPECT_EQ(1u, map.find_as("a")->second);
+ EXPECT_EQ(2u, map.find_as("b")->second);
+ EXPECT_EQ(3u, map.find_as("c")->second);
+ EXPECT_EQ(nullptr, map.find_as("d"));
+}
+
+TEST(DenseMapCustomTest, TryEmplaceTest) {
+ DenseMap<int, std::unique_ptr<int>> Map;
+ std::unique_ptr<int> P(new int(2));
+ auto Try1 = Map.try_emplace(0, new int(1));
+ EXPECT_TRUE(Try1.second);
+ auto Try2 = Map.try_emplace(0, std::move(P));
+ EXPECT_FALSE(Try2.second);
+ EXPECT_EQ(Try1.first, Try2.first);
+ EXPECT_NE(nullptr, P);
+}
+
+struct IncompleteStruct;
+
+TEST(DenseMapCustomTest, OpaquePointerKey) {
+ // Test that we can use a pointer to an incomplete type as a DenseMap key.
+ // This is an important build time optimization, since many classes have
+ // DenseMap members.
+ DenseMap<IncompleteStruct *, int> Map;
+ int Keys[3] = {0, 0, 0};
+ IncompleteStruct *K1 = reinterpret_cast<IncompleteStruct *>(&Keys[0]);
+ IncompleteStruct *K2 = reinterpret_cast<IncompleteStruct *>(&Keys[1]);
+ IncompleteStruct *K3 = reinterpret_cast<IncompleteStruct *>(&Keys[2]);
+ Map.insert({K1, 1});
+ Map.insert({K2, 2});
+ Map.insert({K3, 3});
+ EXPECT_EQ(Map.count(K1), 1u);
+ EXPECT_EQ(Map[K1], 1);
+ EXPECT_EQ(Map[K2], 2);
+ EXPECT_EQ(Map[K3], 3);
+ Map.clear();
+ EXPECT_EQ(nullptr, Map.find(K1));
+ EXPECT_EQ(nullptr, Map.find(K2));
+ EXPECT_EQ(nullptr, Map.find(K3));
+}
+} // namespace
--- /dev/null
+//===-- sanitizer_flat_map_test.cpp ---------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_flat_map.h"
+
+#include "gtest/gtest.h"
+#include "sanitizer_common/tests/sanitizer_pthread_wrappers.h"
+
+using namespace __sanitizer;
+
+namespace {
+struct TestMapUnmapCallback1 {
+ static int map_count, unmap_count;
+ void OnMap(uptr p, uptr size) const { map_count++; }
+ void OnUnmap(uptr p, uptr size) const { unmap_count++; }
+};
+int TestMapUnmapCallback1::map_count;
+int TestMapUnmapCallback1::unmap_count;
+
+struct TestStruct {
+ int data[125] = {};
+ TestStruct(uptr v = 0) { data[11] = v; }
+ bool operator==(const TestStruct &other) const {
+ return 0 == memcmp(data, other.data, sizeof(data));
+ }
+};
+
+template <typename T>
+class FlatMapTest : public ::testing::Test {};
+
+using FlatMapTestTypes = ::testing::Types<u8, u64, TestStruct>;
+TYPED_TEST_SUITE(FlatMapTest, FlatMapTestTypes, );
+
+TYPED_TEST(FlatMapTest, TwoLevelByteMap) {
+ const u64 kSize1 = 1 << 6, kSize2 = 1 << 12;
+ const u64 n = kSize1 * kSize2;
+ TwoLevelMap<TypeParam, kSize1, kSize2> m;
+ m.Init();
+
+ m[7] = {10};
+ for (u64 i = 0; i < kSize2; ++i) {
+ EXPECT_TRUE(m.contains(i));
+ }
+ EXPECT_FALSE(m.contains(kSize2));
+
+ for (u64 i = 0; i < n; i += 7) {
+ m[i] = TypeParam((i % 100) + 1);
+ }
+ for (u64 j = 0; j < n; j++) {
+ EXPECT_TRUE(m.contains(j));
+ if (j % 7)
+ EXPECT_EQ(m[j], TypeParam());
+ else
+ EXPECT_EQ(m[j], TypeParam((j % 100) + 1));
+ }
+
+ m.TestOnlyUnmap();
+}
+
+template <typename TypeParam, typename AddressSpaceView>
+using TestMapASVT = TwoLevelMap<TypeParam, 1 << 8, 1 << 7, AddressSpaceView,
+ TestMapUnmapCallback1>;
+template <typename TypeParam>
+using TestMap = TestMapASVT<TypeParam, LocalAddressSpaceView>;
+
+template <typename TypeParam>
+struct TestMapParam {
+ TestMap<TypeParam> *m;
+ size_t shard;
+ size_t num_shards;
+};
+
+template <typename TypeParam>
+static void *TwoLevelMapUserThread(void *param) {
+ TestMapParam<TypeParam> *p = (TestMapParam<TypeParam> *)param;
+ for (size_t i = p->shard; i < p->m->size(); i += p->num_shards) {
+ TypeParam val = (i % 100) + 1;
+ (*p->m)[i] = val;
+ EXPECT_EQ((*p->m)[i], val);
+ }
+ return 0;
+}
+
+TYPED_TEST(FlatMapTest, ThreadedTwoLevelByteMap) {
+ TestMap<TypeParam> m;
+ m.Init();
+ TestMapUnmapCallback1::map_count = 0;
+ TestMapUnmapCallback1::unmap_count = 0;
+ static const int kNumThreads = 4;
+ pthread_t t[kNumThreads];
+ TestMapParam<TypeParam> p[kNumThreads];
+ for (int i = 0; i < kNumThreads; i++) {
+ p[i].m = &m;
+ p[i].shard = i;
+ p[i].num_shards = kNumThreads;
+ PTHREAD_CREATE(&t[i], 0, TwoLevelMapUserThread<TypeParam>, &p[i]);
+ }
+ for (int i = 0; i < kNumThreads; i++) {
+ PTHREAD_JOIN(t[i], 0);
+ }
+ EXPECT_EQ((uptr)TestMapUnmapCallback1::map_count, m.size1());
+ EXPECT_EQ((uptr)TestMapUnmapCallback1::unmap_count, 0UL);
+ m.TestOnlyUnmap();
+ EXPECT_EQ((uptr)TestMapUnmapCallback1::map_count, m.size1());
+ EXPECT_EQ((uptr)TestMapUnmapCallback1::unmap_count, m.size1());
+}
+
+} // namespace
--- /dev/null
+//===-- sanitizer_hash_test.cpp -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of Sanitizers.
+//
+//===----------------------------------------------------------------------===//
+#include "sanitizer_common/sanitizer_hash.h"
+
+#include "gtest/gtest.h"
+
+namespace __sanitizer {
+
+// Tests matche a few hashes generated by https://github.com/aappleby/smhasher.
+
+TEST(SanitizerCommon, Hash32Seed) {
+ EXPECT_EQ(MurMur2HashBuilder(0).get(), 275646681u);
+ EXPECT_EQ(MurMur2HashBuilder(1).get(), 3030210376u);
+ EXPECT_EQ(MurMur2HashBuilder(3).get(), 1816185114u);
+}
+
+TEST(SanitizerCommon, Hash32Add) {
+ MurMur2HashBuilder h(123 * sizeof(u32));
+ for (u32 i = 0; i < 123; ++i) h.add(i);
+ EXPECT_EQ(h.get(), 351963665u);
+ for (u32 i = 0; i < 123; ++i) h.add(-i);
+ EXPECT_EQ(h.get(), 2640061027u);
+}
+
+TEST(SanitizerCommon, Hash64Seed) {
+ EXPECT_EQ(MurMur2Hash64Builder(0).get(), 4469829599815726255ull);
+ EXPECT_EQ(MurMur2Hash64Builder(1).get(), 14121968454562043709ull);
+ EXPECT_EQ(MurMur2Hash64Builder(3).get(), 8040757559320203998ull);
+}
+
+TEST(SanitizerCommon, Hash64Add) {
+ MurMur2Hash64Builder h(123 * sizeof(u64));
+ for (u32 i = 0; i < 123; ++i) h.add(i);
+ EXPECT_EQ(h.get(), 11366430808886012537ull);
+ for (u32 i = 0; i < 123; ++i) h.add(-i);
+ EXPECT_EQ(h.get(), 10843188204560467446ull);
+}
+
+} // namespace __sanitizer
--- /dev/null
+//===-- sanitizer_leb128.cpp ------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include "sanitizer_common/sanitizer_leb128.h"
+
+#include <type_traits>
+
+#include "gtest/gtest.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_internal_defs.h"
+
+namespace __sanitizer {
+
+template <typename T>
+class Leb128Test : public ::testing::Test {};
+
+using Leb128TestTypes = ::testing::Types<u8, u16, u32, u64>;
+TYPED_TEST_SUITE(Leb128Test, Leb128TestTypes, );
+
+static uptr BitsNeeded(u64 v) {
+ if (!v)
+ return 1;
+ uptr r = 0;
+ if (sizeof(uptr) != sizeof(u64)) {
+ uptr uptr_bits = 8 * sizeof(uptr);
+ while (v >> uptr_bits) {
+ r += uptr_bits;
+ v >>= uptr_bits;
+ }
+ }
+ return r + MostSignificantSetBitIndex(v) + 1;
+}
+
+TYPED_TEST(Leb128Test, SignedOverflow) {
+ using T = typename std::make_signed<TypeParam>::type;
+ u8 buffer[16] = {255};
+ T v = -128;
+ EXPECT_EQ(buffer + 1, EncodeSLEB128(v, buffer, buffer + 1));
+ EXPECT_EQ(buffer + 1, DecodeSLEB128(buffer, buffer + 1, &v));
+}
+
+TYPED_TEST(Leb128Test, Signed) {
+ using T = typename std::make_signed<TypeParam>::type;
+ T v = 0;
+ for (int i = 0; i < 100; ++i) {
+ u8 buffer[16] = {};
+ u8* p = EncodeSLEB128(v, std::begin(buffer), std::end(buffer));
+ EXPECT_EQ(int(BitsNeeded(v < 0 ? (-v - 1) : v) + 6 + 1) / 7, p - buffer)
+ << (int)v;
+ T v2;
+ u8* p2 = DecodeSLEB128(std::begin(buffer), std::end(buffer), &v2);
+ EXPECT_EQ(v, v2);
+ EXPECT_EQ(p, p2);
+ v = -TypeParam(v) * 3u + 1u;
+ }
+}
+
+TYPED_TEST(Leb128Test, UnsignedOverflow) {
+ using T = TypeParam;
+ u8 buffer[16] = {255};
+ T v = 255;
+ EXPECT_EQ(buffer + 1, EncodeULEB128(v, buffer, buffer + 1));
+ EXPECT_EQ(buffer + 1, DecodeULEB128(buffer, buffer + 1, &v));
+}
+
+TYPED_TEST(Leb128Test, Unsigned) {
+ using T = TypeParam;
+ T v = 0;
+ for (int i = 0; i < 100; ++i) {
+ u8 buffer[16] = {};
+ u8* p = EncodeULEB128(v, std::begin(buffer), std::end(buffer));
+ EXPECT_EQ(int(BitsNeeded(v) + 6) / 7, p - buffer);
+ T v2;
+ u8* p2 = DecodeULEB128(std::begin(buffer), std::end(buffer), &v2);
+ EXPECT_EQ(v, v2);
+ EXPECT_EQ(p, p2);
+ v = v * 3 + 1;
+ }
+}
+
+} // namespace __sanitizer
unsigned char z;
};
+static void get_temp_dir(char *buf, size_t bufsize) {
+#if SANITIZER_WINDOWS
+ buf[0] = '\0';
+ if (!::GetTempPathA(bufsize, buf))
+ return;
+#else
+ const char *tmpdir = "/tmp";
+# if SANITIZER_ANDROID
+ tmpdir = GetEnv("TMPDIR");
+# endif
+ internal_snprintf(buf, bufsize, "%s", tmpdir);
+#endif
+}
+
static void temp_file_name(char *buf, size_t bufsize, const char *prefix) {
#if SANITIZER_WINDOWS
buf[0] = '\0';
#else
const char *tmpdir = "/tmp";
#if SANITIZER_ANDROID
- // I don't know a way to query temp directory location on Android without
- // going through Java interfaces. The code below is not ideal, but should
- // work. May require "adb root", but it is needed for almost any use of ASan
- // on Android already.
- tmpdir = GetEnv("EXTERNAL_STORAGE");
+ tmpdir = GetEnv("TMPDIR");
#endif
internal_snprintf(buf, bufsize, "%s/%sXXXXXX", tmpdir, prefix);
ASSERT_TRUE(mkstemp(buf));
fd = OpenFile(tmpfile, WrOnly);
ASSERT_NE(fd, kInvalidFd);
-#if SANITIZER_POSIX && !SANITIZER_MAC
+#if SANITIZER_POSIX && !SANITIZER_APPLE
EXPECT_EQ(internal_lseek(fd, 0, SEEK_END), 0u);
#endif
uptr bytes_written = 0;
EXPECT_EQ(retval, (uptr)9);
}
+TEST(SanitizerCommon, InternalWideStringFunctions) {
+ const wchar_t *emptystr = L"";
+ const wchar_t *samesizestr = L"1234567";
+ const wchar_t *shortstr = L"123";
+ const wchar_t *longerstr = L"123456789";
+
+ ASSERT_EQ(internal_wcslen(emptystr), 0ul);
+ ASSERT_EQ(internal_wcslen(samesizestr), 7ul);
+ ASSERT_EQ(internal_wcslen(shortstr), 3ul);
+ ASSERT_EQ(internal_wcslen(longerstr), 9ul);
+
+ ASSERT_EQ(internal_wcsnlen(emptystr, 7), 0ul);
+ ASSERT_EQ(internal_wcsnlen(samesizestr, 7), 7ul);
+ ASSERT_EQ(internal_wcsnlen(shortstr, 7), 3ul);
+ ASSERT_EQ(internal_wcsnlen(longerstr, 7), 7ul);
+}
+
// FIXME: File manipulations are not yet supported on Windows
-#if SANITIZER_POSIX && !SANITIZER_MAC
+#if SANITIZER_POSIX && !SANITIZER_APPLE
TEST(SanitizerCommon, InternalMmapWithOffset) {
char tmpfile[128];
temp_file_name(tmpfile, sizeof(tmpfile),
internal_unlink(tmpfile);
}
#endif
+
+TEST(SanitizerCommon, ReportFile) {
+ SpinMutex report_file_mu;
+ ReportFile report_file = {&report_file_mu, kStderrFd, "", "", 0};
+ char tmpfile[128];
+ temp_file_name(tmpfile, sizeof(tmpfile),
+ "dir/sanitizer_common.reportfile.tmp.");
+ report_file.SetReportPath(tmpfile);
+ const char *path = report_file.GetReportPath();
+ EXPECT_EQ(internal_strncmp(tmpfile, path, strlen(tmpfile)), 0);
+ // This will close tmpfile.
+ report_file.SetReportPath("stderr");
+ Unlink(tmpfile);
+}
+
+TEST(SanitizerCommon, FileExists) {
+ char tmpfile[128];
+ temp_file_name(tmpfile, sizeof(tmpfile), "sanitizer_common.fileexists.tmp.");
+ fd_t fd = OpenFile(tmpfile, WrOnly);
+ ASSERT_NE(fd, kInvalidFd);
+ EXPECT_TRUE(FileExists(tmpfile));
+ CloseFile(fd);
+ Unlink(tmpfile);
+}
+
+TEST(SanitizerCommon, DirExists) {
+ char tmpdir[128];
+ get_temp_dir(tmpdir, sizeof(tmpdir));
+ EXPECT_TRUE(DirExists(tmpdir));
+}
--- /dev/null
+//===-- sanitizer_lzw_test.cpp ----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include "sanitizer_common/sanitizer_lzw.h"
+
+#include <iterator>
+
+#include "gtest/gtest.h"
+#include "sanitizer_hash.h"
+
+namespace __sanitizer {
+
+template <typename T>
+struct LzwTest : public ::testing::Test {
+ template <typename Generator>
+ void Run(size_t n, Generator gen) {
+ std::vector<T> data(n);
+ std::generate(data.begin(), data.end(), gen);
+
+ std::vector<u64> lzw;
+ LzwEncode<T>(data.begin(), data.end(), std::back_inserter(lzw));
+
+ std::vector<T> unlzw(data.size() * 2);
+ auto unlzw_end = LzwDecode<T>(lzw.begin(), lzw.end(), unlzw.data());
+ unlzw.resize(unlzw_end - unlzw.data());
+
+ EXPECT_EQ(data, unlzw);
+ }
+};
+
+static constexpr size_t kSizes[] = {0, 1, 2, 7, 13, 32, 129, 10000};
+
+using LzwTestTypes = ::testing::Types<u8, u16, u32, u64>;
+TYPED_TEST_SUITE(LzwTest, LzwTestTypes, );
+
+TYPED_TEST(LzwTest, Same) {
+ MurMur2Hash64Builder h(0);
+ for (size_t sz : kSizes) {
+ u64 v = 0;
+ for (size_t i = 0; i < 100 && !this->HasFailure(); ++i) {
+ this->Run(sz, [&] { return v; });
+ h.add(i);
+ v = h.get();
+ }
+ }
+}
+
+TYPED_TEST(LzwTest, Increment) {
+ MurMur2Hash64Builder h(0);
+ for (size_t sz : kSizes) {
+ u64 v = 0;
+ for (size_t i = 0; i < 100 && !this->HasFailure(); ++i) {
+ this->Run(sz, [&v] { return v++; });
+ h.add(i);
+ v = h.get();
+ }
+ }
+}
+
+TYPED_TEST(LzwTest, IncrementMod) {
+ MurMur2Hash64Builder h(0);
+ for (size_t sz : kSizes) {
+ u64 v = 0;
+ for (size_t i = 1; i < 16 && !this->HasFailure(); ++i) {
+ this->Run(sz, [&] { return v++ % i; });
+ h.add(i);
+ v = h.get();
+ }
+ }
+}
+
+TYPED_TEST(LzwTest, RandomLimited) {
+ for (size_t sz : kSizes) {
+ for (size_t i = 1; i < 1000 && !this->HasFailure(); i *= 2) {
+ u64 v = 0;
+ this->Run(sz, [&] {
+ MurMur2Hash64Builder h(v % i /* Keep unique set limited */);
+ v = h.get();
+ return v;
+ });
+ }
+ }
+}
+
+} // namespace __sanitizer
//===----------------------------------------------------------------------===//
#include "sanitizer_common/sanitizer_platform.h"
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
#include "sanitizer_common/sanitizer_mac.h"
} // namespace __sanitizer
-#endif // SANITIZER_MAC
+#endif // SANITIZER_APPLE
PTHREAD_JOIN(threads[i], 0);
}
-TEST(SanitizerCommon, BlockingMutex) {
- u64 mtxmem[1024] = {};
- BlockingMutex *mtx = new(mtxmem) BlockingMutex(LINKER_INITIALIZED);
- TestData<BlockingMutex> data(mtx);
+TEST(SanitizerCommon, Mutex) {
+ Mutex mtx;
+ TestData<Mutex> data(&mtx);
pthread_t threads[kThreads];
for (int i = 0; i < kThreads; i++)
- PTHREAD_CREATE(&threads[i], 0, lock_thread<BlockingMutex>, &data);
- for (int i = 0; i < kThreads; i++)
- PTHREAD_JOIN(threads[i], 0);
- check_locked(mtx);
+ PTHREAD_CREATE(&threads[i], 0, read_write_thread<Mutex>, &data);
+ for (int i = 0; i < kThreads; i++) PTHREAD_JOIN(threads[i], 0);
}
-TEST(SanitizerCommon, Mutex) {
+TEST(SanitizerCommon, MutexTry) {
Mutex mtx;
TestData<Mutex> data(&mtx);
pthread_t threads[kThreads];
for (int i = 0; i < kThreads; i++)
- PTHREAD_CREATE(&threads[i], 0, read_write_thread<Mutex>, &data);
+ PTHREAD_CREATE(&threads[i], 0, try_thread<Mutex>, &data);
for (int i = 0; i < kThreads; i++) PTHREAD_JOIN(threads[i], 0);
}
#include <string.h>
#include <limits.h>
-#ifdef __x86_64__
-# include <emmintrin.h>
-#endif
-
namespace __sanitizer {
TEST(Printf, Basic) {
TEST(Printf, OverflowStr) {
char buf[] = "123456789";
- uptr len = internal_snprintf(buf, 4, "%s", "abcdef"); // NOLINT
+ uptr len = internal_snprintf(buf, 4, "%s", "abcdef");
EXPECT_EQ(len, (uptr)6);
EXPECT_STREQ("abc", buf);
EXPECT_EQ(buf[3], 0);
TEST(Printf, OverflowInt) {
char buf[] = "123456789";
- internal_snprintf(buf, 4, "%d", -123456789); // NOLINT
+ internal_snprintf(buf, 4, "%d", -123456789);
EXPECT_STREQ("-12", buf);
EXPECT_EQ(buf[3], 0);
EXPECT_EQ(buf[4], '5');
} else {
val = (uptr)0x123456789ULL;
}
- internal_snprintf(buf, 4, "a%zx", val); // NOLINT
+ internal_snprintf(buf, 4, "a%zx", val);
EXPECT_STREQ("a12", buf);
EXPECT_EQ(buf[3], 0);
EXPECT_EQ(buf[4], '5');
} else {
p = (void*)0x123456789ULL;
}
- internal_snprintf(buf, 4, "%p", p); // NOLINT
+ internal_snprintf(buf, 4, "%p", p);
EXPECT_STREQ("0x0", buf);
EXPECT_EQ(buf[3], 0);
EXPECT_EQ(buf[4], '5');
TestAgainstLibc<int>("%d-%d", INT_MIN, INT_MAX);
TestAgainstLibc<unsigned>("%u-%u", 0, UINT_MAX);
TestAgainstLibc<unsigned>("%x-%x", 0, UINT_MAX);
+ TestAgainstLibc<long>("%ld-%ld", LONG_MIN, LONG_MAX);
+ TestAgainstLibc<unsigned long>("%lu-%lu", 0, LONG_MAX);
+ TestAgainstLibc<unsigned long>("%lx-%lx", 0, LONG_MAX);
#if !defined(_WIN32)
// %z* format doesn't seem to be supported by MSVS.
TestAgainstLibc<long>("%zd-%zd", LONG_MIN, LONG_MAX);
EXPECT_STREQ("12345 ", buf);
// Check that width does not overflow the smaller buffer, although
// 10 chars is requested, it stops at the buffer size, 8.
- len = internal_snprintf(buf, 8, "%-10s", "12345"); // NOLINT
+ len = internal_snprintf(buf, 8, "%-10s", "12345");
EXPECT_EQ(10U, len); // The required size reported.
EXPECT_STREQ("12345 ", buf);
}
-#ifdef __x86_64__
-TEST(Printf, M128) {
- __m128i v = _mm_set_epi32(0x12345678, 0x0a0a0a0a, 0xb0b0b0b0, 0xaabbccdd);
- char buf[128];
- internal_snprintf(buf, sizeof(buf), "%V", PRINTF_128(v));
- EXPECT_STREQ("ddccbbaab0b0b0b00a0a0a0a78563412", buf);
- v = _mm_cvtsi32_si128(0x12345678);
- internal_snprintf(buf, sizeof(buf), "%V", PRINTF_128(v));
- EXPECT_STREQ("78563412000000000000000000000000", buf);
- internal_snprintf(buf, sizeof(buf), "%d %V", 0, PRINTF_128(v));
- EXPECT_STREQ("0 78563412000000000000000000000000", buf);
-}
-#endif
-
} // namespace __sanitizer
//===----------------------------------------------------------------------===//
#if !defined(_WIN32) // There are no /proc/maps on Windows.
-#include "sanitizer_common/sanitizer_procmaps.h"
-#include "gtest/gtest.h"
+# include "sanitizer_common/sanitizer_procmaps.h"
-#include <stdlib.h>
+# include <stdlib.h>
+# include <string.h>
+
+# include <vector>
+
+# include "gtest/gtest.h"
static void noop() {}
extern const char *argv0;
}
TEST(MemoryMapping, LoadedModuleArchAndUUID) {
- if (SANITIZER_MAC) {
+ if (SANITIZER_APPLE) {
MemoryMappingLayout memory_mapping(false);
const uptr kMaxModules = 100;
InternalMmapVector<LoadedModule> modules;
memory_mapping.DumpListOfModules(&modules);
for (uptr i = 0; i < modules.size(); ++i) {
ModuleArch arch = modules[i].arch();
- // Darwin unit tests are only run on i386/x86_64/x86_64h.
+ // Darwin unit tests are only run on i386/x86_64/x86_64h/arm64.
if (SANITIZER_WORDSIZE == 32) {
EXPECT_EQ(arch, kModuleArchI386);
} else if (SANITIZER_WORDSIZE == 64) {
- EXPECT_TRUE(arch == kModuleArchX86_64 || arch == kModuleArchX86_64H);
+ EXPECT_TRUE(arch == kModuleArchX86_64 || arch == kModuleArchX86_64H || arch == kModuleArchARM64);
}
const u8 *uuid = modules[i].uuid();
u8 null_uuid[kModuleUUIDSize] = {0};
}
}
+# if (SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \
+ SANITIZER_SOLARIS) && \
+ defined(_LP64)
+const char *const parse_unix_input = R"(
+7fb9862f1000-7fb9862f3000 rw-p 00000000 00:00 0
+Size: 8 kB
+Rss: 4 kB
+7fb9864ae000-7fb9864b1000 r--p 001ba000 fd:01 22413919 /lib/x86_64-linux-gnu/libc-2.32.so
+Size: 12 kB
+Rss: 12 kB
+)";
+
+TEST(MemoryMapping, ParseUnixMemoryProfile) {
+ struct entry {
+ uptr p;
+ uptr rss;
+ bool file;
+ };
+ typedef std::vector<entry> entries_t;
+ entries_t entries;
+ std::vector<char> input(parse_unix_input,
+ parse_unix_input + strlen(parse_unix_input));
+ ParseUnixMemoryProfile(
+ [](uptr p, uptr rss, bool file, uptr *mem) {
+ reinterpret_cast<entries_t *>(mem)->push_back({p, rss, file});
+ },
+ reinterpret_cast<uptr *>(&entries), &input[0], input.size());
+ EXPECT_EQ(entries.size(), 2ul);
+ EXPECT_EQ(entries[0].p, 0x7fb9862f1000ul);
+ EXPECT_EQ(entries[0].rss, 4ul << 10);
+ EXPECT_EQ(entries[0].file, false);
+ EXPECT_EQ(entries[1].p, 0x7fb9864ae000ul);
+ EXPECT_EQ(entries[1].rss, 12ul << 10);
+ EXPECT_EQ(entries[1].file, true);
+}
+
+TEST(MemoryMapping, ParseUnixMemoryProfileTruncated) {
+ // ParseUnixMemoryProfile used to crash on truncated inputs.
+ // This test allocates 2 pages, protects the second one
+ // and places the input at the very end of the first page
+ // to test for over-reads.
+ uptr page = GetPageSizeCached();
+ char *mem = static_cast<char *>(
+ MmapOrDie(2 * page, "ParseUnixMemoryProfileTruncated"));
+ EXPECT_TRUE(MprotectNoAccess(reinterpret_cast<uptr>(mem + page), page));
+ const uptr len = strlen(parse_unix_input);
+ for (uptr i = 0; i < len; i++) {
+ char *smaps = mem + page - len + i;
+ memcpy(smaps, parse_unix_input, len - i);
+ ParseUnixMemoryProfile([](uptr p, uptr rss, bool file, uptr *mem) {},
+ nullptr, smaps, len - i);
+ }
+ UnmapOrDie(mem, 2 * page);
+}
+# endif
+
} // namespace __sanitizer
#endif // !defined(_WIN32)
--- /dev/null
+//===-- sanitizer_stack_store_test.cpp --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include "sanitizer_common/sanitizer_stack_store.h"
+
+#include <algorithm>
+#include <numeric>
+#include <vector>
+
+#include "gtest/gtest.h"
+#include "sanitizer_atomic.h"
+#include "sanitizer_hash.h"
+#include "sanitizer_stacktrace.h"
+
+namespace __sanitizer {
+
+class StackStoreTest : public testing::Test {
+ protected:
+ void SetUp() override {}
+ void TearDown() override { store_.TestOnlyUnmap(); }
+
+ template <typename Fn>
+ void ForEachTrace(Fn fn, uptr n = 1000000) {
+ std::vector<uptr> frames(kStackTraceMax);
+ std::iota(frames.begin(), frames.end(), 0x100000);
+ MurMur2HashBuilder h(0);
+ for (uptr i = 0; i < n; ++i) {
+ h.add(i);
+ u32 size = h.get() % kStackTraceMax;
+ h.add(i);
+ uptr tag = h.get() % 256;
+ StackTrace s(frames.data(), size, tag);
+ if (!s.size && !s.tag)
+ continue;
+ fn(s);
+ if (HasFailure())
+ return;
+ std::next_permutation(frames.begin(), frames.end());
+ };
+ }
+
+ using BlockInfo = StackStore::BlockInfo;
+
+ uptr GetTotalFramesCount() const {
+ return atomic_load_relaxed(&store_.total_frames_);
+ }
+
+ uptr CountReadyToPackBlocks() {
+ uptr res = 0;
+ for (BlockInfo& b : store_.blocks_) res += b.Stored(0);
+ return res;
+ }
+
+ uptr CountPackedBlocks() const {
+ uptr res = 0;
+ for (const BlockInfo& b : store_.blocks_) res += b.IsPacked();
+ return res;
+ }
+
+ uptr IdToOffset(StackStore::Id id) const { return store_.IdToOffset(id); }
+
+ static constexpr uptr kBlockSizeFrames = StackStore::kBlockSizeFrames;
+ static constexpr uptr kBlockSizeBytes = StackStore::kBlockSizeBytes;
+
+ StackStore store_ = {};
+};
+
+TEST_F(StackStoreTest, Empty) {
+ uptr before = store_.Allocated();
+ uptr pack = 0;
+ EXPECT_EQ(0u, store_.Store({}, &pack));
+ uptr after = store_.Allocated();
+ EXPECT_EQ(before, after);
+}
+
+TEST_F(StackStoreTest, Basic) {
+ std::vector<StackStore::Id> ids;
+ ForEachTrace([&](const StackTrace& s) {
+ uptr pack = 0;
+ ids.push_back(store_.Store(s, &pack));
+ });
+
+ auto id = ids.begin();
+ ForEachTrace([&](const StackTrace& s) {
+ StackTrace trace = store_.Load(*(id++));
+ EXPECT_EQ(s.size, trace.size);
+ EXPECT_EQ(s.tag, trace.tag);
+ EXPECT_EQ(std::vector<uptr>(s.trace, s.trace + s.size),
+ std::vector<uptr>(trace.trace, trace.trace + trace.size));
+ });
+}
+
+TEST_F(StackStoreTest, Allocated) {
+ EXPECT_LE(store_.Allocated(), 0x100000u);
+ std::vector<StackStore::Id> ids;
+ ForEachTrace([&](const StackTrace& s) {
+ uptr pack = 0;
+ ids.push_back(store_.Store(s, &pack));
+ });
+ EXPECT_NEAR(store_.Allocated(), FIRST_32_SECOND_64(500000000u, 1000000000u),
+ FIRST_32_SECOND_64(50000000u, 100000000u));
+ store_.TestOnlyUnmap();
+ EXPECT_LE(store_.Allocated(), 0x100000u);
+}
+
+TEST_F(StackStoreTest, ReadyToPack) {
+ uptr next_pack = kBlockSizeFrames;
+ uptr total_ready = 0;
+ ForEachTrace(
+ [&](const StackTrace& s) {
+ uptr pack = 0;
+ StackStore::Id id = store_.Store(s, &pack);
+ uptr end_idx = IdToOffset(id) + 1 + s.size;
+ if (end_idx >= next_pack) {
+ EXPECT_EQ(1u, pack);
+ next_pack += kBlockSizeFrames;
+ } else {
+ EXPECT_EQ(0u, pack);
+ }
+ total_ready += pack;
+ EXPECT_EQ(CountReadyToPackBlocks(), total_ready);
+ },
+ 100000);
+ EXPECT_EQ(GetTotalFramesCount() / kBlockSizeFrames, total_ready);
+}
+
+struct StackStorePackTest : public StackStoreTest,
+ public ::testing::WithParamInterface<
+ std::pair<StackStore::Compression, uptr>> {};
+
+INSTANTIATE_TEST_SUITE_P(
+ PackUnpacks, StackStorePackTest,
+ ::testing::ValuesIn({
+ StackStorePackTest::ParamType(StackStore::Compression::Delta,
+ FIRST_32_SECOND_64(2, 6)),
+ StackStorePackTest::ParamType(StackStore::Compression::LZW,
+ FIRST_32_SECOND_64(60, 125)),
+ }));
+
+TEST_P(StackStorePackTest, PackUnpack) {
+ std::vector<StackStore::Id> ids;
+ StackStore::Compression type = GetParam().first;
+ uptr expected_ratio = GetParam().second;
+ ForEachTrace([&](const StackTrace& s) {
+ uptr pack = 0;
+ ids.push_back(store_.Store(s, &pack));
+ if (pack) {
+ uptr before = store_.Allocated();
+ uptr diff = store_.Pack(type);
+ uptr after = store_.Allocated();
+ EXPECT_EQ(before - after, diff);
+ EXPECT_LT(after, before);
+ EXPECT_GE(kBlockSizeBytes / (kBlockSizeBytes - (before - after)),
+ expected_ratio);
+ }
+ });
+ uptr packed_blocks = CountPackedBlocks();
+ // Unpack random block.
+ store_.Load(kBlockSizeFrames * 7 + 123);
+ EXPECT_EQ(packed_blocks - 1, CountPackedBlocks());
+
+ // Unpack all blocks.
+ auto id = ids.begin();
+ ForEachTrace([&](const StackTrace& s) {
+ StackTrace trace = store_.Load(*(id++));
+ EXPECT_EQ(s.size, trace.size);
+ EXPECT_EQ(s.tag, trace.tag);
+ EXPECT_EQ(std::vector<uptr>(s.trace, s.trace + s.size),
+ std::vector<uptr>(trace.trace, trace.trace + trace.size));
+ });
+ EXPECT_EQ(0u, CountPackedBlocks());
+
+ EXPECT_EQ(0u, store_.Pack(type));
+ EXPECT_EQ(0u, CountPackedBlocks());
+}
+
+TEST_P(StackStorePackTest, Failed) {
+ MurMur2Hash64Builder h(0);
+ StackStore::Compression type = GetParam().first;
+ std::vector<uptr> frames(200);
+ for (uptr i = 0; i < kBlockSizeFrames * 4 / frames.size(); ++i) {
+ for (uptr& f : frames) {
+ h.add(1);
+ // Make it difficult to pack.
+ f = h.get();
+ }
+ uptr pack = 0;
+ store_.Store(StackTrace(frames.data(), frames.size()), &pack);
+ if (pack)
+ EXPECT_EQ(0u, store_.Pack(type));
+ }
+
+ EXPECT_EQ(0u, CountPackedBlocks());
+}
+
+} // namespace __sanitizer
//===----------------------------------------------------------------------===//
#include "sanitizer_common/sanitizer_stackdepot.h"
+#include <atomic>
+#include <numeric>
+#include <regex>
+#include <sstream>
+#include <string>
+#include <thread>
+
#include "gtest/gtest.h"
#include "sanitizer_common/sanitizer_internal_defs.h"
#include "sanitizer_common/sanitizer_libc.h"
namespace __sanitizer {
-TEST(SanitizerCommon, StackDepotBasic) {
+class StackDepotTest : public testing::Test {
+ protected:
+ void SetUp() override { StackDepotTestOnlyUnmap(); }
+ void TearDown() override {
+ StackDepotStats stack_depot_stats = StackDepotGetStats();
+ Printf("StackDepot: %zd ids; %zdM allocated\n",
+ stack_depot_stats.n_uniq_ids, stack_depot_stats.allocated >> 20);
+ StackDepotTestOnlyUnmap();
+ }
+};
+
+TEST_F(StackDepotTest, Basic) {
uptr array[] = {1, 2, 3, 4, 5};
StackTrace s1(array, ARRAY_SIZE(array));
u32 i1 = StackDepotPut(s1);
EXPECT_EQ(0, internal_memcmp(stack.trace, array, sizeof(array)));
}
-TEST(SanitizerCommon, StackDepotAbsent) {
+TEST_F(StackDepotTest, Absent) {
StackTrace stack = StackDepotGet((1 << 30) - 1);
EXPECT_EQ((uptr*)0, stack.trace);
}
-TEST(SanitizerCommon, StackDepotEmptyStack) {
+TEST_F(StackDepotTest, EmptyStack) {
u32 i1 = StackDepotPut(StackTrace());
StackTrace stack = StackDepotGet(i1);
EXPECT_EQ((uptr*)0, stack.trace);
}
-TEST(SanitizerCommon, StackDepotZeroId) {
+TEST_F(StackDepotTest, ZeroId) {
StackTrace stack = StackDepotGet(0);
EXPECT_EQ((uptr*)0, stack.trace);
}
-TEST(SanitizerCommon, StackDepotSame) {
+TEST_F(StackDepotTest, Same) {
uptr array[] = {1, 2, 3, 4, 6};
StackTrace s1(array, ARRAY_SIZE(array));
u32 i1 = StackDepotPut(s1);
EXPECT_EQ(0, internal_memcmp(stack.trace, array, sizeof(array)));
}
-TEST(SanitizerCommon, StackDepotSeveral) {
+TEST_F(StackDepotTest, Several) {
uptr array1[] = {1, 2, 3, 4, 7};
StackTrace s1(array1, ARRAY_SIZE(array1));
u32 i1 = StackDepotPut(s1);
EXPECT_NE(i1, i2);
}
-#if SANITIZER_WINDOWS
-// CaptureStderr does not work on Windows.
-#define Maybe_StackDepotPrint DISABLED_StackDepotPrint
-#else
-#define Maybe_StackDepotPrint StackDepotPrint
-#endif
-TEST(SanitizerCommon, Maybe_StackDepotPrint) {
+TEST_F(StackDepotTest, Print) {
uptr array1[] = {0x111, 0x222, 0x333, 0x444, 0x777};
StackTrace s1(array1, ARRAY_SIZE(array1));
u32 i1 = StackDepotPut(s1);
StackTrace s2(array2, ARRAY_SIZE(array2));
u32 i2 = StackDepotPut(s2);
EXPECT_NE(i1, i2);
- EXPECT_EXIT((StackDepotPrintAll(), exit(0)), ::testing::ExitedWithCode(0),
- "Stack for id .*#0 0x1.*#1 0x2.*#2 0x3.*#3 0x4.*#4 0x7.*");
+
+ auto fix_regex = [](const std::string& s) -> std::string {
+ if (!SANITIZER_WINDOWS)
+ return s;
+ return std::regex_replace(s, std::regex("\\.\\*"), ".*\\n.*");
+ };
+ EXPECT_EXIT(
+ (StackDepotPrintAll(), exit(0)), ::testing::ExitedWithCode(0),
+ fix_regex("Stack for id .*#0 0x1.*#1 0x2.*#2 0x3.*#3 0x4.*#4 0x7.*"));
EXPECT_EXIT(
(StackDepotPrintAll(), exit(0)), ::testing::ExitedWithCode(0),
- "Stack for id .*#0 0x1.*#1 0x2.*#2 0x3.*#3 0x4.*#4 0x8.*#5 0x9.*");
+ fix_regex(
+ "Stack for id .*#0 0x1.*#1 0x2.*#2 0x3.*#3 0x4.*#4 0x8.*#5 0x9.*"));
}
-TEST(SanitizerCommon, StackDepotReverseMap) {
- uptr array1[] = {1, 2, 3, 4, 5};
- uptr array2[] = {7, 1, 3, 0};
- uptr array3[] = {10, 2, 5, 3};
- uptr array4[] = {1, 3, 2, 5};
- u32 ids[4] = {0};
- StackTrace s1(array1, ARRAY_SIZE(array1));
- StackTrace s2(array2, ARRAY_SIZE(array2));
- StackTrace s3(array3, ARRAY_SIZE(array3));
- StackTrace s4(array4, ARRAY_SIZE(array4));
- ids[0] = StackDepotPut(s1);
- ids[1] = StackDepotPut(s2);
- ids[2] = StackDepotPut(s3);
- ids[3] = StackDepotPut(s4);
-
- StackDepotReverseMap map;
-
- for (uptr i = 0; i < 4; i++) {
- StackTrace stack = StackDepotGet(ids[i]);
- StackTrace from_map = map.Get(ids[i]);
- EXPECT_EQ(stack.size, from_map.size);
- EXPECT_EQ(stack.trace, from_map.trace);
+TEST_F(StackDepotTest, PrintNoLock) {
+ u32 n = 2000;
+ std::vector<u32> idx2id(n);
+ for (u32 i = 0; i < n; ++i) {
+ uptr array[] = {0x111, 0x222, i, 0x444, 0x777};
+ StackTrace s(array, ARRAY_SIZE(array));
+ idx2id[i] = StackDepotPut(s);
+ }
+ StackDepotPrintAll();
+ for (u32 i = 0; i < n; ++i) {
+ uptr array[] = {0x111, 0x222, i, 0x444, 0x777};
+ StackTrace s(array, ARRAY_SIZE(array));
+ CHECK_EQ(idx2id[i], StackDepotPut(s));
}
}
+static struct StackDepotBenchmarkParams {
+ int UniqueStacksPerThread;
+ int RepeatPerThread;
+ int Threads;
+ bool UniqueThreads;
+ bool UseCount;
+} params[] = {
+ // All traces are unique, very unusual.
+ {10000000, 1, 1, false, false},
+ {8000000, 1, 4, false, false},
+ {8000000, 1, 16, false, false},
+ // Probably most realistic sets.
+ {3000000, 10, 1, false, false},
+ {3000000, 10, 4, false, false},
+ {3000000, 10, 16, false, false},
+ // Update use count as msan/dfsan.
+ {3000000, 10, 1, false, true},
+ {3000000, 10, 4, false, true},
+ {3000000, 10, 16, false, true},
+ // Unrealistic, as above, but traces are unique inside of thread.
+ {4000000, 1, 4, true, false},
+ {2000000, 1, 16, true, false},
+ {2000000, 10, 4, true, false},
+ {500000, 10, 16, true, false},
+ {1500000, 10, 4, true, true},
+ {800000, 10, 16, true, true},
+};
+
+static std::string PrintStackDepotBenchmarkParams(
+ const testing::TestParamInfo<StackDepotBenchmarkParams>& info) {
+ std::stringstream name;
+ name << info.param.UniqueStacksPerThread << "_" << info.param.RepeatPerThread
+ << "_" << info.param.Threads << (info.param.UseCount ? "_UseCount" : "")
+ << (info.param.UniqueThreads ? "_UniqueThreads" : "");
+ return name.str();
+}
+
+class StackDepotBenchmark
+ : public StackDepotTest,
+ public testing::WithParamInterface<StackDepotBenchmarkParams> {};
+
+// Test which can be used as a simple benchmark. It's disabled to avoid slowing
+// down check-sanitizer.
+// Usage: Sanitizer-<ARCH>-Test --gtest_also_run_disabled_tests \
+// '--gtest_filter=*Benchmark*'
+TEST_P(StackDepotBenchmark, DISABLED_Benchmark) {
+ auto Param = GetParam();
+ std::atomic<unsigned int> here = {};
+
+ auto thread = [&](int idx) {
+ here++;
+ while (here < Param.UniqueThreads) std::this_thread::yield();
+
+ std::vector<uptr> frames(64);
+ for (int r = 0; r < Param.RepeatPerThread; ++r) {
+ std::iota(frames.begin(), frames.end(), idx + 1);
+ for (int i = 0; i < Param.UniqueStacksPerThread; ++i) {
+ StackTrace s(frames.data(), frames.size());
+ auto h = StackDepotPut_WithHandle(s);
+ if (Param.UseCount)
+ h.inc_use_count_unsafe();
+ std::next_permutation(frames.begin(), frames.end());
+ };
+ }
+ };
+
+ std::vector<std::thread> threads;
+ for (int i = 0; i < Param.Threads; ++i)
+ threads.emplace_back(thread, Param.UniqueThreads * i);
+ for (auto& t : threads) t.join();
+}
+
+INSTANTIATE_TEST_SUITE_P(StackDepotBenchmarkSuite, StackDepotBenchmark,
+ testing::ValuesIn(params),
+ PrintStackDepotBenchmarkParams);
+
} // namespace __sanitizer
EXPECT_STREQ("(/path/to/module+0x200)", str.data());
str.clear();
+ RenderFrame(&str, "%b", frame_no, info.address, &info, false);
+ EXPECT_STREQ("", str.data());
+ str.clear();
+
+ info.uuid_size = 2;
+ info.uuid[0] = 0x55;
+ info.uuid[1] = 0x66;
+
+ RenderFrame(&str, "%M", frame_no, info.address, &info, false);
+ EXPECT_NE(nullptr, internal_strstr(str.data(), "(module+0x"));
+ EXPECT_NE(nullptr, internal_strstr(str.data(), "200"));
+ EXPECT_NE(nullptr, internal_strstr(str.data(), "BuildId: 5566"));
+ str.clear();
+
+ RenderFrame(&str, "%L", frame_no, info.address, &info, false);
+ EXPECT_STREQ("(/path/to/module+0x200) (BuildId: 5566)", str.data());
+ str.clear();
+
+ RenderFrame(&str, "%b", frame_no, info.address, &info, false);
+ EXPECT_STREQ("(BuildId: 5566)", str.data());
+ str.clear();
+
info.function = internal_strdup("my_function");
RenderFrame(&str, "%F", frame_no, info.address, &info, false);
EXPECT_STREQ("in my_function", str.data());
uhwptr fake_bottom;
BufferedStackTrace trace;
-#if defined(__riscv)
+#if defined(__loongarch__) || defined(__riscv)
const uptr kFpOffset = 4;
const uptr kBpOffset = 2;
#else
StackTrace::GetCurrentPc(),
};
for (uptr i = 0; i < ARRAY_SIZE(pcs); i++)
- Printf("pc%zu: %p\n", i, pcs[i]);
+ Printf("pc%zu: 0x%zx\n", i, pcs[i]);
for (uptr i = 1; i < ARRAY_SIZE(pcs); i++) {
EXPECT_GT(pcs[i], pcs[0]);
EXPECT_LT(pcs[i], pcs[0] + 1000);
//
//===----------------------------------------------------------------------===//
-#include "sanitizer_common/sanitizer_platform.h"
-#if SANITIZER_LINUX && defined(__x86_64__)
-
#include "sanitizer_common/sanitizer_stoptheworld.h"
-#include "gtest/gtest.h"
-#include "sanitizer_common/sanitizer_libc.h"
-#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_platform.h"
+#if (SANITIZER_LINUX || SANITIZER_WINDOWS) && defined(__x86_64__)
-#include <pthread.h>
-#include <sched.h>
+# include <atomic>
+# include <mutex>
+# include <thread>
+
+# include "gtest/gtest.h"
+# include "sanitizer_common/sanitizer_common.h"
+# include "sanitizer_common/sanitizer_libc.h"
namespace __sanitizer {
-static pthread_mutex_t incrementer_thread_exit_mutex;
+static std::mutex mutex;
struct CallbackArgument {
- volatile int counter;
- volatile bool threads_stopped;
- volatile bool callback_executed;
- CallbackArgument()
- : counter(0),
- threads_stopped(false),
- callback_executed(false) {}
+ std::atomic_int counter = {};
+ std::atomic_bool threads_stopped = {};
+ std::atomic_bool callback_executed = {};
};
-void *IncrementerThread(void *argument) {
- CallbackArgument *callback_argument = (CallbackArgument *)argument;
+void IncrementerThread(CallbackArgument &callback_argument) {
while (true) {
- __sync_fetch_and_add(&callback_argument->counter, 1);
- if (pthread_mutex_trylock(&incrementer_thread_exit_mutex) == 0) {
- pthread_mutex_unlock(&incrementer_thread_exit_mutex);
- return NULL;
- } else {
- sched_yield();
+ callback_argument.counter++;
+
+ if (mutex.try_lock()) {
+ mutex.unlock();
+ return;
}
+
+ std::this_thread::yield();
}
}
void *argument) {
CallbackArgument *callback_argument = (CallbackArgument *)argument;
callback_argument->callback_executed = true;
- int counter_at_init = __sync_fetch_and_add(&callback_argument->counter, 0);
+ int counter_at_init = callback_argument->counter;
for (uptr i = 0; i < 1000; i++) {
- sched_yield();
- if (__sync_fetch_and_add(&callback_argument->counter, 0) !=
- counter_at_init) {
+ std::this_thread::yield();
+ if (callback_argument->counter != counter_at_init) {
callback_argument->threads_stopped = false;
return;
}
}
TEST(StopTheWorld, SuspendThreadsSimple) {
- pthread_mutex_init(&incrementer_thread_exit_mutex, NULL);
CallbackArgument argument;
- pthread_t thread_id;
- int pthread_create_result;
- pthread_mutex_lock(&incrementer_thread_exit_mutex);
- pthread_create_result = pthread_create(&thread_id, NULL, IncrementerThread,
- &argument);
- ASSERT_EQ(0, pthread_create_result);
- StopTheWorld(&Callback, &argument);
- pthread_mutex_unlock(&incrementer_thread_exit_mutex);
+ std::thread thread;
+ {
+ std::lock_guard<std::mutex> lock(mutex);
+ thread = std::thread(IncrementerThread, std::ref(argument));
+ StopTheWorld(&Callback, &argument);
+ }
EXPECT_TRUE(argument.callback_executed);
EXPECT_TRUE(argument.threads_stopped);
// argument is on stack, so we have to wait for the incrementer thread to
// terminate before we can return from this function.
- ASSERT_EQ(0, pthread_join(thread_id, NULL));
- pthread_mutex_destroy(&incrementer_thread_exit_mutex);
+ ASSERT_NO_THROW(thread.join());
}
// A more comprehensive test where we spawn a bunch of threads while executing
// StopTheWorld in parallel.
static const uptr kThreadCount = 50;
-static const uptr kStopWorldAfter = 10; // let this many threads spawn first
-
-static pthread_mutex_t advanced_incrementer_thread_exit_mutex;
+static const uptr kStopWorldAfter = 10; // let this many threads spawn first
struct AdvancedCallbackArgument {
- volatile uptr thread_index;
- volatile int counters[kThreadCount];
- pthread_t thread_ids[kThreadCount];
- volatile bool threads_stopped;
- volatile bool callback_executed;
- volatile bool fatal_error;
- AdvancedCallbackArgument()
- : thread_index(0),
- threads_stopped(false),
- callback_executed(false),
- fatal_error(false) {}
+ std::atomic_uintptr_t thread_index = {};
+ std::atomic_int counters[kThreadCount] = {};
+ std::thread threads[kThreadCount];
+ std::atomic_bool threads_stopped = {};
+ std::atomic_bool callback_executed = {};
};
-void *AdvancedIncrementerThread(void *argument) {
- AdvancedCallbackArgument *callback_argument =
- (AdvancedCallbackArgument *)argument;
- uptr this_thread_index = __sync_fetch_and_add(
- &callback_argument->thread_index, 1);
+void AdvancedIncrementerThread(AdvancedCallbackArgument &callback_argument) {
+ uptr this_thread_index = callback_argument.thread_index++;
// Spawn the next thread.
- int pthread_create_result;
if (this_thread_index + 1 < kThreadCount) {
- pthread_create_result =
- pthread_create(&callback_argument->thread_ids[this_thread_index + 1],
- NULL, AdvancedIncrementerThread, argument);
- // Cannot use ASSERT_EQ in non-void-returning functions. If there's a
- // problem, defer failing to the main thread.
- if (pthread_create_result != 0) {
- callback_argument->fatal_error = true;
- __sync_fetch_and_add(&callback_argument->thread_index,
- kThreadCount - callback_argument->thread_index);
- }
+ callback_argument.threads[this_thread_index + 1] =
+ std::thread(AdvancedIncrementerThread, std::ref(callback_argument));
}
// Do the actual work.
while (true) {
- __sync_fetch_and_add(&callback_argument->counters[this_thread_index], 1);
- if (pthread_mutex_trylock(&advanced_incrementer_thread_exit_mutex) == 0) {
- pthread_mutex_unlock(&advanced_incrementer_thread_exit_mutex);
- return NULL;
- } else {
- sched_yield();
+ callback_argument.counters[this_thread_index]++;
+ if (mutex.try_lock()) {
+ mutex.unlock();
+ return;
}
+
+ std::this_thread::yield();
}
}
void AdvancedCallback(const SuspendedThreadsList &suspended_threads_list,
- void *argument) {
+ void *argument) {
AdvancedCallbackArgument *callback_argument =
(AdvancedCallbackArgument *)argument;
callback_argument->callback_executed = true;
int counters_at_init[kThreadCount];
for (uptr j = 0; j < kThreadCount; j++)
- counters_at_init[j] = __sync_fetch_and_add(&callback_argument->counters[j],
- 0);
+ counters_at_init[j] = callback_argument->counters[j];
for (uptr i = 0; i < 10; i++) {
- sched_yield();
+ std::this_thread::yield();
for (uptr j = 0; j < kThreadCount; j++)
- if (__sync_fetch_and_add(&callback_argument->counters[j], 0) !=
- counters_at_init[j]) {
+ if (callback_argument->counters[j] != counters_at_init[j]) {
callback_argument->threads_stopped = false;
return;
}
}
TEST(StopTheWorld, SuspendThreadsAdvanced) {
- pthread_mutex_init(&advanced_incrementer_thread_exit_mutex, NULL);
AdvancedCallbackArgument argument;
- pthread_mutex_lock(&advanced_incrementer_thread_exit_mutex);
- int pthread_create_result;
- pthread_create_result = pthread_create(&argument.thread_ids[0], NULL,
- AdvancedIncrementerThread,
- &argument);
- ASSERT_EQ(0, pthread_create_result);
- // Wait for several threads to spawn before proceeding.
- while (__sync_fetch_and_add(&argument.thread_index, 0) < kStopWorldAfter)
- sched_yield();
- StopTheWorld(&AdvancedCallback, &argument);
- EXPECT_TRUE(argument.callback_executed);
- EXPECT_TRUE(argument.threads_stopped);
-
- // Wait for all threads to spawn before we start terminating them.
- while (__sync_fetch_and_add(&argument.thread_index, 0) < kThreadCount)
- sched_yield();
- ASSERT_FALSE(argument.fatal_error); // a pthread_create has failed
+ {
+ std::lock_guard<std::mutex> lock(mutex);
+ argument.threads[0] =
+ std::thread(AdvancedIncrementerThread, std::ref(argument));
+ // Wait for several threads to spawn before proceeding.
+ while (argument.thread_index < kStopWorldAfter) std::this_thread::yield();
+ StopTheWorld(&AdvancedCallback, &argument);
+ EXPECT_TRUE(argument.callback_executed);
+ EXPECT_TRUE(argument.threads_stopped);
+
+ // Wait for all threads to spawn before we start terminating them.
+ while (argument.thread_index < kThreadCount) std::this_thread::yield();
+ }
// Signal the threads to terminate.
- pthread_mutex_unlock(&advanced_incrementer_thread_exit_mutex);
- for (uptr i = 0; i < kThreadCount; i++)
- ASSERT_EQ(0, pthread_join(argument.thread_ids[i], NULL));
- pthread_mutex_destroy(&advanced_incrementer_thread_exit_mutex);
+ for (auto &t : argument.threads) t.join();
}
static void SegvCallback(const SuspendedThreadsList &suspended_threads_list,
void *argument) {
- *(volatile int*)0x1234 = 0;
+ *(volatile int *)0x1234 = 0;
}
-TEST(StopTheWorld, SegvInCallback) {
+# if SANITIZER_WINDOWS
+# define MAYBE_SegvInCallback DISABLED_SegvInCallback
+# else
+# define MAYBE_SegvInCallback SegvInCallback
+# endif
+
+TEST(StopTheWorld, MAYBE_SegvInCallback) {
// Test that tracer thread catches SIGSEGV.
StopTheWorld(&SegvCallback, NULL);
}
namespace __sanitizer {
-static BlockingMutex tctx_allocator_lock(LINKER_INITIALIZED);
+static Mutex tctx_allocator_lock;
static LowLevelAllocator tctx_allocator;
template<typename TCTX>
static ThreadContextBase *GetThreadContext(u32 tid) {
- BlockingMutexLock l(&tctx_allocator_lock);
+ Lock l(&tctx_allocator_lock);
return new(tctx_allocator) TCTX(tid);
}
//
//===----------------------------------------------------------------------===//
#include "sanitizer_common/sanitizer_type_traits.h"
+
+#include <vector>
+
#include "gtest/gtest.h"
#include "sanitizer_common/sanitizer_internal_defs.h"
-using namespace __sanitizer;
+namespace __sanitizer {
TEST(SanitizerCommon, IsSame) {
ASSERT_TRUE((is_same<unsigned, unsigned>::value));
ASSERT_TRUE((is_same<int, conditional<true, int, double>::type>::value));
ASSERT_TRUE((is_same<double, conditional<false, int, double>::type>::value));
}
+
+TEST(SanitizerCommon, RemoveReference) {
+ ASSERT_TRUE((is_same<int, remove_reference<int>::type>::value));
+ ASSERT_TRUE((is_same<const int, remove_reference<const int>::type>::value));
+ ASSERT_TRUE((is_same<int, remove_reference<int&>::type>::value));
+ ASSERT_TRUE((is_same<const int, remove_reference<const int&>::type>::value));
+ ASSERT_TRUE((is_same<int, remove_reference<int&&>::type>::value));
+}
+
+TEST(SanitizerCommon, Move) {
+ std::vector<int> v = {1, 2, 3};
+ auto v2 = __sanitizer::move(v);
+ EXPECT_EQ(3u, v2.size());
+ EXPECT_TRUE(v.empty());
+}
+
+TEST(SanitizerCommon, Forward) {
+ std::vector<int> v = {1, 2, 3};
+ auto v2 = __sanitizer::forward<std::vector<int>>(v);
+ EXPECT_EQ(3u, v2.size());
+ EXPECT_TRUE(v.empty());
+}
+
+TEST(SanitizerCommon, ForwardConst) {
+ const std::vector<int> v = {1, 2, 3};
+ auto v2 = __sanitizer::forward<const std::vector<int>&>(v);
+ EXPECT_EQ(3u, v2.size());
+ EXPECT_EQ(3u, v.size());
+}
+
+struct TestStruct {
+ int a;
+ float b;
+};
+
+TEST(SanitizerCommon, IsTriviallyDestructible) {
+ ASSERT_TRUE((is_trivially_destructible<int>::value));
+ ASSERT_TRUE((is_trivially_destructible<TestStruct>::value));
+ ASSERT_FALSE((is_trivially_destructible<std::vector<int>>::value));
+}
+
+TEST(SanitizerCommon, IsTriviallyCopyable) {
+ ASSERT_TRUE((is_trivially_copyable<int>::value));
+ ASSERT_TRUE((is_trivially_copyable<TestStruct>::value));
+ ASSERT_FALSE((is_trivially_copyable<std::vector<int>>::value));
+}
+
+} // namespace __sanitizer
\ No newline at end of file
___sanitizer_symbolize_data
___sanitizer_symbolize_demangle
___sanitizer_symbolize_flush
+___sanitizer_symbolize_set_demangle
+___sanitizer_symbolize_set_inline_frames
list(APPEND SCUDO_CFLAGS
-Werror=conversion
-Wall
+ -Wextra
+ -pedantic
-g
-nostdinc++)
# Remove -stdlib= which is unused when passing -nostdinc++.
-string(REGEX REPLACE "-stdlib=[a-zA-Z+]*" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
+string(REGEX REPLACE "-stdlib=[a-zA-Z+]*" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
append_list_if(COMPILER_RT_HAS_FVISIBILITY_HIDDEN_FLAG -fvisibility=hidden SCUDO_CFLAGS)
# We don't use the C++ standard library, so avoid including it by mistake.
append_list_if(COMPILER_RT_HAS_NOSTDLIBXX_FLAG -nostdlib++ SCUDO_LINK_FLAGS)
+append_list_if(CXX_SUPPORTS_UNWINDLIB_NONE_FLAG --unwindlib=none SCUDO_LINK_FLAGS)
+
+if(COMPILER_RT_SCUDO_STANDALONE_SYSROOT_PATH)
+ list(APPEND SCUDO_CFLAGS "--sysroot=${COMPILER_RT_SCUDO_STANDALONE_SYSROOT_PATH}")
+endif()
if(ANDROID)
list(APPEND SCUDO_CFLAGS -fno-emulated-tls)
quarantine.h
release.h
report.h
+ rss_limit_checker.h
secondary.h
size_class_map.h
stack_depot.h
linux.cpp
release.cpp
report.cpp
+ rss_limit_checker.cpp
string_utils.cpp
)
-# Enable the SSE 4.2 instruction set for crc32_hw.cpp, if available.
-if (COMPILER_RT_HAS_MSSE4_2_FLAG)
+# Enable the necessary instruction set for scudo_crc32.cpp, if available.
+# Newer compiler versions use -mcrc32 rather than -msse4.2.
+if (COMPILER_RT_HAS_MCRC32_FLAG)
+ set_source_files_properties(crc32_hw.cpp PROPERTIES COMPILE_FLAGS -mcrc32)
+elseif (COMPILER_RT_HAS_MSSE4_2_FLAG)
set_source_files_properties(crc32_hw.cpp PROPERTIES COMPILE_FLAGS -msse4.2)
endif()
)
set(SCUDO_OBJECT_LIBS)
+set(SCUDO_LINK_LIBS)
if (COMPILER_RT_HAS_GWP_ASAN)
+ if(COMPILER_RT_USE_LLVM_UNWINDER)
+ list(APPEND SCUDO_LINK_LIBS ${COMPILER_RT_UNWINDER_LINK_LIBS} dl)
+ elseif (COMPILER_RT_HAS_GCC_S_LIB)
+ list(APPEND SCUDO_LINK_LIBS gcc_s)
+ elseif (COMPILER_RT_HAS_GCC_LIB)
+ list(APPEND SCUDO_LINK_LIBS gcc)
+ elseif (NOT COMPILER_RT_USE_BUILTINS_LIBRARY)
+ message(FATAL_ERROR "No suitable unwinder library")
+ endif()
+
add_dependencies(scudo_standalone gwp_asan)
list(APPEND SCUDO_OBJECT_LIBS
RTGwpAsan RTGwpAsanBacktraceLibc RTGwpAsanSegvHandler
endif()
-set(SCUDO_LINK_LIBS)
+if(COMPILER_RT_BUILD_SCUDO_STANDALONE_WITH_LLVM_LIBC)
+ include_directories(${COMPILER_RT_BINARY_DIR}/../libc/include/)
+
+ set(SCUDO_DEPS libc-headers)
+
+ list(APPEND SCUDO_CFLAGS "-ffreestanding")
+endif()
append_list_if(COMPILER_RT_HAS_LIBPTHREAD -pthread SCUDO_LINK_FLAGS)
append_list_if(FUCHSIA zircon SCUDO_LINK_LIBS)
+if(COMPILER_RT_DEFAULT_TARGET_ARCH MATCHES "mips|mips64|mipsel|mips64el")
+ list(APPEND SCUDO_LINK_LIBS atomic)
+endif()
+
if(COMPILER_RT_HAS_SCUDO_STANDALONE)
add_compiler_rt_object_libraries(RTScudoStandalone
ARCHS ${SCUDO_STANDALONE_SUPPORTED_ARCH}
SOURCES ${SCUDO_SOURCES}
ADDITIONAL_HEADERS ${SCUDO_HEADERS}
- CFLAGS ${SCUDO_CFLAGS})
+ CFLAGS ${SCUDO_CFLAGS}
+ DEPS ${SCUDO_DEPS})
add_compiler_rt_object_libraries(RTScudoStandaloneCWrappers
ARCHS ${SCUDO_STANDALONE_SUPPORTED_ARCH}
SOURCES ${SCUDO_SOURCES_C_WRAPPERS}
ADDITIONAL_HEADERS ${SCUDO_HEADERS}
- CFLAGS ${SCUDO_CFLAGS})
+ CFLAGS ${SCUDO_CFLAGS}
+ DEPS ${SCUDO_DEPS})
add_compiler_rt_object_libraries(RTScudoStandaloneCxxWrappers
ARCHS ${SCUDO_STANDALONE_SUPPORTED_ARCH}
SOURCES ${SCUDO_SOURCES_CXX_WRAPPERS}
ADDITIONAL_HEADERS ${SCUDO_HEADERS}
- CFLAGS ${SCUDO_CFLAGS})
+ CFLAGS ${SCUDO_CFLAGS}
+ DEPS ${SCUDO_DEPS})
add_compiler_rt_runtime(clang_rt.scudo_standalone
STATIC
SOURCES ${SCUDO_SOURCES} ${SCUDO_SOURCES_C_WRAPPERS}
ADDITIONAL_HEADERS ${SCUDO_HEADERS}
CFLAGS ${SCUDO_CFLAGS}
+ DEPS ${SCUDO_DEPS}
OBJECT_LIBS ${SCUDO_OBJECT_LIBS}
PARENT_TARGET scudo_standalone)
add_compiler_rt_runtime(clang_rt.scudo_standalone_cxx
SOURCES ${SCUDO_SOURCES_CXX_WRAPPERS}
ADDITIONAL_HEADERS ${SCUDO_HEADERS}
CFLAGS ${SCUDO_CFLAGS}
+ DEPS ${SCUDO_DEPS}
PARENT_TARGET scudo_standalone)
- add_compiler_rt_runtime(clang_rt.scudo_standalone
- SHARED
- ARCHS ${SCUDO_STANDALONE_SUPPORTED_ARCH}
- SOURCES ${SCUDO_SOURCES} ${SCUDO_SOURCES_C_WRAPPERS} ${SCUDO_SOURCES_CXX_WRAPPERS}
- ADDITIONAL_HEADERS ${SCUDO_HEADERS}
- CFLAGS ${SCUDO_CFLAGS}
- OBJECT_LIBS ${SCUDO_OBJECT_LIBS}
- LINK_FLAGS ${SCUDO_LINK_FLAGS}
- LINK_LIBS ${SCUDO_LINK_LIBS}
- PARENT_TARGET scudo_standalone)
+ if(COMPILER_RT_SCUDO_STANDALONE_BUILD_SHARED)
+ add_compiler_rt_runtime(clang_rt.scudo_standalone
+ SHARED
+ ARCHS ${SCUDO_STANDALONE_SUPPORTED_ARCH}
+ SOURCES ${SCUDO_SOURCES} ${SCUDO_SOURCES_C_WRAPPERS} ${SCUDO_SOURCES_CXX_WRAPPERS}
+ ADDITIONAL_HEADERS ${SCUDO_HEADERS}
+ CFLAGS ${SCUDO_CFLAGS}
+ DEPS ${SCUDO_DEPS}
+ OBJECT_LIBS ${SCUDO_OBJECT_LIBS}
+ LINK_FLAGS ${SCUDO_LINK_FLAGS}
+ LINK_LIBS ${SCUDO_LINK_LIBS}
+ PARENT_TARGET scudo_standalone)
+ endif()
add_subdirectory(benchmarks)
if(COMPILER_RT_INCLUDE_TESTS)
// typedef SizeClassAllocator64<ExampleConfig> Primary;
// // Log2 of the size of a size class region, as used by the Primary.
// static const uptr PrimaryRegionSizeLog = 30U;
+// // Log2 of the size of block group, as used by the Primary. Each group
+// // contains a range of memory addresses, blocks in the range will belong to
+// // the same group. In general, single region may have 1 or 2MB group size.
+// // Multiple regions will have the group size equal to the region size
+// // because the region size is usually smaller than 1 MB.
+// // Smaller value gives fine-grained control of memory usage but the trade
+// // off is that it may take longer time of deallocation.
+// static const uptr PrimaryGroupSizeLog = 20U;
// // Defines the type and scale of a compact pointer. A compact pointer can
// // be understood as the offset of a pointer within the region it belongs
// // to, in increments of a power-of-2 scale.
#if SCUDO_CAN_USE_PRIMARY64
typedef SizeClassAllocator64<DefaultConfig> Primary;
static const uptr PrimaryRegionSizeLog = 32U;
+ static const uptr PrimaryGroupSizeLog = 21U;
typedef uptr PrimaryCompactPtrT;
static const uptr PrimaryCompactPtrScale = 0;
static const bool PrimaryEnableRandomOffset = true;
#else
typedef SizeClassAllocator32<DefaultConfig> Primary;
static const uptr PrimaryRegionSizeLog = 19U;
+ static const uptr PrimaryGroupSizeLog = 19U;
typedef uptr PrimaryCompactPtrT;
#endif
static const s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN;
static const uptr PrimaryRegionSizeLog = 28U;
typedef u32 PrimaryCompactPtrT;
static const uptr PrimaryCompactPtrScale = SCUDO_MIN_ALIGNMENT_LOG;
+ static const uptr PrimaryGroupSizeLog = 20U;
static const bool PrimaryEnableRandomOffset = true;
static const uptr PrimaryMapSizeIncrement = 1UL << 18;
#else
typedef SizeClassAllocator32<AndroidConfig> Primary;
static const uptr PrimaryRegionSizeLog = 18U;
+ static const uptr PrimaryGroupSizeLog = 18U;
typedef uptr PrimaryCompactPtrT;
#endif
static const s32 PrimaryMinReleaseToOsIntervalMs = 1000;
static const uptr PrimaryRegionSizeLog = 27U;
typedef u32 PrimaryCompactPtrT;
static const uptr PrimaryCompactPtrScale = SCUDO_MIN_ALIGNMENT_LOG;
+ static const uptr PrimaryGroupSizeLog = 18U;
static const bool PrimaryEnableRandomOffset = true;
static const uptr PrimaryMapSizeIncrement = 1UL << 18;
#else
typedef SizeClassAllocator32<AndroidSvelteConfig> Primary;
static const uptr PrimaryRegionSizeLog = 16U;
+ static const uptr PrimaryGroupSizeLog = 16U;
typedef uptr PrimaryCompactPtrT;
#endif
static const s32 PrimaryMinReleaseToOsIntervalMs = 1000;
typedef SizeClassAllocator64<FuchsiaConfig> Primary;
static const uptr PrimaryRegionSizeLog = 30U;
+ static const uptr PrimaryGroupSizeLog = 21U;
typedef u32 PrimaryCompactPtrT;
static const bool PrimaryEnableRandomOffset = true;
static const uptr PrimaryMapSizeIncrement = 1UL << 18;
typedef SizeClassAllocator64<TrustyConfig> Primary;
// Some apps have 1 page of heap total so small regions are necessary.
static const uptr PrimaryRegionSizeLog = 10U;
+ static const uptr PrimaryGroupSizeLog = 10U;
typedef u32 PrimaryCompactPtrT;
static const bool PrimaryEnableRandomOffset = false;
// Trusty is extremely memory-constrained so minimally round up map calls.
#include "checksum.h"
#include "atomic_helpers.h"
+#include "chunk.h"
#if defined(__x86_64__) || defined(__i386__)
#include <cpuid.h>
#include "internal_defs.h"
// Hardware CRC32 is supported at compilation via the following:
-// - for i386 & x86_64: -msse4.2
+// - for i386 & x86_64: -mcrc32 (earlier: -msse4.2)
// - for ARM & AArch64: -march=armv8-a+crc or -mcrc
// An additional check must be performed at runtime as well to make sure the
// emitted instructions are valid on the target host.
-#ifdef __SSE4_2__
+#if defined(__CRC32__)
+// NB: clang has <crc32intrin.h> but GCC does not
+#include <smmintrin.h>
+#define CRC32_INTRINSIC \
+ FIRST_32_SECOND_64(__builtin_ia32_crc32si, __builtin_ia32_crc32di)
+#elif defined(__SSE4_2__)
#include <smmintrin.h>
#define CRC32_INTRINSIC FIRST_32_SECOND_64(_mm_crc32_u32, _mm_crc32_u64)
#endif
// as opposed to only for crc32_hw.cpp. This means that other hardware
// specific instructions were likely emitted at other places, and as a result
// there is no reason to not use it here.
-#if defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32)
+#if defined(__CRC32__) || defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32)
u32 Crc = static_cast<u32>(CRC32_INTRINSIC(Seed, Value));
for (uptr I = 0; I < ArraySize; I++)
Crc = static_cast<u32>(CRC32_INTRINSIC(Crc, Array[I]));
Checksum = computeBSDChecksum(Checksum, Array[I]);
return Checksum;
}
-#endif // defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32)
+#endif // defined(__CRC32__) || defined(__SSE4_2__) ||
+ // defined(__ARM_FEATURE_CRC32)
}
namespace Chunk {
#include "options.h"
#include "quarantine.h"
#include "report.h"
+#include "rss_limit_checker.h"
#include "secondary.h"
#include "stack_depot.h"
#include "string_utils.h"
initFlags();
reportUnrecognizedFlags();
+ RssChecker.init(scudo::getFlags()->soft_rss_limit_mb,
+ scudo::getFlags()->hard_rss_limit_mb);
+
// Store some flags locally.
if (getFlags()->may_return_null)
Primary.Options.set(OptionBit::MayReturnNull);
Quarantine.init(
static_cast<uptr>(getFlags()->quarantine_size_kb << 10),
static_cast<uptr>(getFlags()->thread_local_quarantine_size_kb << 10));
+
+ initRingBuffer();
}
// Initialize the embedded GWP-ASan instance. Requires the main allocator to
getFlags()->GWP_ASAN_MaxSimultaneousAllocations;
Opt.SampleRate = getFlags()->GWP_ASAN_SampleRate;
Opt.InstallSignalHandlers = getFlags()->GWP_ASAN_InstallSignalHandlers;
+ Opt.Recoverable = getFlags()->GWP_ASAN_Recoverable;
// Embedded GWP-ASan is locked through the Scudo atfork handler (via
// Allocator::disable calling GWPASan.disable). Disable GWP-ASan's atfork
// handler.
gwp_asan::segv_handler::installSignalHandlers(
&GuardedAlloc, Printf,
gwp_asan::backtrace::getPrintBacktraceFunction(),
- gwp_asan::backtrace::getSegvBacktraceFunction());
+ gwp_asan::backtrace::getSegvBacktraceFunction(),
+ Opt.Recoverable);
GuardedAllocSlotSize =
GuardedAlloc.getAllocatorState()->maximumAllocationSize();
#endif // GWP_ASAN_HOOKS
}
+#ifdef GWP_ASAN_HOOKS
+ const gwp_asan::AllocationMetadata *getGwpAsanAllocationMetadata() {
+ return GuardedAlloc.getMetadataRegion();
+ }
+
+ const gwp_asan::AllocatorState *getGwpAsanAllocatorState() {
+ return GuardedAlloc.getAllocatorState();
+ }
+#endif // GWP_ASAN_HOOKS
+
ALWAYS_INLINE void initThreadMaybe(bool MinimalInit = false) {
TSDRegistry.initThreadMaybe(this, MinimalInit);
}
}
DCHECK_LE(Size, NeededSize);
+ switch (RssChecker.getRssLimitExceeded()) {
+ case RssLimitChecker::Neither:
+ break;
+ case RssLimitChecker::Soft:
+ if (Options.get(OptionBit::MayReturnNull))
+ return nullptr;
+ reportSoftRSSLimit(RssChecker.getSoftRssLimit());
+ break;
+ case RssLimitChecker::Hard:
+ reportHardRSSLimit(RssChecker.getHardRssLimit());
+ break;
+ }
+
void *Block = nullptr;
uptr ClassId = 0;
uptr SecondaryBlockEnd = 0;
Header.State == Chunk::State::Allocated;
}
+ void setRssLimitsTestOnly(int SoftRssLimitMb, int HardRssLimitMb,
+ bool MayReturnNull) {
+ RssChecker.init(SoftRssLimitMb, HardRssLimitMb);
+ if (MayReturnNull)
+ Primary.Options.set(OptionBit::MayReturnNull);
+ }
+
bool useMemoryTaggingTestOnly() const {
return useMemoryTagging<Params>(Primary.Options.load());
}
void setTrackAllocationStacks(bool Track) {
initThreadMaybe();
+ if (getFlags()->allocation_ring_buffer_size == 0) {
+ DCHECK(!Primary.Options.load().get(OptionBit::TrackAllocationStacks));
+ return;
+ }
if (Track)
Primary.Options.set(OptionBit::TrackAllocationStacks);
else
return PrimaryT::getRegionInfoArraySize();
}
- const char *getRingBufferAddress() const {
- return reinterpret_cast<const char *>(&RingBuffer);
+ const char *getRingBufferAddress() {
+ initThreadMaybe();
+ return RawRingBuffer;
}
- static uptr getRingBufferSize() { return sizeof(RingBuffer); }
+ uptr getRingBufferSize() {
+ initThreadMaybe();
+ auto *RingBuffer = getRingBuffer();
+ return RingBuffer ? ringBufferSizeInBytes(RingBuffer->Size) : 0;
+ }
+
+ static bool setRingBufferSizeForBuffer(char *Buffer, size_t Size) {
+ // Need at least one entry.
+ if (Size < sizeof(AllocationRingBuffer) +
+ sizeof(typename AllocationRingBuffer::Entry)) {
+ return false;
+ }
+ AllocationRingBuffer *RingBuffer =
+ reinterpret_cast<AllocationRingBuffer *>(Buffer);
+ RingBuffer->Size = (Size - sizeof(AllocationRingBuffer)) /
+ sizeof(typename AllocationRingBuffer::Entry);
+ return true;
+ }
static const uptr MaxTraceSize = 64;
if (!Depot->find(Hash, &RingPos, &Size))
return;
for (unsigned I = 0; I != Size && I != MaxTraceSize; ++I)
- Trace[I] = (*Depot)[RingPos + I];
+ Trace[I] = static_cast<uintptr_t>((*Depot)[RingPos + I]);
}
static void getErrorInfo(struct scudo_error_info *ErrorInfo,
QuarantineT Quarantine;
TSDRegistryT TSDRegistry;
pthread_once_t PostInitNonce = PTHREAD_ONCE_INIT;
+ RssLimitChecker RssChecker;
#ifdef GWP_ASAN_HOOKS
gwp_asan::GuardedPoolAllocator GuardedAlloc;
};
atomic_uptr Pos;
-#ifdef SCUDO_FUZZ
- static const uptr NumEntries = 2;
-#else
- static const uptr NumEntries = 32768;
-#endif
- Entry Entries[NumEntries];
+ u32 Size;
+ // An array of Size (at least one) elements of type Entry is immediately
+ // following to this struct.
};
- AllocationRingBuffer RingBuffer = {};
+ // Pointer to memory mapped area starting with AllocationRingBuffer struct,
+ // and immediately followed by Size elements of type Entry.
+ char *RawRingBuffer = {};
// The following might get optimized out by the compiler.
NOINLINE void performSanityChecks() {
void storeRingBufferEntry(void *Ptr, u32 AllocationTrace, u32 AllocationTid,
uptr AllocationSize, u32 DeallocationTrace,
u32 DeallocationTid) {
- uptr Pos = atomic_fetch_add(&RingBuffer.Pos, 1, memory_order_relaxed);
+ uptr Pos = atomic_fetch_add(&getRingBuffer()->Pos, 1, memory_order_relaxed);
typename AllocationRingBuffer::Entry *Entry =
- &RingBuffer.Entries[Pos % AllocationRingBuffer::NumEntries];
+ getRingBufferEntry(RawRingBuffer, Pos % getRingBuffer()->Size);
// First invalidate our entry so that we don't attempt to interpret a
// partially written state in getSecondaryErrorInfo(). The fences below
}
static const size_t NumErrorReports =
- sizeof(((scudo_error_info *)0)->reports) /
- sizeof(((scudo_error_info *)0)->reports[0]);
+ sizeof(((scudo_error_info *)nullptr)->reports) /
+ sizeof(((scudo_error_info *)nullptr)->reports[0]);
static void getInlineErrorInfo(struct scudo_error_info *ErrorInfo,
size_t &NextErrorReport, uintptr_t FaultAddr,
const char *RingBufferPtr) {
auto *RingBuffer =
reinterpret_cast<const AllocationRingBuffer *>(RingBufferPtr);
+ if (!RingBuffer || RingBuffer->Size == 0)
+ return;
uptr Pos = atomic_load_relaxed(&RingBuffer->Pos);
- for (uptr I = Pos - 1; I != Pos - 1 - AllocationRingBuffer::NumEntries &&
- NextErrorReport != NumErrorReports;
+ for (uptr I = Pos - 1;
+ I != Pos - 1 - RingBuffer->Size && NextErrorReport != NumErrorReports;
--I) {
- auto *Entry = &RingBuffer->Entries[I % AllocationRingBuffer::NumEntries];
+ auto *Entry = getRingBufferEntry(RingBufferPtr, I % RingBuffer->Size);
uptr EntryPtr = atomic_load_relaxed(&Entry->Ptr);
if (!EntryPtr)
continue;
Quarantine.getStats(Str);
return Str->length();
}
+
+ static typename AllocationRingBuffer::Entry *
+ getRingBufferEntry(char *RawRingBuffer, uptr N) {
+ return &reinterpret_cast<typename AllocationRingBuffer::Entry *>(
+ &RawRingBuffer[sizeof(AllocationRingBuffer)])[N];
+ }
+ static const typename AllocationRingBuffer::Entry *
+ getRingBufferEntry(const char *RawRingBuffer, uptr N) {
+ return &reinterpret_cast<const typename AllocationRingBuffer::Entry *>(
+ &RawRingBuffer[sizeof(AllocationRingBuffer)])[N];
+ }
+
+ void initRingBuffer() {
+ u32 AllocationRingBufferSize =
+ static_cast<u32>(getFlags()->allocation_ring_buffer_size);
+ if (AllocationRingBufferSize < 1)
+ return;
+ MapPlatformData Data = {};
+ RawRingBuffer = static_cast<char *>(
+ map(/*Addr=*/nullptr,
+ roundUpTo(ringBufferSizeInBytes(AllocationRingBufferSize), getPageSizeCached()),
+ "AllocatorRingBuffer", /*Flags=*/0, &Data));
+ auto *RingBuffer = reinterpret_cast<AllocationRingBuffer *>(RawRingBuffer);
+ RingBuffer->Size = AllocationRingBufferSize;
+ static_assert(sizeof(AllocationRingBuffer) %
+ alignof(typename AllocationRingBuffer::Entry) ==
+ 0,
+ "invalid alignment");
+ }
+
+ static constexpr size_t ringBufferSizeInBytes(u32 AllocationRingBufferSize) {
+ return sizeof(AllocationRingBuffer) +
+ AllocationRingBufferSize *
+ sizeof(typename AllocationRingBuffer::Entry);
+ }
+
+ inline AllocationRingBuffer *getRingBuffer() {
+ return reinterpret_cast<AllocationRingBuffer *>(RawRingBuffer);
+ }
};
} // namespace scudo
die();
}
+#if !SCUDO_LINUX
+uptr GetRSS() { return 0; }
+#endif
+
} // namespace scudo
// Hardware specific inlinable functions.
-inline void yieldProcessor(u8 Count) {
+inline void yieldProcessor(UNUSED u8 Count) {
#if defined(__i386__) || defined(__x86_64__)
__asm__ __volatile__("" ::: "memory");
for (u8 I = 0; I < Count; I++)
const char *getEnv(const char *Name);
+uptr GetRSS();
+
u64 getMonotonicTime();
u32 getThreadID();
#define MAP_NOACCESS (1U << 1)
#define MAP_RESIZABLE (1U << 2)
#define MAP_MEMTAG (1U << 3)
+#define MAP_PRECOMMIT (1U << 4)
// Our platform memory mapping use is restricted to 3 scenarios:
// - reserve memory at a random address (MAP_NOACCESS);
namespace scudo {
-#if defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32)
+#if defined(__CRC32__) || defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32)
u32 computeHardwareCRC32(u32 Crc, uptr Data) {
return static_cast<u32>(CRC32_INTRINSIC(Crc, Data));
}
-#endif // defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32)
+#endif // defined(__CRC32__) || defined(__SSE4_2__) ||
+ // defined(__ARM_FEATURE_CRC32)
} // namespace scudo
SCUDO_FLAG(int, release_to_os_interval_ms, SCUDO_ANDROID ? INT32_MIN : 5000,
"Interval (in milliseconds) at which to attempt release of unused "
"memory to the OS. Negative values disable the feature.")
+
+SCUDO_FLAG(int, hard_rss_limit_mb, 0,
+ "Hard RSS Limit in Mb. If non-zero, once the limit is achieved, "
+ "abort the process")
+
+SCUDO_FLAG(int, soft_rss_limit_mb, 0,
+ "Soft RSS Limit in Mb. If non-zero, once the limit is reached, all "
+ "subsequent calls will fail or return NULL until the RSS goes below "
+ "the soft limit")
+
+SCUDO_FLAG(int, allocation_ring_buffer_size, 32768,
+ "Entries to keep in the allocation ring buffer for scudo.")
#include <lib/sync/mutex.h> // for sync_mutex_t
#include <stdlib.h> // for getenv()
#include <zircon/compiler.h>
+#include <zircon/process.h>
#include <zircon/sanitizer.h>
#include <zircon/syscalls.h>
if (Flags & MAP_NOACCESS)
return allocateVmar(Size, Data, AllowNoMem);
- const zx_handle_t Vmar = Data ? Data->Vmar : _zx_vmar_root_self();
- CHECK_NE(Vmar, ZX_HANDLE_INVALID);
+ const zx_handle_t Vmar = (Data && Data->Vmar != ZX_HANDLE_INVALID)
+ ? Data->Vmar
+ : _zx_vmar_root_self();
zx_status_t Status;
zx_handle_t Vmo;
uintptr_t P;
zx_vm_option_t MapFlags =
ZX_VM_PERM_READ | ZX_VM_PERM_WRITE | ZX_VM_ALLOW_FAULTS;
+ if (Addr)
+ DCHECK(Data);
const uint64_t Offset =
Addr ? reinterpret_cast<uintptr_t>(Addr) - Data->VmarBase : 0;
if (Offset)
MapFlags |= ZX_VM_SPECIFIC;
Status = _zx_vmar_map(Vmar, MapFlags, Offset, Vmo, VmoSize, Size, &P);
+ if (UNLIKELY(Status != ZX_OK)) {
+ if (Status != ZX_ERR_NO_MEMORY || !AllowNoMem)
+ dieOnMapUnmapError(Status == ZX_ERR_NO_MEMORY ? Size : 0);
+ return nullptr;
+ }
+
+ if (Flags & MAP_PRECOMMIT) {
+ Status = _zx_vmar_op_range(Vmar, ZX_VMAR_OP_COMMIT, P, Size,
+ /*buffer=*/nullptr, /*buffer_size=*/0);
+ }
+
// No need to track the Vmo if we don't intend on resizing it. Close it.
if (Flags & MAP_RESIZABLE) {
DCHECK(Data);
dieOnMapUnmapError(Status == ZX_ERR_NO_MEMORY ? Size : 0);
return nullptr;
}
+
if (Data)
Data->VmoSize += Size;
CHECK_EQ(_zx_vmar_destroy(Vmar), ZX_OK);
CHECK_EQ(_zx_handle_close(Vmar), ZX_OK);
} else {
- const zx_handle_t Vmar = Data ? Data->Vmar : _zx_vmar_root_self();
+ const zx_handle_t Vmar = (Data && Data->Vmar != ZX_HANDLE_INVALID)
+ ? Data->Vmar
+ : _zx_vmar_root_self();
const zx_status_t Status =
_zx_vmar_unmap(Vmar, reinterpret_cast<uintptr_t>(Addr), Size);
if (UNLIKELY(Status != ZX_OK))
#if SCUDO_FUCHSIA
-#include <zircon/process.h>
+#include <stdint.h>
+#include <zircon/types.h>
namespace scudo {
}
std::string RingBufferBytes = FDP.ConsumeRemainingBytesAsString();
- std::vector<char> RingBuffer(AllocatorT::getRingBufferSize(), 0);
- for (size_t i = 0; i < RingBufferBytes.length() && i < RingBuffer.size();
- ++i) {
- RingBuffer[i] = RingBufferBytes[i];
- }
+ // RingBuffer is too short.
+ if (!AllocatorT::setRingBufferSizeForBuffer(RingBufferBytes.data(),
+ RingBufferBytes.size()))
+ return 0;
scudo_error_info ErrorInfo;
AllocatorT::getErrorInfo(&ErrorInfo, FaultAddr, StackDepot.data(),
- RegionInfo.data(), RingBuffer.data(), Memory,
+ RegionInfo.data(), RingBufferBytes.data(), Memory,
MemoryTags, MemoryAddr, MemorySize);
return 0;
}
extern "C" {
-__attribute__((weak)) const char *__scudo_default_options();
+__attribute__((weak)) const char *__scudo_default_options(void);
// Post-allocation & pre-deallocation hooks.
// They must be thread-safe and not use heap related functions.
struct scudo_error_report reports[3];
};
-const char *__scudo_get_stack_depot_addr();
-size_t __scudo_get_stack_depot_size();
+const char *__scudo_get_stack_depot_addr(void);
+size_t __scudo_get_stack_depot_size(void);
-const char *__scudo_get_region_info_addr();
-size_t __scudo_get_region_info_size();
+const char *__scudo_get_region_info_addr(void);
+size_t __scudo_get_region_info_size(void);
-const char *__scudo_get_ring_buffer_addr();
-size_t __scudo_get_ring_buffer_size();
+const char *__scudo_get_ring_buffer_addr(void);
+size_t __scudo_get_ring_buffer_size(void);
#ifndef M_DECAY_TIME
#define M_DECAY_TIME -100
namespace scudo {
-typedef unsigned long uptr;
-typedef unsigned char u8;
-typedef unsigned short u16;
-typedef unsigned int u32;
-typedef unsigned long long u64;
-typedef signed long sptr;
-typedef signed char s8;
-typedef signed short s16;
-typedef signed int s32;
-typedef signed long long s64;
+typedef uintptr_t uptr;
+typedef uint8_t u8;
+typedef uint16_t u16;
+typedef uint32_t u32;
+typedef uint64_t u64;
+typedef intptr_t sptr;
+typedef int8_t s8;
+typedef int16_t s16;
+typedef int32_t s32;
+typedef int64_t s64;
// The following two functions have platform specific implementations.
void outputRaw(const char *Buffer);
#else
#define DCHECK(A) \
do { \
- } while (false)
+ } while (false && (A))
#define DCHECK_EQ(A, B) \
do { \
- } while (false)
+ } while (false && (A) == (B))
#define DCHECK_NE(A, B) \
do { \
- } while (false)
+ } while (false && (A) != (B))
#define DCHECK_LT(A, B) \
do { \
- } while (false)
+ } while (false && (A) < (B))
#define DCHECK_LE(A, B) \
do { \
- } while (false)
+ } while (false && (A) <= (B))
#define DCHECK_GT(A, B) \
do { \
- } while (false)
+ } while (false && (A) > (B))
#define DCHECK_GE(A, B) \
do { \
- } while (false)
+ } while (false && (A) >= (B))
#endif
// The superfluous die() call effectively makes this macro NORETURN.
#include <fcntl.h>
#include <linux/futex.h>
#include <sched.h>
+#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
extern "C" WEAK int async_safe_write_log(int pri, const char *tag,
const char *msg);
+static uptr GetRSSFromBuffer(const char *Buf) {
+ // The format of the file is:
+ // 1084 89 69 11 0 79 0
+ // We need the second number which is RSS in pages.
+ const char *Pos = Buf;
+ // Skip the first number.
+ while (*Pos >= '0' && *Pos <= '9')
+ Pos++;
+ // Skip whitespaces.
+ while (!(*Pos >= '0' && *Pos <= '9') && *Pos != 0)
+ Pos++;
+ // Read the number.
+ u64 Rss = 0;
+ for (; *Pos >= '0' && *Pos <= '9'; Pos++)
+ Rss = Rss * 10 + static_cast<u64>(*Pos) - '0';
+ return static_cast<uptr>(Rss * getPageSizeCached());
+}
+
+uptr GetRSS() {
+ // TODO: We currently use sanitizer_common's GetRSS which reads the
+ // RSS from /proc/self/statm by default. We might want to
+ // call getrusage directly, even if it's less accurate.
+ auto Fd = open("/proc/self/statm", O_RDONLY);
+ char Buf[64];
+ s64 Len = read(Fd, Buf, sizeof(Buf) - 1);
+ close(Fd);
+ if (Len <= 0)
+ return 0;
+ Buf[Len] = 0;
+
+ return GetRSSFromBuffer(Buf);
+}
+
void outputRaw(const char *Buffer) {
if (&async_safe_write_log) {
constexpr s32 AndroidLogInfo = 4;
Size--;
}
+ // Insert X next to Prev
+ void insert(T *Prev, T *X) {
+ DCHECK(!empty());
+ DCHECK_NE(Prev, nullptr);
+ DCHECK_NE(X, nullptr);
+ X->Next = Prev->Next;
+ Prev->Next = X;
+ if (Last == Prev)
+ Last = X;
+ ++Size;
+ }
+
void extract(T *Prev, T *X) {
DCHECK(!empty());
DCHECK_NE(Prev, nullptr);
#define SCUDO_LOCAL_CACHE_H_
#include "internal_defs.h"
+#include "list.h"
+#include "platform.h"
#include "report.h"
#include "stats.h"
typedef typename SizeClassAllocator::CompactPtrT CompactPtrT;
struct TransferBatch {
- static const u32 MaxNumCached = SizeClassMap::MaxNumCachedHint;
- void setFromArray(CompactPtrT *Array, u32 N) {
+ static const u16 MaxNumCached = SizeClassMap::MaxNumCachedHint;
+ void setFromArray(CompactPtrT *Array, u16 N) {
DCHECK_LE(N, MaxNumCached);
Count = N;
memcpy(Batch, Array, sizeof(Batch[0]) * Count);
}
+ void appendFromArray(CompactPtrT *Array, u16 N) {
+ DCHECK_LE(N, MaxNumCached - Count);
+ memcpy(Batch + Count, Array, sizeof(Batch[0]) * N);
+ // u16 will be promoted to int by arithmetic type conversion.
+ Count = static_cast<u16>(Count + N);
+ }
void clear() { Count = 0; }
void add(CompactPtrT P) {
DCHECK_LT(Count, MaxNumCached);
void copyToArray(CompactPtrT *Array) const {
memcpy(Array, Batch, sizeof(Batch[0]) * Count);
}
- u32 getCount() const { return Count; }
- CompactPtrT get(u32 I) const {
+ u16 getCount() const { return Count; }
+ CompactPtrT get(u16 I) const {
DCHECK_LE(I, Count);
return Batch[I];
}
- static u32 getMaxCached(uptr Size) {
+ static u16 getMaxCached(uptr Size) {
return Min(MaxNumCached, SizeClassMap::getMaxCachedHint(Size));
}
TransferBatch *Next;
private:
- u32 Count;
CompactPtrT Batch[MaxNumCached];
+ u16 Count;
+ };
+
+ // A BatchGroup is used to collect blocks. Each group has a group id to
+ // identify the group kind of contained blocks.
+ struct BatchGroup {
+ // `Next` is used by IntrusiveList.
+ BatchGroup *Next;
+ // The identifier of each group
+ uptr GroupId;
+ // Cache value of TransferBatch::getMaxCached()
+ u16 MaxCachedPerBatch;
+ // Number of blocks pushed into this group. This is an increment-only
+ // counter.
+ uptr PushedBlocks;
+ // This is used to track how many blocks are pushed since last time we
+ // checked `PushedBlocks`. It's useful for page releasing to determine the
+ // usage of a BatchGroup.
+ uptr PushedBlocksAtLastCheckpoint;
+ // Blocks are managed by TransferBatch in a list.
+ SinglyLinkedList<TransferBatch> Batches;
};
+ static_assert(sizeof(BatchGroup) <= sizeof(TransferBatch),
+ "BatchGroup uses the same class size as TransferBatch");
+
void init(GlobalStats *S, SizeClassAllocator *A) {
DCHECK(isEmpty());
Stats.init();
TransferBatch *createBatch(uptr ClassId, void *B) {
if (ClassId != BatchClassId)
B = allocate(BatchClassId);
+ if (UNLIKELY(!B))
+ reportOutOfMemory(SizeClassAllocator::getSizeByClassId(BatchClassId));
return reinterpret_cast<TransferBatch *>(B);
}
+ BatchGroup *createGroup() {
+ void *Ptr = allocate(BatchClassId);
+ if (UNLIKELY(!Ptr))
+ reportOutOfMemory(SizeClassAllocator::getSizeByClassId(BatchClassId));
+ return reinterpret_cast<BatchGroup *>(Ptr);
+ }
+
LocalStats &getStats() { return Stats; }
private:
static const uptr NumClasses = SizeClassMap::NumClasses;
static const uptr BatchClassId = SizeClassMap::BatchClassId;
- struct PerClass {
- u32 Count;
- u32 MaxCount;
+ struct alignas(SCUDO_CACHE_LINE_SIZE) PerClass {
+ u16 Count;
+ u16 MaxCount;
// Note: ClassSize is zero for the transfer batch.
uptr ClassSize;
CompactPtrT Chunks[2 * TransferBatch::MaxNumCached];
for (uptr I = 0; I < NumClasses; I++) {
PerClass *P = &PerClassArray[I];
const uptr Size = SizeClassAllocator::getSizeByClassId(I);
- P->MaxCount = 2 * TransferBatch::getMaxCached(Size);
+ P->MaxCount = static_cast<u16>(2 * TransferBatch::getMaxCached(Size));
if (I != BatchClassId) {
P->ClassSize = Size;
} else {
}
NOINLINE void drain(PerClass *C, uptr ClassId) {
- const u32 Count = Min(C->MaxCount / 2, C->Count);
- TransferBatch *B =
- createBatch(ClassId, Allocator->decompactPtr(ClassId, C->Chunks[0]));
- if (UNLIKELY(!B))
- reportOutOfMemory(SizeClassAllocator::getSizeByClassId(BatchClassId));
- B->setFromArray(&C->Chunks[0], Count);
- C->Count -= Count;
- for (uptr I = 0; I < C->Count; I++)
+ const u16 Count = Min(static_cast<u16>(C->MaxCount / 2), C->Count);
+ Allocator->pushBlocks(this, ClassId, &C->Chunks[0], Count);
+ // u16 will be promoted to int by arithmetic type conversion.
+ C->Count = static_cast<u16>(C->Count - Count);
+ for (u16 I = 0; I < C->Count; I++)
C->Chunks[I] = C->Chunks[I + Count];
- Allocator->pushBatch(ClassId, B);
}
};
namespace scudo {
-#if (__clang_major__ >= 12 && defined(__aarch64__)) || defined(SCUDO_FUZZ)
+#if (__clang_major__ >= 12 && defined(__aarch64__) && !defined(__ILP32__)) || \
+ defined(SCUDO_FUZZ)
// We assume that Top-Byte Ignore is enabled if the architecture supports memory
// tagging. Not all operating systems enable TBI, so we only claim architectural
// support for memory tagging if the operating system enables TBI.
-#if SCUDO_LINUX && !defined(SCUDO_DISABLE_TBI)
+// HWASan uses the top byte for its own purpose and Scudo should not touch it.
+#if SCUDO_LINUX && !defined(SCUDO_DISABLE_TBI) && \
+ !__has_feature(hwaddress_sanitizer)
inline constexpr bool archSupportsMemoryTagging() { return true; }
#else
inline constexpr bool archSupportsMemoryTagging() { return false; }
inline constexpr bool archSupportsMemoryTagging() { return false; }
-inline uptr archMemoryTagGranuleSize() {
+inline NORETURN uptr archMemoryTagGranuleSize() {
UNREACHABLE("memory tagging not supported");
}
-inline uptr untagPointer(uptr Ptr) {
+inline NORETURN uptr untagPointer(uptr Ptr) {
(void)Ptr;
UNREACHABLE("memory tagging not supported");
}
-inline uint8_t extractTag(uptr Ptr) {
+inline NORETURN uint8_t extractTag(uptr Ptr) {
(void)Ptr;
UNREACHABLE("memory tagging not supported");
}
#endif
-#if __clang_major__ >= 12 && defined(__aarch64__)
+#if __clang_major__ >= 12 && defined(__aarch64__) && !defined(__ILP32__)
#if SCUDO_LINUX
#ifndef PR_MTE_TCF_MASK
#define PR_MTE_TCF_MASK (3UL << PR_MTE_TCF_SHIFT)
#endif
- return (static_cast<unsigned long>(
- prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0)) &
- PR_MTE_TCF_MASK) != PR_MTE_TCF_NONE;
+ int res = prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0);
+ if (res == -1)
+ return false;
+ return (static_cast<unsigned long>(res) & PR_MTE_TCF_MASK) != PR_MTE_TCF_NONE;
}
inline void enableSystemMemoryTaggingTestOnly() {
inline bool systemSupportsMemoryTagging() { return false; }
-inline bool systemDetectsMemoryTagFaultsTestOnly() {
+inline NORETURN bool systemDetectsMemoryTagFaultsTestOnly() {
UNREACHABLE("memory tagging not supported");
}
-inline void enableSystemMemoryTaggingTestOnly() {
+inline NORETURN void enableSystemMemoryTaggingTestOnly() {
UNREACHABLE("memory tagging not supported");
}
#else
-inline bool systemSupportsMemoryTagging() {
+inline NORETURN bool systemSupportsMemoryTagging() {
UNREACHABLE("memory tagging not supported");
}
-inline bool systemDetectsMemoryTagFaultsTestOnly() {
+inline NORETURN bool systemDetectsMemoryTagFaultsTestOnly() {
UNREACHABLE("memory tagging not supported");
}
-inline void enableSystemMemoryTaggingTestOnly() {
+inline NORETURN void enableSystemMemoryTaggingTestOnly() {
UNREACHABLE("memory tagging not supported");
}
ScopedDisableMemoryTagChecks() {}
};
-inline uptr selectRandomTag(uptr Ptr, uptr ExcludeMask) {
+inline NORETURN uptr selectRandomTag(uptr Ptr, uptr ExcludeMask) {
(void)Ptr;
(void)ExcludeMask;
UNREACHABLE("memory tagging not supported");
}
-inline uptr addFixedTag(uptr Ptr, uptr Tag) {
+inline NORETURN uptr addFixedTag(uptr Ptr, uptr Tag) {
(void)Ptr;
(void)Tag;
UNREACHABLE("memory tagging not supported");
}
-inline uptr storeTags(uptr Begin, uptr End) {
+inline NORETURN uptr storeTags(uptr Begin, uptr End) {
(void)Begin;
(void)End;
UNREACHABLE("memory tagging not supported");
}
-inline void storeTag(uptr Ptr) {
+inline NORETURN void storeTag(uptr Ptr) {
(void)Ptr;
UNREACHABLE("memory tagging not supported");
}
-inline uptr loadTag(uptr Ptr) {
+inline NORETURN uptr loadTag(uptr Ptr) {
(void)Ptr;
UNREACHABLE("memory tagging not supported");
}
#endif
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wmissing-noreturn"
inline void setRandomTag(void *Ptr, uptr Size, uptr ExcludeMask,
uptr *TaggedBegin, uptr *TaggedEnd) {
*TaggedBegin = selectRandomTag(reinterpret_cast<uptr>(Ptr), ExcludeMask);
*TaggedEnd = storeTags(*TaggedBegin, *TaggedBegin + Size);
}
+#pragma GCC diagnostic pop
inline void *untagPointer(void *Ptr) {
return reinterpret_cast<void *>(untagPointer(reinterpret_cast<uptr>(Ptr)));
#define SCUDO_TRUSTY 0
#endif
-#if __LP64__
+#if defined(__LP64__)
#define SCUDO_WORDSIZE 64U
#else
#define SCUDO_WORDSIZE 32U
public:
typedef typename Config::PrimaryCompactPtrT CompactPtrT;
typedef typename Config::SizeClassMap SizeClassMap;
+ static const uptr GroupSizeLog = Config::PrimaryGroupSizeLog;
// The bytemap can only track UINT8_MAX - 1 classes.
static_assert(SizeClassMap::LargestClassId <= (UINT8_MAX - 1), "");
// Regions should be large enough to hold the largest Block.
typedef SizeClassAllocator32<Config> ThisT;
typedef SizeClassAllocatorLocalCache<ThisT> CacheT;
typedef typename CacheT::TransferBatch TransferBatch;
+ typedef typename CacheT::BatchGroup BatchGroup;
static uptr getSizeByClassId(uptr ClassId) {
return (ClassId == SizeClassMap::BatchClassId)
return reinterpret_cast<void *>(static_cast<uptr>(CompactPtr));
}
+ uptr compactPtrGroup(CompactPtrT CompactPtr) {
+ return CompactPtr >> GroupSizeLog;
+ }
+
TransferBatch *popBatch(CacheT *C, uptr ClassId) {
DCHECK_LT(ClassId, NumClasses);
SizeClassInfo *Sci = getSizeClassInfo(ClassId);
ScopedLock L(Sci->Mutex);
- TransferBatch *B = Sci->FreeList.front();
- if (B) {
- Sci->FreeList.pop_front();
- } else {
- B = populateFreeList(C, ClassId, Sci);
- if (UNLIKELY(!B))
+ TransferBatch *B = popBatchImpl(C, ClassId);
+ if (UNLIKELY(!B)) {
+ if (UNLIKELY(!populateFreeList(C, ClassId, Sci)))
return nullptr;
+ B = popBatchImpl(C, ClassId);
+ // if `populateFreeList` succeeded, we are supposed to get free blocks.
+ DCHECK_NE(B, nullptr);
}
- DCHECK_GT(B->getCount(), 0);
Sci->Stats.PoppedBlocks += B->getCount();
return B;
}
- void pushBatch(uptr ClassId, TransferBatch *B) {
+ // Push the array of free blocks to the designated batch group.
+ void pushBlocks(CacheT *C, uptr ClassId, CompactPtrT *Array, u32 Size) {
DCHECK_LT(ClassId, NumClasses);
- DCHECK_GT(B->getCount(), 0);
+ DCHECK_GT(Size, 0);
+
SizeClassInfo *Sci = getSizeClassInfo(ClassId);
+ if (ClassId == SizeClassMap::BatchClassId) {
+ ScopedLock L(Sci->Mutex);
+ // Constructing a batch group in the free list will use two blocks in
+ // BatchClassId. If we are pushing BatchClassId blocks, we will use the
+ // blocks in the array directly (can't delegate local cache which will
+ // cause a recursive allocation). However, The number of free blocks may
+ // be less than two. Therefore, populate the free list before inserting
+ // the blocks.
+ if (Size == 1 && !populateFreeList(C, ClassId, Sci))
+ return;
+ pushBlocksImpl(C, ClassId, Array, Size);
+ Sci->Stats.PushedBlocks += Size;
+ return;
+ }
+
+ // TODO(chiahungduan): Consider not doing grouping if the group size is not
+ // greater than the block size with a certain scale.
+
+ // Sort the blocks so that blocks belonging to the same group can be pushed
+ // together.
+ bool SameGroup = true;
+ for (u32 I = 1; I < Size; ++I) {
+ if (compactPtrGroup(Array[I - 1]) != compactPtrGroup(Array[I]))
+ SameGroup = false;
+ CompactPtrT Cur = Array[I];
+ u32 J = I;
+ while (J > 0 && compactPtrGroup(Cur) < compactPtrGroup(Array[J - 1])) {
+ Array[J] = Array[J - 1];
+ --J;
+ }
+ Array[J] = Cur;
+ }
+
ScopedLock L(Sci->Mutex);
- Sci->FreeList.push_front(B);
- Sci->Stats.PushedBlocks += B->getCount();
+ pushBlocksImpl(C, ClassId, Array, Size, SameGroup);
+
+ Sci->Stats.PushedBlocks += Size;
if (ClassId != SizeClassMap::BatchClassId)
releaseToOSMaybe(Sci, ClassId);
}
struct alignas(SCUDO_CACHE_LINE_SIZE) SizeClassInfo {
HybridMutex Mutex;
- SinglyLinkedList<TransferBatch> FreeList;
+ SinglyLinkedList<BatchGroup> FreeList;
uptr CurrentRegion;
uptr CurrentRegionAllocated;
SizeClassStats Stats;
return &SizeClassInfoArray[ClassId];
}
- NOINLINE TransferBatch *populateFreeList(CacheT *C, uptr ClassId,
- SizeClassInfo *Sci) {
+ // Push the blocks to their batch group. The layout will be like,
+ //
+ // FreeList - > BG -> BG -> BG
+ // | | |
+ // v v v
+ // TB TB TB
+ // |
+ // v
+ // TB
+ //
+ // Each BlockGroup(BG) will associate with unique group id and the free blocks
+ // are managed by a list of TransferBatch(TB). To reduce the time of inserting
+ // blocks, BGs are sorted and the input `Array` are supposed to be sorted so
+ // that we can get better performance of maintaining sorted property.
+ // Use `SameGroup=true` to indicate that all blocks in the array are from the
+ // same group then we will skip checking the group id of each block.
+ //
+ // Note that this aims to have a better management of dirty pages, i.e., the
+ // RSS usage won't grow indefinitely. There's an exception that we may not put
+ // a block to its associated group. While populating new blocks, we may have
+ // blocks cross different groups. However, most cases will fall into same
+ // group and they are supposed to be popped soon. In that case, it's not worth
+ // sorting the array with the almost-sorted property. Therefore, we use
+ // `SameGroup=true` instead.
+ //
+ // The region mutex needs to be held while calling this method.
+ void pushBlocksImpl(CacheT *C, uptr ClassId, CompactPtrT *Array, u32 Size,
+ bool SameGroup = false) {
+ DCHECK_GT(Size, 0U);
+ SizeClassInfo *Sci = getSizeClassInfo(ClassId);
+
+ auto CreateGroup = [&](uptr GroupId) {
+ BatchGroup *BG = nullptr;
+ TransferBatch *TB = nullptr;
+ if (ClassId == SizeClassMap::BatchClassId) {
+ DCHECK_GE(Size, 2U);
+ BG = reinterpret_cast<BatchGroup *>(
+ decompactPtr(ClassId, Array[Size - 1]));
+ BG->Batches.clear();
+
+ TB = reinterpret_cast<TransferBatch *>(
+ decompactPtr(ClassId, Array[Size - 2]));
+ TB->clear();
+ } else {
+ BG = C->createGroup();
+ BG->Batches.clear();
+
+ TB = C->createBatch(ClassId, nullptr);
+ TB->clear();
+ }
+
+ BG->GroupId = GroupId;
+ BG->Batches.push_front(TB);
+ BG->PushedBlocks = 0;
+ BG->PushedBlocksAtLastCheckpoint = 0;
+ BG->MaxCachedPerBatch =
+ TransferBatch::getMaxCached(getSizeByClassId(ClassId));
+
+ return BG;
+ };
+
+ auto InsertBlocks = [&](BatchGroup *BG, CompactPtrT *Array, u32 Size) {
+ SinglyLinkedList<TransferBatch> &Batches = BG->Batches;
+ TransferBatch *CurBatch = Batches.front();
+ DCHECK_NE(CurBatch, nullptr);
+
+ for (u32 I = 0; I < Size;) {
+ DCHECK_GE(BG->MaxCachedPerBatch, CurBatch->getCount());
+ u16 UnusedSlots =
+ static_cast<u16>(BG->MaxCachedPerBatch - CurBatch->getCount());
+ if (UnusedSlots == 0) {
+ CurBatch = C->createBatch(
+ ClassId,
+ reinterpret_cast<void *>(decompactPtr(ClassId, Array[I])));
+ CurBatch->clear();
+ Batches.push_front(CurBatch);
+ UnusedSlots = BG->MaxCachedPerBatch;
+ }
+ // `UnusedSlots` is u16 so the result will be also fit in u16.
+ u16 AppendSize = static_cast<u16>(Min<u32>(UnusedSlots, Size - I));
+ CurBatch->appendFromArray(&Array[I], AppendSize);
+ I += AppendSize;
+ }
+
+ BG->PushedBlocks += Size;
+ };
+
+ BatchGroup *Cur = Sci->FreeList.front();
+
+ if (ClassId == SizeClassMap::BatchClassId) {
+ if (Cur == nullptr) {
+ // Don't need to classify BatchClassId.
+ Cur = CreateGroup(/*GroupId=*/0);
+ Sci->FreeList.push_front(Cur);
+ }
+ InsertBlocks(Cur, Array, Size);
+ return;
+ }
+
+ // In the following, `Cur` always points to the BatchGroup for blocks that
+ // will be pushed next. `Prev` is the element right before `Cur`.
+ BatchGroup *Prev = nullptr;
+
+ while (Cur != nullptr && compactPtrGroup(Array[0]) > Cur->GroupId) {
+ Prev = Cur;
+ Cur = Cur->Next;
+ }
+
+ if (Cur == nullptr || compactPtrGroup(Array[0]) != Cur->GroupId) {
+ Cur = CreateGroup(compactPtrGroup(Array[0]));
+ if (Prev == nullptr)
+ Sci->FreeList.push_front(Cur);
+ else
+ Sci->FreeList.insert(Prev, Cur);
+ }
+
+ // All the blocks are from the same group, just push without checking group
+ // id.
+ if (SameGroup) {
+ InsertBlocks(Cur, Array, Size);
+ return;
+ }
+
+ // The blocks are sorted by group id. Determine the segment of group and
+ // push them to their group together.
+ u32 Count = 1;
+ for (u32 I = 1; I < Size; ++I) {
+ if (compactPtrGroup(Array[I - 1]) != compactPtrGroup(Array[I])) {
+ DCHECK_EQ(compactPtrGroup(Array[I - 1]), Cur->GroupId);
+ InsertBlocks(Cur, Array + I - Count, Count);
+
+ while (Cur != nullptr && compactPtrGroup(Array[I]) > Cur->GroupId) {
+ Prev = Cur;
+ Cur = Cur->Next;
+ }
+
+ if (Cur == nullptr || compactPtrGroup(Array[I]) != Cur->GroupId) {
+ Cur = CreateGroup(compactPtrGroup(Array[I]));
+ DCHECK_NE(Prev, nullptr);
+ Sci->FreeList.insert(Prev, Cur);
+ }
+
+ Count = 1;
+ } else {
+ ++Count;
+ }
+ }
+
+ InsertBlocks(Cur, Array + Size - Count, Count);
+ }
+
+ // Pop one TransferBatch from a BatchGroup. The BatchGroup with the smallest
+ // group id will be considered first.
+ //
+ // The region mutex needs to be held while calling this method.
+ TransferBatch *popBatchImpl(CacheT *C, uptr ClassId) {
+ SizeClassInfo *Sci = getSizeClassInfo(ClassId);
+ if (Sci->FreeList.empty())
+ return nullptr;
+
+ SinglyLinkedList<TransferBatch> &Batches = Sci->FreeList.front()->Batches;
+ DCHECK(!Batches.empty());
+
+ TransferBatch *B = Batches.front();
+ Batches.pop_front();
+ DCHECK_NE(B, nullptr);
+ DCHECK_GT(B->getCount(), 0U);
+
+ if (Batches.empty()) {
+ BatchGroup *BG = Sci->FreeList.front();
+ Sci->FreeList.pop_front();
+
+ // We don't keep BatchGroup with zero blocks to avoid empty-checking while
+ // allocating. Note that block used by constructing BatchGroup is recorded
+ // as free blocks in the last element of BatchGroup::Batches. Which means,
+ // once we pop the last TransferBatch, the block is implicitly
+ // deallocated.
+ if (ClassId != SizeClassMap::BatchClassId)
+ C->deallocate(SizeClassMap::BatchClassId, BG);
+ }
+
+ return B;
+ }
+
+ NOINLINE bool populateFreeList(CacheT *C, uptr ClassId, SizeClassInfo *Sci) {
uptr Region;
uptr Offset;
// If the size-class currently has a region associated to it, use it. The
DCHECK_EQ(Sci->CurrentRegionAllocated, 0U);
Region = allocateRegion(Sci, ClassId);
if (UNLIKELY(!Region))
- return nullptr;
+ return false;
C->getStats().add(StatMapped, RegionSize);
Sci->CurrentRegion = Region;
Offset = 0;
}
const uptr Size = getSizeByClassId(ClassId);
- const u32 MaxCount = TransferBatch::getMaxCached(Size);
+ const u16 MaxCount = TransferBatch::getMaxCached(Size);
DCHECK_GT(MaxCount, 0U);
// The maximum number of blocks we should carve in the region is dictated
// by the maximum number of batches we want to fill, and the amount of
if (ClassId != SizeClassMap::BatchClassId)
shuffle(ShuffleArray, NumberOfBlocks, &Sci->RandState);
for (u32 I = 0; I < NumberOfBlocks;) {
- TransferBatch *B =
- C->createBatch(ClassId, reinterpret_cast<void *>(ShuffleArray[I]));
- if (UNLIKELY(!B))
- return nullptr;
- const u32 N = Min(MaxCount, NumberOfBlocks - I);
- B->setFromArray(&ShuffleArray[I], N);
- Sci->FreeList.push_back(B);
+ // `MaxCount` is u16 so the result will also fit in u16.
+ const u16 N = static_cast<u16>(Min<u32>(MaxCount, NumberOfBlocks - I));
+ // Note that the N blocks here may have different group ids. Given that
+ // it only happens when it crosses the group size boundary. Instead of
+ // sorting them, treat them as same group here to avoid sorting the
+ // almost-sorted blocks.
+ pushBlocksImpl(C, ClassId, &ShuffleArray[I], N, /*SameGroup=*/true);
I += N;
}
- TransferBatch *B = Sci->FreeList.front();
- Sci->FreeList.pop_front();
- DCHECK(B);
- DCHECK_GT(B->getCount(), 0);
const uptr AllocatedUser = Size * NumberOfBlocks;
C->getStats().add(StatFree, AllocatedUser);
}
Sci->AllocatedUser += AllocatedUser;
- return B;
+ return true;
}
void getStats(ScopedString *Str, uptr ClassId, uptr Rss) {
if (BytesPushed < PageSize)
return 0; // Nothing new to release.
+ const bool CheckDensity = BlockSize < PageSize / 16U;
// Releasing smaller blocks is expensive, so we want to make sure that a
// significant amount of bytes are free, and that there has been a good
// amount of batches pushed to the freelist before attempting to release.
- if (BlockSize < PageSize / 16U) {
+ if (CheckDensity) {
if (!Force && BytesPushed < Sci->AllocatedUser / 16U)
return 0;
- // We want 8x% to 9x% free bytes (the larger the block, the lower the %).
- if ((BytesInFreeList * 100U) / Sci->AllocatedUser <
- (100U - 1U - BlockSize / 16U))
- return 0;
}
if (!Force) {
uptr TotalReleasedBytes = 0;
const uptr Base = First * RegionSize;
const uptr NumberOfRegions = Last - First + 1U;
+ const uptr GroupSize = (1U << GroupSizeLog);
+ const uptr CurRegionGroupId =
+ compactPtrGroup(compactPtr(ClassId, Sci->CurrentRegion));
+
ReleaseRecorder Recorder(Base);
- auto SkipRegion = [this, First, ClassId](uptr RegionIndex) {
- return (PossibleRegions[First + RegionIndex] - 1U) != ClassId;
- };
+ PageReleaseContext Context(BlockSize, RegionSize, NumberOfRegions);
+
auto DecompactPtr = [](CompactPtrT CompactPtr) {
return reinterpret_cast<uptr>(CompactPtr);
};
- releaseFreeMemoryToOS(Sci->FreeList, RegionSize, NumberOfRegions, BlockSize,
- &Recorder, DecompactPtr, SkipRegion);
+ for (BatchGroup &BG : Sci->FreeList) {
+ const uptr PushedBytesDelta =
+ BG.PushedBlocks - BG.PushedBlocksAtLastCheckpoint;
+ if (PushedBytesDelta * BlockSize < PageSize)
+ continue;
+
+ uptr AllocatedGroupSize = BG.GroupId == CurRegionGroupId
+ ? Sci->CurrentRegionAllocated
+ : GroupSize;
+ if (AllocatedGroupSize == 0)
+ continue;
+
+ // TransferBatches are pushed in front of BG.Batches. The first one may
+ // not have all caches used.
+ const uptr NumBlocks = (BG.Batches.size() - 1) * BG.MaxCachedPerBatch +
+ BG.Batches.front()->getCount();
+ const uptr BytesInBG = NumBlocks * BlockSize;
+ // Given the randomness property, we try to release the pages only if the
+ // bytes used by free blocks exceed certain proportion of allocated
+ // spaces.
+ if (CheckDensity && (BytesInBG * 100U) / AllocatedGroupSize <
+ (100U - 1U - BlockSize / 16U)) {
+ continue;
+ }
+
+ BG.PushedBlocksAtLastCheckpoint = BG.PushedBlocks;
+ // Note that we don't always visit blocks in each BatchGroup so that we
+ // may miss the chance of releasing certain pages that cross BatchGroups.
+ Context.markFreeBlocks(BG.Batches, DecompactPtr, Base);
+ }
+
+ if (!Context.hasBlockMarked())
+ return 0;
+
+ auto SkipRegion = [this, First, ClassId](uptr RegionIndex) {
+ return (PossibleRegions[First + RegionIndex] - 1U) != ClassId;
+ };
+ releaseFreeMemoryToOS(Context, Recorder, SkipRegion);
+
if (Recorder.getReleasedRangesCount() > 0) {
Sci->ReleaseInfo.PushedBlocksAtLastRelease = Sci->Stats.PushedBlocks;
Sci->ReleaseInfo.RangesReleased += Recorder.getReleasedRangesCount();
public:
typedef typename Config::PrimaryCompactPtrT CompactPtrT;
static const uptr CompactPtrScale = Config::PrimaryCompactPtrScale;
+ static const uptr GroupSizeLog = Config::PrimaryGroupSizeLog;
typedef typename Config::SizeClassMap SizeClassMap;
typedef SizeClassAllocator64<Config> ThisT;
typedef SizeClassAllocatorLocalCache<ThisT> CacheT;
typedef typename CacheT::TransferBatch TransferBatch;
+ typedef typename CacheT::BatchGroup BatchGroup;
static uptr getSizeByClassId(uptr ClassId) {
return (ClassId == SizeClassMap::BatchClassId)
RegionInfo *Region = getRegionInfo(I);
*Region = {};
}
- unmap(reinterpret_cast<void *>(PrimaryBase), PrimarySize, UNMAP_ALL, &Data);
+ if (PrimaryBase)
+ unmap(reinterpret_cast<void *>(PrimaryBase), PrimarySize, UNMAP_ALL,
+ &Data);
PrimaryBase = 0U;
}
DCHECK_LT(ClassId, NumClasses);
RegionInfo *Region = getRegionInfo(ClassId);
ScopedLock L(Region->Mutex);
- TransferBatch *B = Region->FreeList.front();
- if (B) {
- Region->FreeList.pop_front();
- } else {
- B = populateFreeList(C, ClassId, Region);
- if (UNLIKELY(!B))
+ TransferBatch *B = popBatchImpl(C, ClassId);
+ if (UNLIKELY(!B)) {
+ if (UNLIKELY(!populateFreeList(C, ClassId, Region)))
return nullptr;
+ B = popBatchImpl(C, ClassId);
+ // if `populateFreeList` succeeded, we are supposed to get free blocks.
+ DCHECK_NE(B, nullptr);
}
- DCHECK_GT(B->getCount(), 0);
Region->Stats.PoppedBlocks += B->getCount();
return B;
}
- void pushBatch(uptr ClassId, TransferBatch *B) {
- DCHECK_GT(B->getCount(), 0);
+ // Push the array of free blocks to the designated batch group.
+ void pushBlocks(CacheT *C, uptr ClassId, CompactPtrT *Array, u32 Size) {
+ DCHECK_LT(ClassId, NumClasses);
+ DCHECK_GT(Size, 0);
+
RegionInfo *Region = getRegionInfo(ClassId);
+ if (ClassId == SizeClassMap::BatchClassId) {
+ ScopedLock L(Region->Mutex);
+ // Constructing a batch group in the free list will use two blocks in
+ // BatchClassId. If we are pushing BatchClassId blocks, we will use the
+ // blocks in the array directly (can't delegate local cache which will
+ // cause a recursive allocation). However, The number of free blocks may
+ // be less than two. Therefore, populate the free list before inserting
+ // the blocks.
+ if (Size == 1 && UNLIKELY(!populateFreeList(C, ClassId, Region)))
+ return;
+ pushBlocksImpl(C, ClassId, Array, Size);
+ Region->Stats.PushedBlocks += Size;
+ return;
+ }
+
+ // TODO(chiahungduan): Consider not doing grouping if the group size is not
+ // greater than the block size with a certain scale.
+
+ // Sort the blocks so that blocks belonging to the same group can be pushed
+ // together.
+ bool SameGroup = true;
+ for (u32 I = 1; I < Size; ++I) {
+ if (compactPtrGroup(Array[I - 1]) != compactPtrGroup(Array[I]))
+ SameGroup = false;
+ CompactPtrT Cur = Array[I];
+ u32 J = I;
+ while (J > 0 && compactPtrGroup(Cur) < compactPtrGroup(Array[J - 1])) {
+ Array[J] = Array[J - 1];
+ --J;
+ }
+ Array[J] = Cur;
+ }
+
ScopedLock L(Region->Mutex);
- Region->FreeList.push_front(B);
- Region->Stats.PushedBlocks += B->getCount();
+ pushBlocksImpl(C, ClassId, Array, Size, SameGroup);
+
+ Region->Stats.PushedBlocks += Size;
if (ClassId != SizeClassMap::BatchClassId)
releaseToOSMaybe(Region, ClassId);
}
PoppedBlocks += Region->Stats.PoppedBlocks;
PushedBlocks += Region->Stats.PushedBlocks;
}
- Str->append("Stats: SizeClassAllocator64: %zuM mapped (%zuM rss) in %zu "
+ Str->append("Stats: SizeClassAllocator64: %zuM mapped (%uM rss) in %zu "
"allocations; remains %zu\n",
- TotalMapped >> 20, 0, PoppedBlocks,
+ TotalMapped >> 20, 0U, PoppedBlocks,
PoppedBlocks - PushedBlocks);
for (uptr I = 0; I < NumClasses; I++)
struct UnpaddedRegionInfo {
HybridMutex Mutex;
- SinglyLinkedList<TransferBatch> FreeList;
+ SinglyLinkedList<BatchGroup> FreeList;
uptr RegionBeg = 0;
RegionStats Stats = {};
u32 RandState = 0;
return Base + (static_cast<uptr>(CompactPtr) << CompactPtrScale);
}
- NOINLINE TransferBatch *populateFreeList(CacheT *C, uptr ClassId,
- RegionInfo *Region) {
+ static uptr compactPtrGroup(CompactPtrT CompactPtr) {
+ return static_cast<uptr>(CompactPtr) >> (GroupSizeLog - CompactPtrScale);
+ }
+ static uptr batchGroupBase(uptr Base, uptr GroupId) {
+ return (GroupId << GroupSizeLog) + Base;
+ }
+
+ // Push the blocks to their batch group. The layout will be like,
+ //
+ // FreeList - > BG -> BG -> BG
+ // | | |
+ // v v v
+ // TB TB TB
+ // |
+ // v
+ // TB
+ //
+ // Each BlockGroup(BG) will associate with unique group id and the free blocks
+ // are managed by a list of TransferBatch(TB). To reduce the time of inserting
+ // blocks, BGs are sorted and the input `Array` are supposed to be sorted so
+ // that we can get better performance of maintaining sorted property.
+ // Use `SameGroup=true` to indicate that all blocks in the array are from the
+ // same group then we will skip checking the group id of each block.
+ //
+ // Note that this aims to have a better management of dirty pages, i.e., the
+ // RSS usage won't grow indefinitely. There's an exception that we may not put
+ // a block to its associated group. While populating new blocks, we may have
+ // blocks cross different groups. However, most cases will fall into same
+ // group and they are supposed to be popped soon. In that case, it's not worth
+ // sorting the array with the almost-sorted property. Therefore, we use
+ // `SameGroup=true` instead.
+ //
+ // The region mutex needs to be held while calling this method.
+ void pushBlocksImpl(CacheT *C, uptr ClassId, CompactPtrT *Array, u32 Size,
+ bool SameGroup = false) {
+ DCHECK_GT(Size, 0U);
+ RegionInfo *Region = getRegionInfo(ClassId);
+
+ auto CreateGroup = [&](uptr GroupId) {
+ BatchGroup *BG = nullptr;
+ TransferBatch *TB = nullptr;
+ if (ClassId == SizeClassMap::BatchClassId) {
+ DCHECK_GE(Size, 2U);
+ BG = reinterpret_cast<BatchGroup *>(
+ decompactPtr(ClassId, Array[Size - 1]));
+ BG->Batches.clear();
+
+ TB = reinterpret_cast<TransferBatch *>(
+ decompactPtr(ClassId, Array[Size - 2]));
+ TB->clear();
+ } else {
+ BG = C->createGroup();
+ BG->Batches.clear();
+
+ TB = C->createBatch(ClassId, nullptr);
+ TB->clear();
+ }
+
+ BG->GroupId = GroupId;
+ BG->Batches.push_front(TB);
+ BG->PushedBlocks = 0;
+ BG->PushedBlocksAtLastCheckpoint = 0;
+ BG->MaxCachedPerBatch =
+ TransferBatch::getMaxCached(getSizeByClassId(ClassId));
+
+ return BG;
+ };
+
+ auto InsertBlocks = [&](BatchGroup *BG, CompactPtrT *Array, u32 Size) {
+ SinglyLinkedList<TransferBatch> &Batches = BG->Batches;
+ TransferBatch *CurBatch = Batches.front();
+ DCHECK_NE(CurBatch, nullptr);
+
+ for (u32 I = 0; I < Size;) {
+ DCHECK_GE(BG->MaxCachedPerBatch, CurBatch->getCount());
+ u16 UnusedSlots =
+ static_cast<u16>(BG->MaxCachedPerBatch - CurBatch->getCount());
+ if (UnusedSlots == 0) {
+ CurBatch = C->createBatch(
+ ClassId,
+ reinterpret_cast<void *>(decompactPtr(ClassId, Array[I])));
+ CurBatch->clear();
+ Batches.push_front(CurBatch);
+ UnusedSlots = BG->MaxCachedPerBatch;
+ }
+ // `UnusedSlots` is u16 so the result will be also fit in u16.
+ u16 AppendSize = static_cast<u16>(Min<u32>(UnusedSlots, Size - I));
+ CurBatch->appendFromArray(&Array[I], AppendSize);
+ I += AppendSize;
+ }
+
+ BG->PushedBlocks += Size;
+ };
+
+ BatchGroup *Cur = Region->FreeList.front();
+
+ if (ClassId == SizeClassMap::BatchClassId) {
+ if (Cur == nullptr) {
+ // Don't need to classify BatchClassId.
+ Cur = CreateGroup(/*GroupId=*/0);
+ Region->FreeList.push_front(Cur);
+ }
+ InsertBlocks(Cur, Array, Size);
+ return;
+ }
+
+ // In the following, `Cur` always points to the BatchGroup for blocks that
+ // will be pushed next. `Prev` is the element right before `Cur`.
+ BatchGroup *Prev = nullptr;
+
+ while (Cur != nullptr && compactPtrGroup(Array[0]) > Cur->GroupId) {
+ Prev = Cur;
+ Cur = Cur->Next;
+ }
+
+ if (Cur == nullptr || compactPtrGroup(Array[0]) != Cur->GroupId) {
+ Cur = CreateGroup(compactPtrGroup(Array[0]));
+ if (Prev == nullptr)
+ Region->FreeList.push_front(Cur);
+ else
+ Region->FreeList.insert(Prev, Cur);
+ }
+
+ // All the blocks are from the same group, just push without checking group
+ // id.
+ if (SameGroup) {
+ InsertBlocks(Cur, Array, Size);
+ return;
+ }
+
+ // The blocks are sorted by group id. Determine the segment of group and
+ // push them to their group together.
+ u32 Count = 1;
+ for (u32 I = 1; I < Size; ++I) {
+ if (compactPtrGroup(Array[I - 1]) != compactPtrGroup(Array[I])) {
+ DCHECK_EQ(compactPtrGroup(Array[I - 1]), Cur->GroupId);
+ InsertBlocks(Cur, Array + I - Count, Count);
+
+ while (Cur != nullptr && compactPtrGroup(Array[I]) > Cur->GroupId) {
+ Prev = Cur;
+ Cur = Cur->Next;
+ }
+
+ if (Cur == nullptr || compactPtrGroup(Array[I]) != Cur->GroupId) {
+ Cur = CreateGroup(compactPtrGroup(Array[I]));
+ DCHECK_NE(Prev, nullptr);
+ Region->FreeList.insert(Prev, Cur);
+ }
+
+ Count = 1;
+ } else {
+ ++Count;
+ }
+ }
+
+ InsertBlocks(Cur, Array + Size - Count, Count);
+ }
+
+ // Pop one TransferBatch from a BatchGroup. The BatchGroup with the smallest
+ // group id will be considered first.
+ //
+ // The region mutex needs to be held while calling this method.
+ TransferBatch *popBatchImpl(CacheT *C, uptr ClassId) {
+ RegionInfo *Region = getRegionInfo(ClassId);
+ if (Region->FreeList.empty())
+ return nullptr;
+
+ SinglyLinkedList<TransferBatch> &Batches =
+ Region->FreeList.front()->Batches;
+ DCHECK(!Batches.empty());
+
+ TransferBatch *B = Batches.front();
+ Batches.pop_front();
+ DCHECK_NE(B, nullptr);
+ DCHECK_GT(B->getCount(), 0U);
+
+ if (Batches.empty()) {
+ BatchGroup *BG = Region->FreeList.front();
+ Region->FreeList.pop_front();
+
+ // We don't keep BatchGroup with zero blocks to avoid empty-checking while
+ // allocating. Note that block used by constructing BatchGroup is recorded
+ // as free blocks in the last element of BatchGroup::Batches. Which means,
+ // once we pop the last TransferBatch, the block is implicitly
+ // deallocated.
+ if (ClassId != SizeClassMap::BatchClassId)
+ C->deallocate(SizeClassMap::BatchClassId, BG);
+ }
+
+ return B;
+ }
+
+ NOINLINE bool populateFreeList(CacheT *C, uptr ClassId, RegionInfo *Region) {
const uptr Size = getSizeByClassId(ClassId);
- const u32 MaxCount = TransferBatch::getMaxCached(Size);
+ const u16 MaxCount = TransferBatch::getMaxCached(Size);
const uptr RegionBeg = Region->RegionBeg;
const uptr MappedUser = Region->MappedUser;
RegionSize >> 20, Size);
Str.output();
}
- return nullptr;
+ return false;
}
if (MappedUser == 0)
Region->Data = Data;
"scudo:primary",
MAP_ALLOWNOMEM | MAP_RESIZABLE |
(useMemoryTagging<Config>(Options.load()) ? MAP_MEMTAG : 0),
- &Region->Data)))
- return nullptr;
+ &Region->Data))) {
+ return false;
+ }
Region->MappedUser += MapSize;
C->getStats().add(StatMapped, MapSize);
}
if (ClassId != SizeClassMap::BatchClassId)
shuffle(ShuffleArray, NumberOfBlocks, &Region->RandState);
for (u32 I = 0; I < NumberOfBlocks;) {
- TransferBatch *B =
- C->createBatch(ClassId, reinterpret_cast<void *>(decompactPtrInternal(
- CompactPtrBase, ShuffleArray[I])));
- if (UNLIKELY(!B))
- return nullptr;
- const u32 N = Min(MaxCount, NumberOfBlocks - I);
- B->setFromArray(&ShuffleArray[I], N);
- Region->FreeList.push_back(B);
+ // `MaxCount` is u16 so the result will also fit in u16.
+ const u16 N = static_cast<u16>(Min<u32>(MaxCount, NumberOfBlocks - I));
+ // Note that the N blocks here may have different group ids. Given that
+ // it only happens when it crosses the group size boundary. Instead of
+ // sorting them, treat them as same group here to avoid sorting the
+ // almost-sorted blocks.
+ pushBlocksImpl(C, ClassId, &ShuffleArray[I], N, /*SameGroup=*/true);
I += N;
}
- TransferBatch *B = Region->FreeList.front();
- Region->FreeList.pop_front();
- DCHECK(B);
- DCHECK_GT(B->getCount(), 0);
const uptr AllocatedUser = Size * NumberOfBlocks;
C->getStats().add(StatFree, AllocatedUser);
Region->AllocatedUser += AllocatedUser;
- return B;
+ return true;
}
void getStats(ScopedString *Str, uptr ClassId, uptr Rss) {
if (BytesPushed < PageSize)
return 0; // Nothing new to release.
+ bool CheckDensity = BlockSize < PageSize / 16U;
// Releasing smaller blocks is expensive, so we want to make sure that a
// significant amount of bytes are free, and that there has been a good
// amount of batches pushed to the freelist before attempting to release.
- if (BlockSize < PageSize / 16U) {
+ if (CheckDensity) {
if (!Force && BytesPushed < Region->AllocatedUser / 16U)
return 0;
- // We want 8x% to 9x% free bytes (the larger the block, the lower the %).
- if ((BytesInFreeList * 100U) / Region->AllocatedUser <
- (100U - 1U - BlockSize / 16U))
- return 0;
}
if (!Force) {
}
}
+ const uptr GroupSize = (1U << GroupSizeLog);
+ const uptr AllocatedUserEnd = Region->AllocatedUser + Region->RegionBeg;
ReleaseRecorder Recorder(Region->RegionBeg, &Region->Data);
+ PageReleaseContext Context(BlockSize, Region->AllocatedUser,
+ /*NumberOfRegions=*/1U);
+
const uptr CompactPtrBase = getCompactPtrBaseByClassId(ClassId);
auto DecompactPtr = [CompactPtrBase](CompactPtrT CompactPtr) {
return decompactPtrInternal(CompactPtrBase, CompactPtr);
};
+ for (BatchGroup &BG : Region->FreeList) {
+ const uptr PushedBytesDelta =
+ BG.PushedBlocks - BG.PushedBlocksAtLastCheckpoint;
+ if (PushedBytesDelta * BlockSize < PageSize)
+ continue;
+
+ // Group boundary does not necessarily have the same alignment as Region.
+ // It may sit across a Region boundary. Which means that we may have the
+ // following two cases,
+ //
+ // 1. Group boundary sits before RegionBeg.
+ //
+ // (BatchGroupBeg)
+ // batchGroupBase RegionBeg BatchGroupEnd
+ // | | |
+ // v v v
+ // +------------+----------------+
+ // \ /
+ // ------ GroupSize ------
+ //
+ // 2. Group boundary sits after RegionBeg.
+ //
+ // (BatchGroupBeg)
+ // RegionBeg batchGroupBase BatchGroupEnd
+ // | | |
+ // v v v
+ // +-----------+-----------------------------+
+ // \ /
+ // ------ GroupSize ------
+ //
+ // Note that in the first case, the group range before RegionBeg is never
+ // used. Therefore, while calculating the used group size, we should
+ // exclude that part to get the correct size.
+ const uptr BatchGroupBeg =
+ Max(batchGroupBase(CompactPtrBase, BG.GroupId), Region->RegionBeg);
+ DCHECK_GE(AllocatedUserEnd, BatchGroupBeg);
+ const uptr BatchGroupEnd =
+ batchGroupBase(CompactPtrBase, BG.GroupId) + GroupSize;
+ const uptr AllocatedGroupSize = AllocatedUserEnd >= BatchGroupEnd
+ ? BatchGroupEnd - BatchGroupBeg
+ : AllocatedUserEnd - BatchGroupBeg;
+ if (AllocatedGroupSize == 0)
+ continue;
+
+ // TransferBatches are pushed in front of BG.Batches. The first one may
+ // not have all caches used.
+ const uptr NumBlocks = (BG.Batches.size() - 1) * BG.MaxCachedPerBatch +
+ BG.Batches.front()->getCount();
+ const uptr BytesInBG = NumBlocks * BlockSize;
+ // Given the randomness property, we try to release the pages only if the
+ // bytes used by free blocks exceed certain proportion of group size. Note
+ // that this heuristic only applies when all the spaces in a BatchGroup
+ // are allocated.
+ if (CheckDensity && (BytesInBG * 100U) / AllocatedGroupSize <
+ (100U - 1U - BlockSize / 16U)) {
+ continue;
+ }
+
+ BG.PushedBlocksAtLastCheckpoint = BG.PushedBlocks;
+ // Note that we don't always visit blocks in each BatchGroup so that we
+ // may miss the chance of releasing certain pages that cross BatchGroups.
+ Context.markFreeBlocks(BG.Batches, DecompactPtr, Region->RegionBeg);
+ }
+
+ if (!Context.hasBlockMarked())
+ return 0;
+
auto SkipRegion = [](UNUSED uptr RegionIndex) { return false; };
- releaseFreeMemoryToOS(Region->FreeList, Region->AllocatedUser, 1U,
- BlockSize, &Recorder, DecompactPtr, SkipRegion);
+ releaseFreeMemoryToOS(Context, Recorder, SkipRegion);
if (Recorder.getReleasedRangesCount() > 0) {
Region->ReleaseInfo.PushedBlocksAtLastRelease =
namespace scudo {
-HybridMutex PackedCounterArray::Mutex = {};
-uptr PackedCounterArray::StaticBuffer[PackedCounterArray::StaticBufferCount];
+HybridMutex RegionPageMap::Mutex = {};
+uptr RegionPageMap::StaticBuffer[RegionPageMap::StaticBufferCount];
} // namespace scudo
MapPlatformData *Data = nullptr;
};
-// A packed array of Counters. Each counter occupies 2^N bits, enough to store
-// counter's MaxValue. Ctor will try to use a static buffer first, and if that
-// fails (the buffer is too small or already locked), will allocate the
+// A Region page map is used to record the usage of pages in the regions. It
+// implements a packed array of Counters. Each counter occupies 2^N bits, enough
+// to store counter's MaxValue. Ctor will try to use a static buffer first, and
+// if that fails (the buffer is too small or already locked), will allocate the
// required Buffer via map(). The caller is expected to check whether the
// initialization was successful by checking isAllocated() result. For
// performance sake, none of the accessors check the validity of the arguments,
// It is assumed that Index is always in [0, N) range and the value is not
// incremented past MaxValue.
-class PackedCounterArray {
+class RegionPageMap {
public:
- PackedCounterArray(uptr NumberOfRegions, uptr CountersPerRegion,
- uptr MaxValue)
- : Regions(NumberOfRegions), NumCounters(CountersPerRegion) {
- DCHECK_GT(Regions, 0);
- DCHECK_GT(NumCounters, 0);
+ RegionPageMap()
+ : Regions(0),
+ NumCounters(0),
+ CounterSizeBitsLog(0),
+ CounterMask(0),
+ PackingRatioLog(0),
+ BitOffsetMask(0),
+ SizePerRegion(0),
+ BufferSize(0),
+ Buffer(nullptr) {}
+ RegionPageMap(uptr NumberOfRegions, uptr CountersPerRegion, uptr MaxValue) {
+ reset(NumberOfRegions, CountersPerRegion, MaxValue);
+ }
+ ~RegionPageMap() {
+ if (!isAllocated())
+ return;
+ if (Buffer == &StaticBuffer[0])
+ Mutex.unlock();
+ else
+ unmap(reinterpret_cast<void *>(Buffer),
+ roundUpTo(BufferSize, getPageSizeCached()));
+ Buffer = nullptr;
+ }
+
+ void reset(uptr NumberOfRegion, uptr CountersPerRegion, uptr MaxValue) {
+ DCHECK_GT(NumberOfRegion, 0);
+ DCHECK_GT(CountersPerRegion, 0);
DCHECK_GT(MaxValue, 0);
+
+ Regions = NumberOfRegion;
+ NumCounters = CountersPerRegion;
+
constexpr uptr MaxCounterBits = sizeof(*Buffer) * 8UL;
// Rounding counter storage size up to the power of two allows for using
// bit shifts calculating particular counter's Index and offset.
Buffer = &StaticBuffer[0];
memset(Buffer, 0, BufferSize);
} else {
+ // When using a heap-based buffer, precommit the pages backing the
+ // Vmar by passing |MAP_PRECOMMIT| flag. This allows an optimization
+ // where page fault exceptions are skipped as the allocated memory
+ // is accessed.
+ const uptr MmapFlags =
+ MAP_ALLOWNOMEM | (SCUDO_FUCHSIA ? MAP_PRECOMMIT : 0);
Buffer = reinterpret_cast<uptr *>(
map(nullptr, roundUpTo(BufferSize, getPageSizeCached()),
- "scudo:counters", MAP_ALLOWNOMEM));
+ "scudo:counters", MmapFlags, &MapData));
}
}
- ~PackedCounterArray() {
- if (!isAllocated())
- return;
- if (Buffer == &StaticBuffer[0])
- Mutex.unlock();
- else
- unmap(reinterpret_cast<void *>(Buffer),
- roundUpTo(BufferSize, getPageSizeCached()));
- }
bool isAllocated() const { return !!Buffer; }
const uptr Index = I >> PackingRatioLog;
const uptr BitOffset = (I & BitOffsetMask) << CounterSizeBitsLog;
DCHECK_LT(BitOffset, SCUDO_WORDSIZE);
+ DCHECK_EQ(isAllCounted(Region, I), false);
Buffer[Region * SizePerRegion + Index] += static_cast<uptr>(1U)
<< BitOffset;
}
inc(Region, I);
}
+ // Set the counter to the max value. Note that the max number of blocks in a
+ // page may vary. To provide an easier way to tell if all the blocks are
+ // counted for different pages, set to the same max value to denote the
+ // all-counted status.
+ void setAsAllCounted(uptr Region, uptr I) const {
+ DCHECK_LE(get(Region, I), CounterMask);
+ const uptr Index = I >> PackingRatioLog;
+ const uptr BitOffset = (I & BitOffsetMask) << CounterSizeBitsLog;
+ DCHECK_LT(BitOffset, SCUDO_WORDSIZE);
+ Buffer[Region * SizePerRegion + Index] |= CounterMask << BitOffset;
+ }
+ bool isAllCounted(uptr Region, uptr I) const {
+ return get(Region, I) == CounterMask;
+ }
+
uptr getBufferSize() const { return BufferSize; }
static const uptr StaticBufferCount = 2048U;
private:
- const uptr Regions;
- const uptr NumCounters;
+ uptr Regions;
+ uptr NumCounters;
uptr CounterSizeBitsLog;
uptr CounterMask;
uptr PackingRatioLog;
uptr SizePerRegion;
uptr BufferSize;
uptr *Buffer;
+ [[no_unique_address]] MapPlatformData MapData = {};
static HybridMutex Mutex;
static uptr StaticBuffer[StaticBufferCount];
template <class ReleaseRecorderT> class FreePagesRangeTracker {
public:
- explicit FreePagesRangeTracker(ReleaseRecorderT *Recorder)
+ explicit FreePagesRangeTracker(ReleaseRecorderT &Recorder)
: Recorder(Recorder), PageSizeLog(getLog2(getPageSizeCached())) {}
- void processNextPage(bool Freed) {
- if (Freed) {
+ void processNextPage(bool Released) {
+ if (Released) {
if (!InRange) {
CurrentRangeStatePage = CurrentPage;
InRange = true;
private:
void closeOpenedRange() {
if (InRange) {
- Recorder->releasePageRangeToOS((CurrentRangeStatePage << PageSizeLog),
- (CurrentPage << PageSizeLog));
+ Recorder.releasePageRangeToOS((CurrentRangeStatePage << PageSizeLog),
+ (CurrentPage << PageSizeLog));
InRange = false;
}
}
- ReleaseRecorderT *const Recorder;
+ ReleaseRecorderT &Recorder;
const uptr PageSizeLog;
bool InRange = false;
uptr CurrentPage = 0;
uptr CurrentRangeStatePage = 0;
};
-template <class TransferBatchT, class ReleaseRecorderT, typename DecompactPtrT,
- typename SkipRegionT>
-NOINLINE void
-releaseFreeMemoryToOS(const IntrusiveList<TransferBatchT> &FreeList,
- uptr RegionSize, uptr NumberOfRegions, uptr BlockSize,
- ReleaseRecorderT *Recorder, DecompactPtrT DecompactPtr,
- SkipRegionT SkipRegion) {
- const uptr PageSize = getPageSizeCached();
-
- // Figure out the number of chunks per page and whether we can take a fast
- // path (the number of chunks per page is the same for all pages).
- uptr FullPagesBlockCountMax;
- bool SameBlockCountPerPage;
- if (BlockSize <= PageSize) {
- if (PageSize % BlockSize == 0) {
- // Same number of chunks per page, no cross overs.
- FullPagesBlockCountMax = PageSize / BlockSize;
- SameBlockCountPerPage = true;
- } else if (BlockSize % (PageSize % BlockSize) == 0) {
- // Some chunks are crossing page boundaries, which means that the page
- // contains one or two partial chunks, but all pages contain the same
- // number of chunks.
- FullPagesBlockCountMax = PageSize / BlockSize + 1;
- SameBlockCountPerPage = true;
- } else {
- // Some chunks are crossing page boundaries, which means that the page
- // contains one or two partial chunks.
- FullPagesBlockCountMax = PageSize / BlockSize + 2;
- SameBlockCountPerPage = false;
- }
- } else {
- if (BlockSize % PageSize == 0) {
- // One chunk covers multiple pages, no cross overs.
- FullPagesBlockCountMax = 1;
- SameBlockCountPerPage = true;
+struct PageReleaseContext {
+ PageReleaseContext(uptr BlockSize, uptr RegionSize, uptr NumberOfRegions) :
+ BlockSize(BlockSize),
+ RegionSize(RegionSize),
+ NumberOfRegions(NumberOfRegions) {
+ PageSize = getPageSizeCached();
+ if (BlockSize <= PageSize) {
+ if (PageSize % BlockSize == 0) {
+ // Same number of chunks per page, no cross overs.
+ FullPagesBlockCountMax = PageSize / BlockSize;
+ SameBlockCountPerPage = true;
+ } else if (BlockSize % (PageSize % BlockSize) == 0) {
+ // Some chunks are crossing page boundaries, which means that the page
+ // contains one or two partial chunks, but all pages contain the same
+ // number of chunks.
+ FullPagesBlockCountMax = PageSize / BlockSize + 1;
+ SameBlockCountPerPage = true;
+ } else {
+ // Some chunks are crossing page boundaries, which means that the page
+ // contains one or two partial chunks.
+ FullPagesBlockCountMax = PageSize / BlockSize + 2;
+ SameBlockCountPerPage = false;
+ }
} else {
- // One chunk covers multiple pages, Some chunks are crossing page
- // boundaries. Some pages contain one chunk, some contain two.
- FullPagesBlockCountMax = 2;
- SameBlockCountPerPage = false;
+ if (BlockSize % PageSize == 0) {
+ // One chunk covers multiple pages, no cross overs.
+ FullPagesBlockCountMax = 1;
+ SameBlockCountPerPage = true;
+ } else {
+ // One chunk covers multiple pages, Some chunks are crossing page
+ // boundaries. Some pages contain one chunk, some contain two.
+ FullPagesBlockCountMax = 2;
+ SameBlockCountPerPage = false;
+ }
}
+
+ PagesCount = roundUpTo(RegionSize, PageSize) / PageSize;
+ PageSizeLog = getLog2(PageSize);
+ RoundedRegionSize = PagesCount << PageSizeLog;
+ RoundedSize = NumberOfRegions * RoundedRegionSize;
}
- const uptr PagesCount = roundUpTo(RegionSize, PageSize) / PageSize;
- PackedCounterArray Counters(NumberOfRegions, PagesCount,
- FullPagesBlockCountMax);
- if (!Counters.isAllocated())
- return;
-
- const uptr PageSizeLog = getLog2(PageSize);
- const uptr RoundedRegionSize = PagesCount << PageSizeLog;
- const uptr RoundedSize = NumberOfRegions * RoundedRegionSize;
-
- // Iterate over free chunks and count how many free chunks affect each
- // allocated page.
- if (BlockSize <= PageSize && PageSize % BlockSize == 0) {
- // Each chunk affects one page only.
- for (const auto &It : FreeList) {
- for (u32 I = 0; I < It.getCount(); I++) {
- const uptr P = DecompactPtr(It.get(I)) - Recorder->getBase();
- if (P >= RoundedSize)
- continue;
- const uptr RegionIndex = NumberOfRegions == 1U ? 0 : P / RegionSize;
- const uptr PInRegion = P - RegionIndex * RegionSize;
- Counters.inc(RegionIndex, PInRegion >> PageSizeLog);
+ // PageMap is lazily allocated when markFreeBlocks() is invoked.
+ bool hasBlockMarked() const {
+ return PageMap.isAllocated();
+ }
+
+ void ensurePageMapAllocated() {
+ if (PageMap.isAllocated())
+ return;
+ PageMap.reset(NumberOfRegions, PagesCount, FullPagesBlockCountMax);
+ DCHECK(PageMap.isAllocated());
+ }
+
+ template<class TransferBatchT, typename DecompactPtrT>
+ void markFreeBlocks(const IntrusiveList<TransferBatchT> &FreeList,
+ DecompactPtrT DecompactPtr, uptr Base) {
+ ensurePageMapAllocated();
+
+ // Iterate over free chunks and count how many free chunks affect each
+ // allocated page.
+ if (BlockSize <= PageSize && PageSize % BlockSize == 0) {
+ // Each chunk affects one page only.
+ for (const auto &It : FreeList) {
+ for (u16 I = 0; I < It.getCount(); I++) {
+ const uptr P = DecompactPtr(It.get(I)) - Base;
+ if (P >= RoundedSize)
+ continue;
+ const uptr RegionIndex = NumberOfRegions == 1U ? 0 : P / RegionSize;
+ const uptr PInRegion = P - RegionIndex * RegionSize;
+ PageMap.inc(RegionIndex, PInRegion >> PageSizeLog);
+ }
}
- }
- } else {
- // In all other cases chunks might affect more than one page.
- DCHECK_GE(RegionSize, BlockSize);
- const uptr LastBlockInRegion = ((RegionSize / BlockSize) - 1U) * BlockSize;
- for (const auto &It : FreeList) {
- for (u32 I = 0; I < It.getCount(); I++) {
- const uptr P = DecompactPtr(It.get(I)) - Recorder->getBase();
- if (P >= RoundedSize)
- continue;
- const uptr RegionIndex = NumberOfRegions == 1U ? 0 : P / RegionSize;
- uptr PInRegion = P - RegionIndex * RegionSize;
- Counters.incRange(RegionIndex, PInRegion >> PageSizeLog,
- (PInRegion + BlockSize - 1) >> PageSizeLog);
- // The last block in a region might straddle a page, so if it's
- // free, we mark the following "pretend" memory block(s) as free.
- if (PInRegion == LastBlockInRegion) {
- PInRegion += BlockSize;
- while (PInRegion < RoundedRegionSize) {
- Counters.incRange(RegionIndex, PInRegion >> PageSizeLog,
- (PInRegion + BlockSize - 1) >> PageSizeLog);
+ } else {
+ // In all other cases chunks might affect more than one page.
+ DCHECK_GE(RegionSize, BlockSize);
+ const uptr LastBlockInRegion =
+ ((RegionSize / BlockSize) - 1U) * BlockSize;
+ for (const auto &It : FreeList) {
+ for (u16 I = 0; I < It.getCount(); I++) {
+ const uptr P = DecompactPtr(It.get(I)) - Base;
+ if (P >= RoundedSize)
+ continue;
+ const uptr RegionIndex = NumberOfRegions == 1U ? 0 : P / RegionSize;
+ uptr PInRegion = P - RegionIndex * RegionSize;
+ PageMap.incRange(RegionIndex, PInRegion >> PageSizeLog,
+ (PInRegion + BlockSize - 1) >> PageSizeLog);
+ // The last block in a region might straddle a page, so if it's
+ // free, we mark the following "pretend" memory block(s) as free.
+ if (PInRegion == LastBlockInRegion) {
PInRegion += BlockSize;
+ while (PInRegion < RoundedRegionSize) {
+ PageMap.incRange(RegionIndex, PInRegion >> PageSizeLog,
+ (PInRegion + BlockSize - 1) >> PageSizeLog);
+ PInRegion += BlockSize;
+ }
}
}
}
}
}
+ uptr BlockSize;
+ uptr RegionSize;
+ uptr NumberOfRegions;
+ uptr PageSize;
+ uptr PagesCount;
+ uptr PageSizeLog;
+ uptr RoundedRegionSize;
+ uptr RoundedSize;
+ uptr FullPagesBlockCountMax;
+ bool SameBlockCountPerPage;
+ RegionPageMap PageMap;
+};
+
+// Try to release the page which doesn't have any in-used block, i.e., they are
+// all free blocks. The `PageMap` will record the number of free blocks in each
+// page.
+template <class ReleaseRecorderT, typename SkipRegionT>
+NOINLINE void
+releaseFreeMemoryToOS(PageReleaseContext &Context,
+ ReleaseRecorderT &Recorder, SkipRegionT SkipRegion) {
+ const uptr PageSize = Context.PageSize;
+ const uptr BlockSize = Context.BlockSize;
+ const uptr PagesCount = Context.PagesCount;
+ const uptr NumberOfRegions = Context.NumberOfRegions;
+ const uptr FullPagesBlockCountMax = Context.FullPagesBlockCountMax;
+ const bool SameBlockCountPerPage = Context.SameBlockCountPerPage;
+ RegionPageMap &PageMap = Context.PageMap;
+
// Iterate over pages detecting ranges of pages with chunk Counters equal
// to the expected number of chunks for the particular page.
FreePagesRangeTracker<ReleaseRecorderT> RangeTracker(Recorder);
RangeTracker.skipPages(PagesCount);
continue;
}
- for (uptr J = 0; J < PagesCount; J++)
- RangeTracker.processNextPage(Counters.get(I, J) ==
- FullPagesBlockCountMax);
+ for (uptr J = 0; J < PagesCount; J++) {
+ const bool CanRelease = PageMap.get(I, J) == FullPagesBlockCountMax;
+ if (CanRelease)
+ PageMap.setAsAllCounted(I, J);
+ RangeTracker.processNextPage(CanRelease);
+ }
}
} else {
// Slow path, go through the pages keeping count how many chunks affect
}
}
PrevPageBoundary = PageBoundary;
- RangeTracker.processNextPage(Counters.get(I, J) == BlocksPerPage);
+ const bool CanRelease = PageMap.get(I, J) == BlocksPerPage;
+ if (CanRelease)
+ PageMap.setAsAllCounted(I, J);
+ RangeTracker.processNextPage(CanRelease);
}
}
}
RangeTracker.finish();
}
+// An overload releaseFreeMemoryToOS which doesn't require the page usage
+// information after releasing.
+template <class TransferBatchT, class ReleaseRecorderT, typename DecompactPtrT,
+ typename SkipRegionT>
+NOINLINE void
+releaseFreeMemoryToOS(const IntrusiveList<TransferBatchT> &FreeList,
+ uptr RegionSize, uptr NumberOfRegions, uptr BlockSize,
+ ReleaseRecorderT &Recorder, DecompactPtrT DecompactPtr,
+ SkipRegionT SkipRegion) {
+ PageReleaseContext Context(BlockSize, RegionSize, NumberOfRegions);
+ Context.markFreeBlocks(FreeList, DecompactPtr, Recorder.getBase());
+ releaseFreeMemoryToOS(Context, Recorder, SkipRegion);
+}
+
} // namespace scudo
#endif // SCUDO_RELEASE_H_
inline void NORETURN trap() { __builtin_trap(); }
+void NORETURN reportSoftRSSLimit(uptr RssLimitMb) {
+ ScopedErrorReport Report;
+ Report.append("Soft RSS limit of %zu MB exhausted, current RSS is %zu MB\n",
+ RssLimitMb, GetRSS() >> 20);
+}
+
+void NORETURN reportHardRSSLimit(uptr RssLimitMb) {
+ ScopedErrorReport Report;
+ Report.append("Hard RSS limit of %zu MB exhausted, current RSS is %zu MB\n",
+ RssLimitMb, GetRSS() >> 20);
+}
+
// This could potentially be called recursively if a CHECK fails in the reports.
void NORETURN reportCheckFailed(const char *File, int Line,
const char *Condition, u64 Value1, u64 Value2) {
void NORETURN reportAllocationSizeTooBig(uptr UserSize, uptr TotalSize,
uptr MaxSize);
void NORETURN reportOutOfMemory(uptr RequestedSize);
+void NORETURN reportSoftRSSLimit(uptr RssLimitMb);
+void NORETURN reportHardRSSLimit(uptr RssLimitMb);
enum class AllocatorAction : u8 {
Recycling,
Deallocating,
--- /dev/null
+//===-- common.cpp ----------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "rss_limit_checker.h"
+#include "atomic_helpers.h"
+#include "string_utils.h"
+
+namespace scudo {
+
+void RssLimitChecker::check(u64 NextCheck) {
+ // The interval for the checks is 250ms.
+ static constexpr u64 CheckInterval = 250 * 1000000;
+
+ // Early return in case another thread already did the calculation.
+ if (!atomic_compare_exchange_strong(&RssNextCheckAtNS, &NextCheck,
+ getMonotonicTime() + CheckInterval,
+ memory_order_relaxed)) {
+ return;
+ }
+
+ const uptr CurrentRssMb = GetRSS() >> 20;
+
+ RssLimitExceeded Result = RssLimitExceeded::Neither;
+ if (UNLIKELY(HardRssLimitMb && HardRssLimitMb < CurrentRssMb))
+ Result = RssLimitExceeded::Hard;
+ else if (UNLIKELY(SoftRssLimitMb && SoftRssLimitMb < CurrentRssMb))
+ Result = RssLimitExceeded::Soft;
+
+ atomic_store_relaxed(&RssLimitStatus, static_cast<u8>(Result));
+}
+
+} // namespace scudo
--- /dev/null
+//===-- common.h ------------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SCUDO_RSS_LIMIT_CHECKER_H_
+#define SCUDO_RSS_LIMIT_CHECKER_H_
+
+#include "atomic_helpers.h"
+#include "common.h"
+#include "internal_defs.h"
+
+namespace scudo {
+
+class RssLimitChecker {
+public:
+ enum RssLimitExceeded {
+ Neither,
+ Soft,
+ Hard,
+ };
+
+ void init(int SoftRssLimitMb, int HardRssLimitMb) {
+ CHECK_GE(SoftRssLimitMb, 0);
+ CHECK_GE(HardRssLimitMb, 0);
+ this->SoftRssLimitMb = static_cast<uptr>(SoftRssLimitMb);
+ this->HardRssLimitMb = static_cast<uptr>(HardRssLimitMb);
+ }
+
+ // Opportunistic RSS limit check. This will update the RSS limit status, if
+ // it can, every 250ms, otherwise it will just return the current one.
+ RssLimitExceeded getRssLimitExceeded() {
+ if (!HardRssLimitMb && !SoftRssLimitMb)
+ return RssLimitExceeded::Neither;
+
+ u64 NextCheck = atomic_load_relaxed(&RssNextCheckAtNS);
+ u64 Now = getMonotonicTime();
+
+ if (UNLIKELY(Now >= NextCheck))
+ check(NextCheck);
+
+ return static_cast<RssLimitExceeded>(atomic_load_relaxed(&RssLimitStatus));
+ }
+
+ uptr getSoftRssLimit() const { return SoftRssLimitMb; }
+ uptr getHardRssLimit() const { return HardRssLimitMb; }
+
+private:
+ void check(u64 NextCheck);
+
+ uptr SoftRssLimitMb = 0;
+ uptr HardRssLimitMb = 0;
+
+ atomic_u64 RssNextCheckAtNS = {};
+ atomic_u8 RssLimitStatus = {};
+};
+
+} // namespace scudo
+
+#endif // SCUDO_RSS_LIMIT_CHECKER_H_
}
}
+// Template specialization to avoid producing zero-length array
+template <typename T, size_t Size> class NonZeroLengthArray {
+public:
+ T &operator[](uptr Idx) { return values[Idx]; }
+
+private:
+ T values[Size];
+};
+template <typename T> class NonZeroLengthArray<T, 0> {
+public:
+ T &operator[](uptr UNUSED Idx) { UNREACHABLE("Unsupported!"); }
+};
+
template <typename Config> class MapAllocatorCache {
public:
// Ensure the default maximum specified fits the array.
const u32 MaxCount = atomic_load_relaxed(&MaxEntriesCount);
bool Found = false;
CachedBlock Entry;
- uptr HeaderPos;
+ uptr HeaderPos = 0;
{
ScopedLock L(Mutex);
if (EntriesCount == 0)
atomic_s32 ReleaseToOsIntervalMs = {};
CachedBlock Entries[Config::SecondaryCacheEntriesArraySize] = {};
- CachedBlock Quarantine[Config::SecondaryCacheQuarantineSize] = {};
+ NonZeroLengthArray<CachedBlock, Config::SecondaryCacheQuarantineSize>
+ Quarantine = {};
};
template <typename Config> class MapAllocator {
}
}
- uptr canCache(uptr Size) { return Cache.canCache(Size); }
+ bool canCache(uptr Size) { return Cache.canCache(Size); }
bool setOption(Option O, sptr Value) { return Cache.setOption(O, Value); }
FillContentsMode FillContents) {
if (Options.get(OptionBit::AddLargeAllocationSlack))
Size += 1UL << SCUDO_MIN_ALIGNMENT_LOG;
- Alignment = Max(Alignment, 1UL << SCUDO_MIN_ALIGNMENT_LOG);
+ Alignment = Max(Alignment, uptr(1U) << SCUDO_MIN_ALIGNMENT_LOG);
const uptr PageSize = getPageSizeCached();
uptr RoundedSize =
roundUpTo(roundUpTo(Size, Alignment) + LargeBlock::getHeaderSize() +
template <typename Config>
void MapAllocator<Config>::getStats(ScopedString *Str) const {
- Str->append(
- "Stats: MapAllocator: allocated %zu times (%zuK), freed %zu times "
- "(%zuK), remains %zu (%zuK) max %zuM\n",
- NumberOfAllocs, AllocatedBytes >> 10, NumberOfFrees, FreedBytes >> 10,
- NumberOfAllocs - NumberOfFrees, (AllocatedBytes - FreedBytes) >> 10,
- LargestSize >> 20);
+ Str->append("Stats: MapAllocator: allocated %u times (%zuK), freed %u times "
+ "(%zuK), remains %u (%zuK) max %zuM\n",
+ NumberOfAllocs, AllocatedBytes >> 10, NumberOfFrees,
+ FreedBytes >> 10, NumberOfAllocs - NumberOfFrees,
+ (AllocatedBytes - FreedBytes) >> 10, LargestSize >> 20);
}
} // namespace scudo
}
template <typename Config> struct SizeClassMapBase {
- static u32 getMaxCachedHint(uptr Size) {
+ static u16 getMaxCachedHint(uptr Size) {
DCHECK_NE(Size, 0);
u32 N;
// Force a 32-bit division if the template parameters allow for it.
N = static_cast<u32>((1UL << Config::MaxBytesCachedLog) / Size);
else
N = (1U << Config::MaxBytesCachedLog) / static_cast<u32>(Size);
- return Max(1U, Min(Config::MaxNumCachedHint, N));
+
+ // Note that Config::MaxNumCachedHint is u16 so the result is guaranteed to
+ // fit in u16.
+ return static_cast<u16>(Max(1U, Min<u32>(Config::MaxNumCachedHint, N)));
}
};
static const uptr M = (1UL << S) - 1;
public:
- static const u32 MaxNumCachedHint = Config::MaxNumCachedHint;
+ static const u16 MaxNumCachedHint = Config::MaxNumCachedHint;
static const uptr MaxSize = (1UL << Config::MaxSizeLog) + Config::SizeDelta;
static const uptr NumClasses =
return MidClass + 1 + scaledLog2(Size - 1, Config::MidSizeLog, S);
}
- static u32 getMaxCachedHint(uptr Size) {
+ static u16 getMaxCachedHint(uptr Size) {
DCHECK_LE(Size, MaxSize);
return Base::getMaxCachedHint(Size);
}
static constexpr LSBTable LTable = {};
public:
- static const u32 MaxNumCachedHint = Config::MaxNumCachedHint;
+ static const u16 MaxNumCachedHint = Config::MaxNumCachedHint;
static const uptr NumClasses = ClassesSize + 1;
static_assert(NumClasses < 256, "");
return SzTable.Tab[scaledLog2(Size - 1, Config::MidSizeLog, S)];
}
- static u32 getMaxCachedHint(uptr Size) {
+ static u16 getMaxCachedHint(uptr Size) {
DCHECK_LE(Size, MaxSize);
return Base::getMaxCachedHint(Size);
}
static const uptr MinSizeLog = 5;
static const uptr MidSizeLog = 8;
static const uptr MaxSizeLog = 17;
- static const u32 MaxNumCachedHint = 14;
+ static const u16 MaxNumCachedHint = 14;
static const uptr MaxBytesCachedLog = 10;
static const uptr SizeDelta = 0;
};
static const uptr MinSizeLog = 5;
static const uptr MidSizeLog = 8;
static const uptr MaxSizeLog = 17;
- static const u32 MaxNumCachedHint = 10;
+ static const u16 MaxNumCachedHint = 12;
static const uptr MaxBytesCachedLog = 10;
static const uptr SizeDelta = Chunk::getHeaderSize();
};
static const uptr MinSizeLog = 4;
static const uptr MidSizeLog = 6;
static const uptr MaxSizeLog = 16;
- static const u32 MaxNumCachedHint = 13;
+ static const u16 MaxNumCachedHint = 13;
static const uptr MaxBytesCachedLog = 13;
static constexpr u32 Classes[] = {
static const uptr MinSizeLog = 4;
static const uptr MidSizeLog = 7;
static const uptr MaxSizeLog = 16;
- static const u32 MaxNumCachedHint = 14;
+ static const u16 MaxNumCachedHint = 14;
static const uptr MaxBytesCachedLog = 13;
static constexpr u32 Classes[] = {
static const uptr MinSizeLog = 4;
static const uptr MidSizeLog = 8;
static const uptr MaxSizeLog = 14;
- static const u32 MaxNumCachedHint = 13;
+ static const u16 MaxNumCachedHint = 13;
static const uptr MaxBytesCachedLog = 10;
static const uptr SizeDelta = Chunk::getHeaderSize();
#else
static const uptr MinSizeLog = 3;
static const uptr MidSizeLog = 7;
static const uptr MaxSizeLog = 14;
- static const u32 MaxNumCachedHint = 14;
+ static const u16 MaxNumCachedHint = 14;
static const uptr MaxBytesCachedLog = 10;
static const uptr SizeDelta = Chunk::getHeaderSize();
#endif
static const uptr MinSizeLog = 7;
static const uptr MidSizeLog = 7;
static const uptr MaxSizeLog = 7;
- static const u32 MaxNumCachedHint = 8;
+ static const u16 MaxNumCachedHint = 12;
static const uptr MaxBytesCachedLog = 10;
static const uptr SizeDelta = 0;
};
const uptr L = S ? getMostSignificantSetBitIndex(S) : 0;
const uptr Cached = SCMap::getMaxCachedHint(S) * S;
Buffer.append(
- "C%02zu => S: %zu diff: +%zu %02zu%% L %zu Cached: %zu %zu; id %zu\n",
- I, S, D, P, L, SCMap::getMaxCachedHint(S), Cached,
+ "C%02zu => S: %zu diff: +%zu %02zu%% L %zu Cached: %u %zu; id %zu\n", I,
+ S, D, P, L, SCMap::getMaxCachedHint(S), Cached,
SCMap::getClassIdBySize(S));
TotalCached += Cached;
PrevS = S;
Buffer.output();
}
-template <typename SCMap> static void validateMap() {
+template <typename SCMap> static UNUSED void validateMap() {
for (uptr C = 0; C < SCMap::NumClasses; C++) {
if (C == SCMap::BatchClassId)
continue;
va_end(ArgsCopy);
}
-FORMAT(2, 3)
void ScopedString::append(const char *Format, ...) {
va_list Args;
va_start(Args, Format);
va_end(Args);
}
-FORMAT(1, 2)
void Printf(const char *Format, ...) {
va_list Args;
va_start(Args, Format);
String.push_back('\0');
}
void append(const char *Format, va_list Args);
- void append(const char *Format, ...);
+ void append(const char *Format, ...) FORMAT(2, 3);
void output() const { outputRaw(String.data()); }
+ void reserve(size_t Size) { String.reserve(Size + 1); }
private:
Vector<char> String;
};
-int formatString(char *Buffer, uptr BufferLength, const char *Format, ...);
-void Printf(const char *Format, ...);
+int formatString(char *Buffer, uptr BufferLength, const char *Format, ...)
+ FORMAT(3, 4);
+void Printf(const char *Format, ...) FORMAT(1, 2);
} // namespace scudo
set(SCUDO_UNITTEST_CFLAGS
${COMPILER_RT_UNITTEST_CFLAGS}
${COMPILER_RT_GTEST_CFLAGS}
+ ${SANITIZER_TEST_CXX_CFLAGS}
-I${COMPILER_RT_SOURCE_DIR}/include
-I${COMPILER_RT_SOURCE_DIR}/lib
-I${COMPILER_RT_SOURCE_DIR}/lib/scudo/standalone
set(SCUDO_TEST_ARCH ${SCUDO_STANDALONE_SUPPORTED_ARCH})
# gtests requires c++
-set(LINK_FLAGS ${COMPILER_RT_UNITTEST_LINK_FLAGS})
-foreach(lib ${SANITIZER_TEST_CXX_LIBRARIES})
- list(APPEND LINK_FLAGS -l${lib})
-endforeach()
-list(APPEND LINK_FLAGS -pthread)
+set(SCUDO_UNITTEST_LINK_FLAGS
+ ${COMPILER_RT_UNITTEST_LINK_FLAGS}
+ ${COMPILER_RT_UNWINDER_LINK_LIBS}
+ ${SANITIZER_TEST_CXX_LIBRARIES})
+list(APPEND SCUDO_UNITTEST_LINK_FLAGS -pthread -no-pie)
# Linking against libatomic is required with some compilers
check_library_exists(atomic __atomic_load_8 "" COMPILER_RT_HAS_LIBATOMIC)
if (COMPILER_RT_HAS_LIBATOMIC)
- list(APPEND LINK_FLAGS -latomic)
+ list(APPEND SCUDO_UNITTEST_LINK_FLAGS -latomic)
endif()
set(SCUDO_TEST_HEADERS
"${testname}-${arch}-Test" ${arch}
SOURCES ${TEST_SOURCES} ${COMPILER_RT_GTEST_SOURCE}
COMPILE_DEPS ${SCUDO_TEST_HEADERS}
- DEPS gtest scudo_standalone
+ DEPS llvm_gtest scudo_standalone
RUNTIME ${RUNTIME}
CFLAGS ${SCUDO_UNITTEST_CFLAGS}
- LINK_FLAGS ${LINK_FLAGS})
+ LINK_FLAGS ${SCUDO_UNITTEST_LINK_FLAGS})
endforeach()
endif()
endmacro()
#include <string.h>
-scudo::u16 computeSoftwareChecksum(scudo::u32 Seed, scudo::uptr *Array,
- scudo::uptr ArraySize) {
+static scudo::u16 computeSoftwareChecksum(scudo::u32 Seed, scudo::uptr *Array,
+ scudo::uptr ArraySize) {
scudo::u16 Checksum = static_cast<scudo::u16>(Seed & 0xffff);
for (scudo::uptr I = 0; I < ArraySize; I++)
Checksum = scudo::computeBSDChecksum(Checksum, Array[I]);
return Checksum;
}
-scudo::u16 computeHardwareChecksum(scudo::u32 Seed, scudo::uptr *Array,
- scudo::uptr ArraySize) {
+static scudo::u16 computeHardwareChecksum(scudo::u32 Seed, scudo::uptr *Array,
+ scudo::uptr ArraySize) {
scudo::u32 Crc = Seed;
for (scudo::uptr I = 0; I < ArraySize; I++)
Crc = scudo::computeHardwareCRC32(Crc, Array[I]);
// This verifies that flipping bits in the data being checksummed produces a
// different checksum. We do not use random data to avoid flakyness.
-template <ComputeChecksum F> void verifyChecksumFunctionBitFlip() {
+template <ComputeChecksum F> static void verifyChecksumFunctionBitFlip() {
scudo::uptr Array[sizeof(scudo::u64) / sizeof(scudo::uptr)];
const scudo::uptr ArraySize = ARRAY_SIZE(Array);
memset(Array, 0xaa, sizeof(Array));
#include "tests/scudo_unit_test.h"
#include "allocator_config.h"
+#include "chunk.h"
#include "combined.h"
#include <condition_variable>
#define SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, TYPE) \
using FIXTURE##NAME##_##TYPE = FIXTURE##NAME<scudo::TYPE>; \
- TEST_F(FIXTURE##NAME##_##TYPE, NAME) { Run(); }
+ TEST_F(FIXTURE##NAME##_##TYPE, NAME) { FIXTURE##NAME<scudo::TYPE>::Run(); }
#define SCUDO_TYPED_TEST(FIXTURE, NAME) \
template <class TypeParam> \
for (scudo::uptr AlignLog = MinAlignLog; AlignLog <= 16U; AlignLog++) {
const scudo::uptr Align = 1U << AlignLog;
for (scudo::sptr Delta = -32; Delta <= 32; Delta++) {
- if (static_cast<scudo::sptr>(1U << SizeLog) + Delta <= 0)
+ if (static_cast<scudo::sptr>(1U << SizeLog) + Delta < 0)
continue;
const scudo::uptr Size = (1U << SizeLog) + Delta;
void *P = Allocator->allocate(Size, Origin, Align);
static const scudo::uptr MinSizeLog = 10;
static const scudo::uptr MidSizeLog = 10;
static const scudo::uptr MaxSizeLog = 13;
- static const scudo::u32 MaxNumCachedHint = 4;
+ static const scudo::u16 MaxNumCachedHint = 8;
static const scudo::uptr MaxBytesCachedLog = 12;
static const scudo::uptr SizeDelta = 0;
};
-static const scudo::uptr DeathRegionSizeLog = 20U;
+static const scudo::uptr DeathRegionSizeLog = 21U;
struct DeathConfig {
static const bool MaySupportMemoryTagging = false;
static const scudo::uptr PrimaryCompactPtrScale = 0;
static const bool PrimaryEnableRandomOffset = true;
static const scudo::uptr PrimaryMapSizeIncrement = 1UL << 18;
+ static const scudo::uptr PrimaryGroupSizeLog = 18;
typedef scudo::MapAllocatorNoCache SecondaryCache;
template <class A> using TSDRegistryT = scudo::TSDRegistrySharedT<A, 1U, 1U>;
SCUDO_TYPED_TEST(ScudoCombinedTest, DisableMemInit) {
auto *Allocator = this->Allocator.get();
- std::vector<void *> Ptrs(65536, nullptr);
+ std::vector<void *> Ptrs(65536);
Allocator->setOption(scudo::Option::ThreadDisableMemInit, 1);
Allocator->setOption(scudo::Option::ThreadDisableMemInit, 0);
}
+
+SCUDO_TYPED_TEST(ScudoCombinedTest, ReallocateInPlaceStress) {
+ auto *Allocator = this->Allocator.get();
+
+ // Regression test: make realloc-in-place happen at the very right end of a
+ // mapped region.
+ constexpr int nPtrs = 10000;
+ for (int i = 1; i < 32; ++i) {
+ scudo::uptr Size = 16 * i - 1;
+ std::vector<void *> Ptrs;
+ for (int i = 0; i < nPtrs; ++i) {
+ void *P = Allocator->allocate(Size, Origin);
+ P = Allocator->reallocate(P, Size + 1);
+ Ptrs.push_back(P);
+ }
+
+ for (int i = 0; i < nPtrs; ++i)
+ Allocator->deallocate(Ptrs[i], Origin);
+ }
+}
+
+SCUDO_TYPED_TEST(ScudoCombinedTest, RingBufferSize) {
+ auto *Allocator = this->Allocator.get();
+ auto Size = Allocator->getRingBufferSize();
+ ASSERT_GT(Size, 0u);
+ EXPECT_EQ(Allocator->getRingBufferAddress()[Size - 1], '\0');
+}
+
+SCUDO_TYPED_TEST(ScudoCombinedTest, RingBufferAddress) {
+ auto *Allocator = this->Allocator.get();
+ auto *Addr = Allocator->getRingBufferAddress();
+ EXPECT_NE(Addr, nullptr);
+ EXPECT_EQ(Addr, Allocator->getRingBufferAddress());
+}
+
+#if SCUDO_CAN_USE_PRIMARY64
+#if SCUDO_TRUSTY
+
+// TrustyConfig is designed for a domain-specific allocator. Add a basic test
+// which covers only simple operations and ensure the configuration is able to
+// compile.
+TEST(ScudoCombinedTest, BasicTrustyConfig) {
+ using AllocatorT = scudo::Allocator<scudo::TrustyConfig>;
+ auto Allocator = std::unique_ptr<AllocatorT>(new AllocatorT());
+
+ for (scudo::uptr ClassId = 1U;
+ ClassId <= scudo::TrustyConfig::SizeClassMap::LargestClassId;
+ ClassId++) {
+ const scudo::uptr Size =
+ scudo::TrustyConfig::SizeClassMap::getSizeByClassId(ClassId);
+ void *p = Allocator->allocate(Size - scudo::Chunk::getHeaderSize(), Origin);
+ ASSERT_NE(p, nullptr);
+ free(p);
+ }
+
+ bool UnlockRequired;
+ auto *TSD = Allocator->getTSDRegistry()->getTSDAndLock(&UnlockRequired);
+ TSD->Cache.drain();
+
+ Allocator->releaseToOS();
+}
+
+#endif
+#endif
+
+#if SCUDO_LINUX
+
+SCUDO_TYPED_TEST(ScudoCombinedTest, SoftRssLimit) {
+ auto *Allocator = this->Allocator.get();
+ Allocator->setRssLimitsTestOnly(1, 0, true);
+
+ size_t Megabyte = 1024 * 1024;
+ size_t ChunkSize = 16;
+ size_t Error = 256;
+
+ std::vector<void *> Ptrs;
+ for (size_t index = 0; index < Megabyte + Error; index += ChunkSize) {
+ void *Ptr = Allocator->allocate(ChunkSize, Origin);
+ Ptrs.push_back(Ptr);
+ }
+
+ EXPECT_EQ(nullptr, Allocator->allocate(ChunkSize, Origin));
+
+ for (void *Ptr : Ptrs)
+ Allocator->deallocate(Ptr, Origin);
+}
+
+SCUDO_TYPED_TEST(ScudoCombinedTest, HardRssLimit) {
+ auto *Allocator = this->Allocator.get();
+ Allocator->setRssLimitsTestOnly(0, 1, false);
+
+ size_t Megabyte = 1024 * 1024;
+
+ EXPECT_DEATH(
+ {
+ disableDebuggerdMaybe();
+ Allocator->allocate(Megabyte, Origin);
+ },
+ "");
+}
+
+#endif
unmap(P, Size, 0, &Data);
}
+#if SCUDO_LINUX && !defined(__powerpc__)
+// This test fails intermediately on PPC, which is why this test is disabled
+// for now on this platform.
+TEST(ScudoCommonTest, GetRssFromBuffer) {
+ constexpr int64_t AllocSize = 10000000;
+ constexpr int64_t Error = 3000000;
+ constexpr size_t Runs = 10;
+
+ int64_t Rss = scudo::GetRSS();
+ EXPECT_GT(Rss, 0);
+
+ std::vector<std::unique_ptr<char[]>> Allocs(Runs);
+ for (auto &Alloc : Allocs) {
+ Alloc.reset(new char[AllocSize]());
+ int64_t Prev = Rss;
+ Rss = scudo::GetRSS();
+ EXPECT_LE(std::abs(Rss - AllocSize - Prev), Error);
+ }
+}
+#endif // SCUDO_LINUX
+
} // namespace scudo
setList(&L1, X);
checkList(&L1, X);
+ setList(&L1, X, Y);
+ L1.insert(X, Z);
+ checkList(&L1, X, Z, Y);
+
setList(&L1, X, Y, Z);
setList(&L2, A, B, C);
L1.append_back(&L2);
TEST(ScudoMapTest, PageSize) {
EXPECT_EQ(scudo::getPageSizeCached(),
- static_cast<scudo::uptr>(getpagesize()));
+ static_cast<scudo::uptr>(sysconf(_SC_PAGESIZE)));
}
TEST(ScudoMapDeathTest, MapNoAccessUnmap) {
EXPECT_DEATH(addFixedTag(nullptr, 0), "not supported");
}
-class MemtagTest : public ::testing::Test {
+class MemtagTest : public Test {
protected:
void SetUp() override {
if (!archSupportsMemoryTagging() || !systemDetectsMemoryTagFaultsTestOnly())
void backoff() {
volatile T LocalData[Size] = {};
for (scudo::u32 I = 0; I < Size; I++) {
- LocalData[I]++;
+ LocalData[I] = LocalData[I] + 1;
EXPECT_EQ(LocalData[I], 1U);
}
}
#include "primary64.h"
#include "size_class_map.h"
+#include <algorithm>
+#include <chrono>
#include <condition_variable>
#include <mutex>
+#include <random>
#include <stdlib.h>
#include <thread>
#include <vector>
struct TestConfig1 {
static const scudo::uptr PrimaryRegionSizeLog = 18U;
+ static const scudo::uptr PrimaryGroupSizeLog = 18U;
static const scudo::s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN;
static const scudo::s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX;
static const bool MaySupportMemoryTagging = false;
#else
static const scudo::uptr PrimaryRegionSizeLog = 24U;
#endif
+ static const scudo::uptr PrimaryGroupSizeLog = 20U;
static const scudo::s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN;
static const scudo::s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX;
static const bool MaySupportMemoryTagging = false;
#else
static const scudo::uptr PrimaryRegionSizeLog = 24U;
#endif
+ static const scudo::uptr PrimaryGroupSizeLog = 20U;
static const scudo::s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN;
static const scudo::s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX;
static const bool MaySupportMemoryTagging = true;
static const scudo::uptr PrimaryMapSizeIncrement = 1UL << 18;
};
+struct TestConfig4 {
+#if defined(__mips__)
+ // Unable to allocate greater size on QEMU-user.
+ static const scudo::uptr PrimaryRegionSizeLog = 23U;
+#else
+ static const scudo::uptr PrimaryRegionSizeLog = 24U;
+#endif
+ static const scudo::s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN;
+ static const scudo::s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX;
+ static const bool MaySupportMemoryTagging = true;
+ static const scudo::uptr PrimaryCompactPtrScale = 3U;
+ static const scudo::uptr PrimaryGroupSizeLog = 20U;
+ typedef scudo::u32 PrimaryCompactPtrT;
+ static const bool PrimaryEnableRandomOffset = true;
+ static const scudo::uptr PrimaryMapSizeIncrement = 1UL << 18;
+};
+
template <typename BaseConfig, typename SizeClassMapT>
struct Config : public BaseConfig {
using SizeClassMap = SizeClassMapT;
#define SCUDO_TYPED_TEST_ALL_TYPES(FIXTURE, NAME) \
SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, TestConfig1) \
SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, TestConfig2) \
- SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, TestConfig3)
+ SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, TestConfig3) \
+ SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, TestConfig4)
#endif
#define SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, TYPE) \
using FIXTURE##NAME##_##TYPE = FIXTURE##NAME<TYPE>; \
- TEST_F(FIXTURE##NAME##_##TYPE, NAME) { Run(); }
+ TEST_F(FIXTURE##NAME##_##TYPE, NAME) { FIXTURE##NAME<TYPE>::Run(); }
#define SCUDO_TYPED_TEST(FIXTURE, NAME) \
template <class TypeParam> \
struct SmallRegionsConfig {
using SizeClassMap = scudo::DefaultSizeClassMap;
- static const scudo::uptr PrimaryRegionSizeLog = 20U;
+ static const scudo::uptr PrimaryRegionSizeLog = 21U;
static const scudo::s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN;
static const scudo::s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX;
static const bool MaySupportMemoryTagging = false;
static const scudo::uptr PrimaryCompactPtrScale = 0;
static const bool PrimaryEnableRandomOffset = true;
static const scudo::uptr PrimaryMapSizeIncrement = 1UL << 18;
+ static const scudo::uptr PrimaryGroupSizeLog = 20U;
};
// The 64-bit SizeClassAllocator can be easily OOM'd with small region sizes.
std::vector<TransferBatch *> Batches;
const scudo::uptr ClassId = Primary::SizeClassMap::LargestClassId;
const scudo::uptr Size = Primary::getSizeByClassId(ClassId);
+ typename Primary::CacheT::CompactPtrT Blocks[TransferBatch::MaxNumCached];
+
for (scudo::uptr I = 0; I < 10000U; I++) {
TransferBatch *B = Allocator.popBatch(&Cache, ClassId);
if (!B) {
AllocationFailed = true;
break;
}
- for (scudo::u32 J = 0; J < B->getCount(); J++)
+ for (scudo::u16 J = 0; J < B->getCount(); J++)
memset(Allocator.decompactPtr(ClassId, B->get(J)), 'B', Size);
Batches.push_back(B);
}
while (!Batches.empty()) {
- Allocator.pushBatch(ClassId, Batches.back());
+ TransferBatch *B = Batches.back();
Batches.pop_back();
+ B->copyToArray(Blocks);
+ Allocator.pushBlocks(&Cache, ClassId, Blocks, B->getCount());
+ Cache.deallocate(Primary::SizeClassMap::BatchClassId, B);
}
Cache.destroy(nullptr);
Allocator.releaseToOS();
V.push_back(std::make_pair(ClassId, P));
}
scudo::uptr Found = 0;
- auto Lambda = [V, &Found](scudo::uptr Block) {
+ auto Lambda = [&V, &Found](scudo::uptr Block) {
for (const auto &Pair : V) {
if (Pair.second == reinterpret_cast<void *>(Block))
Found++;
Cache.destroy(nullptr);
EXPECT_GT(Allocator->releaseToOS(), 0U);
}
+
+SCUDO_TYPED_TEST(ScudoPrimaryTest, MemoryGroup) {
+ using Primary = TestAllocator<TypeParam, scudo::DefaultSizeClassMap>;
+ std::unique_ptr<Primary> Allocator(new Primary);
+ Allocator->init(/*ReleaseToOsInterval=*/-1);
+ typename Primary::CacheT Cache;
+ Cache.init(nullptr, Allocator.get());
+ const scudo::uptr Size = 32U;
+ const scudo::uptr ClassId = Primary::SizeClassMap::getClassIdBySize(Size);
+
+ // We will allocate 4 times the group size memory and release all of them. We
+ // expect the free blocks will be classified with groups. Then we will
+ // allocate the same amount of memory as group size and expect the blocks will
+ // have the max address difference smaller or equal to 2 times the group size.
+ // Note that it isn't necessary to be in the range of single group size
+ // because the way we get the group id is doing compact pointer shifting.
+ // According to configuration, the compact pointer may not align to group
+ // size. As a result, the blocks can cross two groups at most.
+ const scudo::uptr GroupSizeMem = (1ULL << Primary::GroupSizeLog);
+ const scudo::uptr PeakAllocationMem = 4 * GroupSizeMem;
+ const scudo::uptr PeakNumberOfAllocations = PeakAllocationMem / Size;
+ const scudo::uptr FinalNumberOfAllocations = GroupSizeMem / Size;
+ std::vector<scudo::uptr> Blocks;
+ std::mt19937 R;
+
+ for (scudo::uptr I = 0; I < PeakNumberOfAllocations; ++I)
+ Blocks.push_back(reinterpret_cast<scudo::uptr>(Cache.allocate(ClassId)));
+
+ std::shuffle(Blocks.begin(), Blocks.end(), R);
+
+ // Release all the allocated blocks, including those held by local cache.
+ while (!Blocks.empty()) {
+ Cache.deallocate(ClassId, reinterpret_cast<void *>(Blocks.back()));
+ Blocks.pop_back();
+ }
+ Cache.drain();
+
+ for (scudo::uptr I = 0; I < FinalNumberOfAllocations; ++I)
+ Blocks.push_back(reinterpret_cast<scudo::uptr>(Cache.allocate(ClassId)));
+
+ EXPECT_LE(*std::max_element(Blocks.begin(), Blocks.end()) -
+ *std::min_element(Blocks.begin(), Blocks.end()),
+ GroupSizeMem * 2);
+}
#include <random>
#include <set>
-TEST(ScudoReleaseTest, PackedCounterArray) {
+TEST(ScudoReleaseTest, RegionPageMap) {
for (scudo::uptr I = 0; I < SCUDO_WORDSIZE; I++) {
// Various valid counter's max values packed into one word.
- scudo::PackedCounterArray Counters2N(1U, 1U, 1UL << I);
- EXPECT_EQ(sizeof(scudo::uptr), Counters2N.getBufferSize());
+ scudo::RegionPageMap PageMap2N(1U, 1U, 1UL << I);
+ EXPECT_EQ(sizeof(scudo::uptr), PageMap2N.getBufferSize());
// Check the "all bit set" values too.
- scudo::PackedCounterArray Counters2N1_1(1U, 1U, ~0UL >> I);
- EXPECT_EQ(sizeof(scudo::uptr), Counters2N1_1.getBufferSize());
+ scudo::RegionPageMap PageMap2N1_1(1U, 1U, ~0UL >> I);
+ EXPECT_EQ(sizeof(scudo::uptr), PageMap2N1_1.getBufferSize());
// Verify the packing ratio, the counter is Expected to be packed into the
// closest power of 2 bits.
- scudo::PackedCounterArray Counters(1U, SCUDO_WORDSIZE, 1UL << I);
+ scudo::RegionPageMap PageMap(1U, SCUDO_WORDSIZE, 1UL << I);
EXPECT_EQ(sizeof(scudo::uptr) * scudo::roundUpToPowerOfTwo(I + 1),
- Counters.getBufferSize());
+ PageMap.getBufferSize());
}
// Go through 1, 2, 4, 8, .. {32,64} bits per counter.
// Make sure counters request one memory page for the buffer.
const scudo::uptr NumCounters =
(scudo::getPageSizeCached() / 8) * (SCUDO_WORDSIZE >> I);
- scudo::PackedCounterArray Counters(1U, NumCounters,
+ scudo::RegionPageMap PageMap(1U, NumCounters,
1UL << ((1UL << I) - 1));
- Counters.inc(0U, 0U);
+ PageMap.inc(0U, 0U);
for (scudo::uptr C = 1; C < NumCounters - 1; C++) {
- EXPECT_EQ(0UL, Counters.get(0U, C));
- Counters.inc(0U, C);
- EXPECT_EQ(1UL, Counters.get(0U, C - 1));
+ EXPECT_EQ(0UL, PageMap.get(0U, C));
+ PageMap.inc(0U, C);
+ EXPECT_EQ(1UL, PageMap.get(0U, C - 1));
}
- EXPECT_EQ(0UL, Counters.get(0U, NumCounters - 1));
- Counters.inc(0U, NumCounters - 1);
+ EXPECT_EQ(0UL, PageMap.get(0U, NumCounters - 1));
+ PageMap.inc(0U, NumCounters - 1);
if (I > 0) {
- Counters.incRange(0u, 0U, NumCounters - 1);
+ PageMap.incRange(0u, 0U, NumCounters - 1);
for (scudo::uptr C = 0; C < NumCounters; C++)
- EXPECT_EQ(2UL, Counters.get(0U, C));
+ EXPECT_EQ(2UL, PageMap.get(0U, C));
}
}
}
for (auto TestCase : TestCases) {
StringRangeRecorder Recorder;
- RangeTracker Tracker(&Recorder);
+ RangeTracker Tracker(Recorder);
for (scudo::uptr I = 0; TestCase[I] != 0; I++)
Tracker.processNextPage(TestCase[I] == 'x');
Tracker.finish();
// Simplified version of a TransferBatch.
template <class SizeClassMap> struct FreeBatch {
- static const scudo::u32 MaxCount = SizeClassMap::MaxNumCachedHint;
+ static const scudo::u16 MaxCount = SizeClassMap::MaxNumCachedHint;
void clear() { Count = 0; }
void add(scudo::uptr P) {
DCHECK_LT(Count, MaxCount);
Batch[Count++] = P;
}
- scudo::u32 getCount() const { return Count; }
- scudo::uptr get(scudo::u32 I) const {
+ scudo::u16 getCount() const { return Count; }
+ scudo::uptr get(scudo::u16 I) const {
DCHECK_LE(I, Count);
return Batch[I];
}
FreeBatch *Next;
private:
- scudo::u32 Count;
scudo::uptr Batch[MaxCount];
+ scudo::u16 Count;
};
template <class SizeClassMap> void testReleaseFreeMemoryToOS() {
typedef FreeBatch<SizeClassMap> Batch;
const scudo::uptr PagesCount = 1024;
const scudo::uptr PageSize = scudo::getPageSizeCached();
+ const scudo::uptr PageSizeLog = scudo::getLog2(PageSize);
std::mt19937 R;
scudo::u32 RandState = 42;
auto SkipRegion = [](UNUSED scudo::uptr RegionIndex) { return false; };
auto DecompactPtr = [](scudo::uptr P) { return P; };
ReleasedPagesRecorder Recorder;
- releaseFreeMemoryToOS(FreeList, MaxBlocks * BlockSize, 1U, BlockSize,
- &Recorder, DecompactPtr, SkipRegion);
+ scudo::PageReleaseContext Context(BlockSize,
+ /*RegionSize=*/MaxBlocks * BlockSize,
+ /*NumberOfRegions=*/1U);
+ ASSERT_FALSE(Context.hasBlockMarked());
+ Context.markFreeBlocks(FreeList, DecompactPtr, Recorder.getBase());
+ ASSERT_TRUE(Context.hasBlockMarked());
+ releaseFreeMemoryToOS(Context, Recorder, SkipRegion);
+ scudo::RegionPageMap &PageMap = Context.PageMap;
// Verify that there are no released pages touched by used chunks and all
// ranges of free chunks big enough to contain the entire memory pages had
const bool PageReleased = Recorder.ReportedPages.find(J * PageSize) !=
Recorder.ReportedPages.end();
EXPECT_EQ(false, PageReleased);
+ EXPECT_EQ(false,
+ PageMap.isAllCounted(0, (J * PageSize) >> PageSizeLog));
}
if (InFreeRange) {
const bool PageReleased =
Recorder.ReportedPages.find(P) != Recorder.ReportedPages.end();
EXPECT_EQ(true, PageReleased);
+ EXPECT_EQ(true, PageMap.isAllCounted(0, P >> PageSizeLog));
VerifiedReleasedPages++;
P += PageSize;
}
const bool PageReleased =
Recorder.ReportedPages.find(P) != Recorder.ReportedPages.end();
EXPECT_EQ(true, PageReleased);
+ EXPECT_EQ(true, PageMap.isAllCounted(0, P >> PageSizeLog));
VerifiedReleasedPages++;
P += PageSize;
}
#include "allocator_config.h"
#include "secondary.h"
+#include <algorithm>
#include <condition_variable>
#include <memory>
#include <mutex>
const scudo::uptr PageSize = scudo::getPageSizeCached();
for (scudo::uptr I = 0; I < 32U; I++)
V.push_back(Allocator->allocate(Options, (std::rand() % 16) * PageSize));
- auto Lambda = [V](scudo::uptr Block) {
+ auto Lambda = [&V](scudo::uptr Block) {
EXPECT_NE(std::find(V.begin(), V.end(), reinterpret_cast<void *>(Block)),
V.end());
};
static const scudo::uptr MinSizeLog = 5;
static const scudo::uptr MidSizeLog = 5;
static const scudo::uptr MaxSizeLog = 5;
- static const scudo::u32 MaxNumCachedHint = 0;
+ static const scudo::u16 MaxNumCachedHint = 0;
static const scudo::uptr MaxBytesCachedLog = 0;
static const scudo::uptr SizeDelta = 0;
};
static const scudo::uptr MinSizeLog = 4;
static const scudo::uptr MidSizeLog = 8;
static const scudo::uptr MaxSizeLog = 63;
- static const scudo::u32 MaxNumCachedHint = 128;
+ static const scudo::u16 MaxNumCachedHint = 128;
static const scudo::uptr MaxBytesCachedLog = 16;
static const scudo::uptr SizeDelta = 0;
};
}
TEST(ScudoStringsTest, ClearLarge) {
+ constexpr char appendString[] = "123";
scudo::ScopedString Str;
+ Str.reserve(sizeof(appendString) * 10000);
for (int i = 0; i < 10000; ++i)
- Str.append("123");
+ Str.append(appendString);
Str.clear();
EXPECT_EQ(0ul, Str.length());
EXPECT_EQ('\0', *Str.data());
// of it with variations of append. The expectation is for nothing to crash.
const scudo::uptr PageSize = scudo::getPageSizeCached();
scudo::ScopedString Str;
+ Str.reserve(2 * PageSize);
Str.clear();
fillString(Str, 2 * PageSize);
Str.clear();
public:
using ThisT = MockAllocator<Config>;
using TSDRegistryT = typename Config::template TSDRegistryT<ThisT>;
- using CacheT = struct MockCache { volatile scudo::uptr Canary; };
+ using CacheT = struct MockCache {
+ volatile scudo::uptr Canary;
+ };
using QuarantineCacheT = struct MockQuarantine {};
void init() {
#include <stdlib.h>
#include <unistd.h>
+#ifndef __GLIBC_PREREQ
+#define __GLIBC_PREREQ(x, y) 0
+#endif
+
extern "C" {
void malloc_enable(void);
void malloc_disable(void);
#if !SCUDO_FUCHSIA
TEST(ScudoWrappersCTest, MallInfo) {
+ // mallinfo is deprecated.
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wdeprecated-declarations"
const size_t BypassQuarantineSize = 1024U;
-
struct mallinfo MI = mallinfo();
size_t Allocated = MI.uordblks;
void *P = malloc(BypassQuarantineSize);
free(P);
MI = mallinfo();
EXPECT_GE(static_cast<size_t>(MI.fordblks), Free + BypassQuarantineSize);
+#pragma clang diagnostic pop
+}
+#endif
+
+#if __GLIBC_PREREQ(2, 33)
+TEST(ScudoWrappersCTest, MallInfo2) {
+ const size_t BypassQuarantineSize = 1024U;
+ struct mallinfo2 MI = mallinfo2();
+ size_t Allocated = MI.uordblks;
+ void *P = malloc(BypassQuarantineSize);
+ EXPECT_NE(P, nullptr);
+ MI = mallinfo2();
+ EXPECT_GE(MI.uordblks, Allocated + BypassQuarantineSize);
+ EXPECT_GT(MI.hblkhd, 0U);
+ size_t Free = MI.fordblks;
+ free(P);
+ MI = mallinfo2();
+ EXPECT_GE(MI.fordblks, Free + BypassQuarantineSize);
}
#endif
#include <atomic>
#include <condition_variable>
+#include <fstream>
#include <memory>
#include <mutex>
#include <thread>
}
#if !SCUDO_FUCHSIA
-// TODO(kostyak): for me, this test fails in a specific configuration when ran
-// by itself with some Scudo or GWP-ASan violation. Other people
-// can't seem to reproduce the failure. Consider skipping this in
-// the event it fails on the upstream bots.
TEST(ScudoWrappersCppTest, AllocAfterFork) {
+ // This test can fail flakily when ran as a part of large number of
+ // other tests if the maxmimum number of mappings allowed is low.
+ // We tried to reduce the number of iterations of the loops with
+ // moderate success, so we will now skip this test under those
+ // circumstances.
+ if (SCUDO_LINUX) {
+ long MaxMapCount = 0;
+ // If the file can't be accessed, we proceed with the test.
+ std::ifstream Stream("/proc/sys/vm/max_map_count");
+ if (Stream.good()) {
+ Stream >> MaxMapCount;
+ if (MaxMapCount < 200000)
+ return;
+ }
+ }
+
std::atomic_bool Stop;
// Create threads that simply allocate and free different sizes.
for (size_t N = 0; N < 5; N++) {
std::thread *T = new std::thread([&Stop] {
while (!Stop) {
- for (size_t SizeLog = 3; SizeLog <= 21; SizeLog++) {
+ for (size_t SizeLog = 3; SizeLog <= 20; SizeLog++) {
char *P = new char[1UL << SizeLog];
EXPECT_NE(P, nullptr);
// Make sure this value is not optimized away.
}
// Create a thread to fork and allocate.
- for (size_t N = 0; N < 100; N++) {
+ for (size_t N = 0; N < 50; N++) {
pid_t Pid;
if ((Pid = fork()) == 0) {
- for (size_t SizeLog = 3; SizeLog <= 21; SizeLog++) {
+ for (size_t SizeLog = 3; SizeLog <= 20; SizeLog++) {
char *P = new char[1UL << SizeLog];
EXPECT_NE(P, nullptr);
// Make sure this value is not optimized away.
static const uptr MinSizeLog = %zu;
static const uptr MidSizeLog = %zu;
static const uptr MaxSizeLog = %zu;
- static const u32 MaxNumCachedHint = 14;
+ static const u16 MaxNumCachedHint = 14;
static const uptr MaxBytesCachedLog = 14;
static constexpr u32 Classes[] = {)",
struct ThreadState {
bool DisableMemInit : 1;
- enum {
+ enum : unsigned {
NotInitialized = 0,
Initialized,
TornDown,
Mutex.unlock();
}
- bool setOption(Option O, UNUSED sptr Value) {
+ bool setOption(Option O, sptr Value) {
if (O == Option::ThreadDisableMemInit)
State.DisableMemInit = Value;
if (O == Option::MaxTSDsCount)
// small vectors. The current implementation supports only POD types.
template <typename T> class VectorNoCtor {
public:
- void init(uptr InitialCapacity = 0) {
- Data = reinterpret_cast<T *>(&LocalData[0]);
+ constexpr void init(uptr InitialCapacity = 0) {
+ Data = &LocalData[0];
CapacityBytes = sizeof(LocalData);
- reserve(InitialCapacity);
+ if (InitialCapacity > capacity())
+ reserve(InitialCapacity);
}
void destroy() {
- if (Data != reinterpret_cast<T *>(&LocalData[0]))
- unmap(Data, CapacityBytes);
+ if (Data != &LocalData[0])
+ unmap(Data, CapacityBytes, 0, &MapData);
}
T &operator[](uptr I) {
DCHECK_LT(I, Size);
uptr size() const { return Size; }
const T *data() const { return Data; }
T *data() { return Data; }
- uptr capacity() const { return CapacityBytes / sizeof(T); }
+ constexpr uptr capacity() const { return CapacityBytes / sizeof(T); }
void reserve(uptr NewSize) {
// Never downsize internal buffer.
if (NewSize > capacity())
DCHECK_GT(NewCapacity, 0);
DCHECK_LE(Size, NewCapacity);
NewCapacity = roundUpTo(NewCapacity * sizeof(T), getPageSizeCached());
- T *NewData =
- reinterpret_cast<T *>(map(nullptr, NewCapacity, "scudo:vector"));
+ T *NewData = reinterpret_cast<T *>(
+ map(nullptr, NewCapacity, "scudo:vector", 0, &MapData));
memcpy(NewData, Data, Size * sizeof(T));
destroy();
Data = NewData;
}
T *Data = nullptr;
- u8 LocalData[256] = {};
+ T LocalData[256 / sizeof(T)] = {};
uptr CapacityBytes = 0;
uptr Size = 0;
+ [[no_unique_address]] MapPlatformData MapData = {};
};
template <typename T> class Vector : public VectorNoCtor<T> {
public:
- Vector() { VectorNoCtor<T>::init(); }
+ constexpr Vector() { VectorNoCtor<T>::init(); }
explicit Vector(uptr Count) {
VectorNoCtor<T>::init(Count);
this->resize(Count);
#define SCUDO_PREFIX(name) name
#define SCUDO_ALLOCATOR Allocator
-extern "C" void SCUDO_PREFIX(malloc_postinit)();
-
// Export the static allocator so that the C++ wrappers can access it.
// Technically we could have a completely separated heap for C & C++ but in
// reality the amount of cross pollination between the two is staggering.
__scudo_mallinfo_data_t keepcost;
};
+struct __scudo_mallinfo2 {
+ size_t arena;
+ size_t ordblks;
+ size_t smblks;
+ size_t hblks;
+ size_t hblkhd;
+ size_t usmblks;
+ size_t fsmblks;
+ size_t uordblks;
+ size_t fordblks;
+ size_t keepcost;
+};
+
// Android sometimes includes malloc.h no matter what, which yields to
// conflicting return types for mallinfo() if we use our own structure. So if
// struct mallinfo is declared (#define courtesy of malloc.h), use it directly.
#define SCUDO_MALLINFO __scudo_mallinfo
#endif
+#if !SCUDO_ANDROID || !_BIONIC
+extern "C" void malloc_postinit();
+extern HIDDEN scudo::Allocator<scudo::Config, malloc_postinit> Allocator;
+#endif
+
#endif // SCUDO_WRAPPERS_C_H_
return Info;
}
+INTERFACE WEAK struct __scudo_mallinfo2 SCUDO_PREFIX(mallinfo2)(void) {
+ struct __scudo_mallinfo2 Info = {};
+ scudo::StatCounters Stats;
+ SCUDO_ALLOCATOR.getStats(Stats);
+ // Space allocated in mmapped regions (bytes)
+ Info.hblkhd = Stats[scudo::StatMapped];
+ // Maximum total allocated space (bytes)
+ Info.usmblks = Info.hblkhd;
+ // Space in freed fastbin blocks (bytes)
+ Info.fsmblks = Stats[scudo::StatFree];
+ // Total allocated space (bytes)
+ Info.uordblks = Stats[scudo::StatAllocated];
+ // Total free space (bytes)
+ Info.fordblks = Info.fsmblks;
+ return Info;
+}
+
INTERFACE WEAK void *SCUDO_PREFIX(malloc)(size_t size) {
return scudo::setErrnoOnNull(SCUDO_ALLOCATOR.allocate(
size, scudo::Chunk::Origin::Malloc, SCUDO_MALLOC_ALIGNMENT));
fputs("<malloc version=\"scudo-1\">\n", stream);
for (scudo::uptr i = 0; i != max_size; ++i)
if (sizes[i])
- fprintf(stream, "<alloc size=\"%lu\" count=\"%lu\"/>\n", i, sizes[i]);
+ fprintf(stream, "<alloc size=\"%zu\" count=\"%zu\"/>\n", i, sizes[i]);
fputs("</malloc>\n", stream);
SCUDO_PREFIX(free)(sizes);
return 0;
// builtin supported by recent clang & GCC if it exists, otherwise fallback to a
// costly division.
inline bool checkForCallocOverflow(uptr Size, uptr N, uptr *Product) {
-#if __has_builtin(__builtin_umull_overflow)
- return __builtin_umull_overflow(Size, N, Product);
+#if __has_builtin(__builtin_umull_overflow) && (SCUDO_WORDSIZE == 64U)
+ return __builtin_umull_overflow(Size, N,
+ reinterpret_cast<unsigned long *>(Product));
+#elif __has_builtin(__builtin_umul_overflow) && (SCUDO_WORDSIZE == 32U)
+ // On, e.g. armv7, uptr/uintptr_t may be defined as unsigned long
+ return __builtin_umul_overflow(Size, N,
+ reinterpret_cast<unsigned int *>(Product));
#else
*Product = Size * N;
if (!Size)
#if !SCUDO_ANDROID || !_BIONIC
#include "allocator_config.h"
+#include "wrappers_c.h"
#include <stdint.h>
-extern "C" void malloc_postinit();
-extern HIDDEN scudo::Allocator<scudo::Config, malloc_postinit> Allocator;
-
namespace std {
struct nothrow_t {};
enum class align_val_t : size_t {};
static_cast<scudo::uptr>(align));
}
-INTERFACE WEAK void operator delete(void *ptr)NOEXCEPT {
+INTERFACE WEAK void operator delete(void *ptr) NOEXCEPT {
Allocator.deallocate(ptr, scudo::Chunk::Origin::New);
}
INTERFACE WEAK void operator delete[](void *ptr) NOEXCEPT {
Allocator.deallocate(ptr, scudo::Chunk::Origin::NewArray);
}
-INTERFACE WEAK void operator delete(void *ptr, std::nothrow_t const &)NOEXCEPT {
+INTERFACE WEAK void operator delete(void *ptr,
+ std::nothrow_t const &) NOEXCEPT {
Allocator.deallocate(ptr, scudo::Chunk::Origin::New);
}
INTERFACE WEAK void operator delete[](void *ptr,
std::nothrow_t const &) NOEXCEPT {
Allocator.deallocate(ptr, scudo::Chunk::Origin::NewArray);
}
-INTERFACE WEAK void operator delete(void *ptr, size_t size)NOEXCEPT {
+INTERFACE WEAK void operator delete(void *ptr, size_t size) NOEXCEPT {
Allocator.deallocate(ptr, scudo::Chunk::Origin::New, size);
}
INTERFACE WEAK void operator delete[](void *ptr, size_t size) NOEXCEPT {
Allocator.deallocate(ptr, scudo::Chunk::Origin::NewArray, size);
}
-INTERFACE WEAK void operator delete(void *ptr, std::align_val_t align)NOEXCEPT {
+INTERFACE WEAK void operator delete(void *ptr,
+ std::align_val_t align) NOEXCEPT {
Allocator.deallocate(ptr, scudo::Chunk::Origin::New, 0,
static_cast<scudo::uptr>(align));
}
static_cast<scudo::uptr>(align));
}
INTERFACE WEAK void operator delete(void *ptr, std::align_val_t align,
- std::nothrow_t const &)NOEXCEPT {
+ std::nothrow_t const &) NOEXCEPT {
Allocator.deallocate(ptr, scudo::Chunk::Origin::New, 0,
static_cast<scudo::uptr>(align));
}
static_cast<scudo::uptr>(align));
}
INTERFACE WEAK void operator delete(void *ptr, size_t size,
- std::align_val_t align)NOEXCEPT {
+ std::align_val_t align) NOEXCEPT {
Allocator.deallocate(ptr, scudo::Chunk::Origin::New, size,
static_cast<scudo::uptr>(align));
}
add_compiler_rt_runtime(clang_rt.stats
${STATS_LIB_FLAVOR}
ARCHS ${SANITIZER_COMMON_SUPPORTED_ARCH}
- OS ${SANITIZER_COMMON_SUPPORTED_OS}
+ OS ${STATS_SUPPORTED_OS}
SOURCES stats.cpp
ADDITIONAL_HEADERS ${STATS_HEADERS}
OBJECT_LIBS RTSanitizerCommon
add_compiler_rt_runtime(clang_rt.stats_client
STATIC
ARCHS ${SANITIZER_COMMON_SUPPORTED_ARCH}
- OS ${SANITIZER_COMMON_SUPPORTED_OS}
+ OS ${STATS_SUPPORTED_OS}
SOURCES stats_client.cpp
ADDITIONAL_HEADERS ${STATS_HEADERS}
CFLAGS ${SANITIZER_COMMON_CFLAGS}
# Build for the ThreadSanitizer runtime support library.
-include_directories(..)
-
set(TSAN_CFLAGS ${SANITIZER_COMMON_CFLAGS})
# SANITIZER_COMMON_CFLAGS contains -fPIC, but it's performance-critical for
# TSan runtime to be built with -fPIE to reduce the number of register spills.
list(APPEND TSAN_CFLAGS -DTSAN_DEBUG_OUTPUT=2)
endif()
-set(TSAN_RTL_CFLAGS ${TSAN_CFLAGS})
-append_list_if(COMPILER_RT_HAS_MSSE3_FLAG -msse3 TSAN_RTL_CFLAGS)
-append_list_if(SANITIZER_LIMIT_FRAME_SIZE -Wframe-larger-than=530
- TSAN_RTL_CFLAGS)
-append_list_if(COMPILER_RT_HAS_WGLOBAL_CONSTRUCTORS_FLAG -Wglobal-constructors
- TSAN_RTL_CFLAGS)
-
-set(TSAN_SOURCES
- rtl/tsan_clock.cpp
- rtl/tsan_debugging.cpp
- rtl/tsan_external.cpp
- rtl/tsan_fd.cpp
- rtl/tsan_flags.cpp
- rtl/tsan_ignoreset.cpp
- rtl/tsan_interceptors_posix.cpp
- rtl/tsan_interface.cpp
- rtl/tsan_interface_ann.cpp
- rtl/tsan_interface_atomic.cpp
- rtl/tsan_interface_java.cpp
- rtl/tsan_malloc_mac.cpp
- rtl/tsan_md5.cpp
- rtl/tsan_mman.cpp
- rtl/tsan_mutexset.cpp
- rtl/tsan_preinit.cpp
- rtl/tsan_report.cpp
- rtl/tsan_rtl.cpp
- rtl/tsan_rtl_mutex.cpp
- rtl/tsan_rtl_proc.cpp
- rtl/tsan_rtl_report.cpp
- rtl/tsan_rtl_thread.cpp
- rtl/tsan_stack_trace.cpp
- rtl/tsan_suppressions.cpp
- rtl/tsan_symbolize.cpp
- rtl/tsan_sync.cpp
- )
-
-set(TSAN_CXX_SOURCES
- rtl/tsan_new_delete.cpp
- )
-
-if(APPLE)
- list(APPEND TSAN_SOURCES
- rtl/tsan_interceptors_mac.cpp
- rtl/tsan_interceptors_mach_vm.cpp
- rtl/tsan_platform_mac.cpp
- rtl/tsan_platform_posix.cpp
- )
-elseif(UNIX)
- # Assume Linux
- list(APPEND TSAN_SOURCES
- rtl/tsan_platform_linux.cpp
- rtl/tsan_platform_posix.cpp
- )
-endif()
-
-if(COMPILER_RT_INTERCEPT_LIBDISPATCH)
- list(APPEND TSAN_SOURCES
- rtl/tsan_interceptors_libdispatch.cpp
- )
- list(APPEND TSAN_RTL_CFLAGS ${COMPILER_RT_LIBDISPATCH_CFLAGS})
-endif()
-
-set(TSAN_HEADERS
- rtl/tsan_clock.h
- rtl/tsan_defs.h
- rtl/tsan_dense_alloc.h
- rtl/tsan_fd.h
- rtl/tsan_flags.h
- rtl/tsan_flags.inc
- rtl/tsan_ignoreset.h
- rtl/tsan_interceptors.h
- rtl/tsan_interface.h
- rtl/tsan_interface_ann.h
- rtl/tsan_interface_inl.h
- rtl/tsan_interface_java.h
- rtl/tsan_mman.h
- rtl/tsan_mutexset.h
- rtl/tsan_platform.h
- rtl/tsan_ppc_regs.h
- rtl/tsan_report.h
- rtl/tsan_rtl.h
- rtl/tsan_stack_trace.h
- rtl/tsan_suppressions.h
- rtl/tsan_symbolize.h
- rtl/tsan_sync.h
- rtl/tsan_trace.h
- rtl/tsan_update_shadow_word_inl.h
- )
-
-set(TSAN_RUNTIME_LIBRARIES)
-add_compiler_rt_component(tsan)
-
-if("${CMAKE_C_FLAGS}" MATCHES "-Wno-(error=)?unused-command-line-argument")
- set(EXTRA_CFLAGS "-Wno-error=unused-command-line-argument ${EXTRA_CFLAGS}")
-endif()
-
-if(APPLE)
- # Ideally we would check the SDK version for the actual platform we are
- # building for here. To make our lifes easier we assume the host SDK setup is
- # sane and use the macOS SDK version as a proxy for aligned SDKs.
- find_darwin_sdk_version(macosx_sdk_version "macosx")
- if ("${macosx_sdk_version}" VERSION_LESS 10.12)
- message(FATAL_ERROR "Building the TSan runtime requires at least macOS SDK 10.12 (or aligned SDK on other platforms)")
- endif()
-
- add_asm_sources(TSAN_ASM_SOURCES
- rtl/tsan_rtl_amd64.S
- rtl/tsan_rtl_aarch64.S
- )
-
- set(TSAN_LINK_LIBS ${SANITIZER_COMMON_LINK_LIBS})
-
- add_weak_symbols("ubsan" WEAK_SYMBOL_LINK_FLAGS)
- add_weak_symbols("sanitizer_common" WEAK_SYMBOL_LINK_FLAGS)
-
- add_compiler_rt_runtime(clang_rt.tsan
- SHARED
- OS ${TSAN_SUPPORTED_OS}
- ARCHS ${TSAN_SUPPORTED_ARCH}
- SOURCES ${TSAN_SOURCES} ${TSAN_CXX_SOURCES} ${TSAN_ASM_SOURCES}
- ADDITIONAL_HEADERS ${TSAN_HEADERS}
- OBJECT_LIBS RTInterception
- RTSanitizerCommon
- RTSanitizerCommonLibc
- RTSanitizerCommonCoverage
- RTSanitizerCommonSymbolizer
- RTUbsan
- CFLAGS ${TSAN_RTL_CFLAGS}
- LINK_FLAGS ${SANITIZER_COMMON_LINK_FLAGS} ${WEAK_SYMBOL_LINK_FLAGS}
- LINK_LIBS ${TSAN_LINK_LIBS} objc
- PARENT_TARGET tsan)
- add_compiler_rt_object_libraries(RTTsan_dynamic
- OS ${TSAN_SUPPORTED_OS}
- ARCHS ${TSAN_SUPPORTED_ARCH}
- SOURCES ${TSAN_SOURCES} ${TSAN_CXX_SOURCES} ${TSAN_ASM_SOURCES}
- ADDITIONAL_HEADERS ${TSAN_HEADERS}
- CFLAGS ${TSAN_RTL_CFLAGS})
-
- # Build and check Go runtime.
- set(BUILDGO_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/go/buildgo.sh)
- add_custom_target(GotsanRuntimeCheck
- COMMAND env "CC=${CMAKE_C_COMPILER} ${OSX_SYSROOT_FLAG}"
- EXTRA_CFLAGS=${EXTRA_CFLAGS}
- IN_TMPDIR=1 SILENT=1 ${BUILDGO_SCRIPT}
- DEPENDS tsan ${BUILDGO_SCRIPT}
- WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/go
- COMMENT "Checking TSan Go runtime..."
- VERBATIM)
- set_target_properties(GotsanRuntimeCheck PROPERTIES FOLDER "Compiler-RT Misc")
-else()
- foreach(arch ${TSAN_SUPPORTED_ARCH})
- if(arch STREQUAL "x86_64")
- add_asm_sources(TSAN_ASM_SOURCES
- rtl/tsan_rtl_amd64.S
- )
- # Sanity check for Go runtime.
- set(BUILDGO_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/go/buildgo.sh)
- add_custom_target(GotsanRuntimeCheck
- COMMAND env "CC=${CMAKE_C_COMPILER} ${CMAKE_C_COMPILER_ARG1}"
- EXTRA_CFLAGS=${EXTRA_CFLAGS}
- IN_TMPDIR=1 SILENT=1 ${BUILDGO_SCRIPT}
- DEPENDS clang_rt.tsan-${arch} ${BUILDGO_SCRIPT}
- WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/go
- COMMENT "Checking TSan Go runtime..."
- VERBATIM)
- elseif(arch STREQUAL "aarch64")
- add_asm_sources(TSAN_ASM_SOURCES
- rtl/tsan_rtl_aarch64.S
- )
- # Sanity check for Go runtime.
- set(BUILDGO_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/go/buildgo.sh)
- add_custom_target(GotsanRuntimeCheck
- COMMAND env "CC=${CMAKE_C_COMPILER} ${CMAKE_C_COMPILER_ARG1}"
- EXTRA_CFLAGS=${EXTRA_CFLAGS}
- IN_TMPDIR=1 SILENT=1 ${BUILDGO_SCRIPT}
- DEPENDS clang_rt.tsan-${arch} ${BUILDGO_SCRIPT}
- WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/go
- COMMENT "Checking TSan Go runtime..."
- VERBATIM)
- elseif(arch MATCHES "powerpc64|powerpc64le")
- add_asm_sources(TSAN_ASM_SOURCES
- rtl/tsan_rtl_ppc64.S
- )
- # Sanity check for Go runtime.
- set(BUILDGO_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/go/buildgo.sh)
- add_custom_target(GotsanRuntimeCheck
- COMMAND env "CC=${CMAKE_C_COMPILER} ${CMAKE_C_COMPILER_ARG1}"
- EXTRA_CFLAGS=${EXTRA_CFLAGS}
- IN_TMPDIR=1 SILENT=1 ${BUILDGO_SCRIPT}
- DEPENDS clang_rt.tsan-${arch} ${BUILDGO_SCRIPT}
- WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/go
- COMMENT "Checking TSan Go runtime..."
- VERBATIM)
- elseif(arch MATCHES "mips64|mips64le")
- add_asm_sources(TSAN_ASM_SOURCES
- rtl/tsan_rtl_mips64.S
- )
- elseif(arch MATCHES "s390x")
- add_asm_sources(TSAN_ASM_SOURCES
- rtl/tsan_rtl_s390x.S
- )
- # Sanity check for Go runtime.
- set(BUILDGO_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/go/buildgo.sh)
- add_custom_target(GotsanRuntimeCheck
- COMMAND env "CC=${CMAKE_C_COMPILER} ${CMAKE_C_COMPILER_ARG1}"
- EXTRA_CFLAGS=${EXTRA_CFLAGS}
- IN_TMPDIR=1 SILENT=1 ${BUILDGO_SCRIPT}
- DEPENDS clang_rt.tsan-${arch} ${BUILDGO_SCRIPT}
- WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/go
- COMMENT "Checking TSan Go runtime..."
- VERBATIM)
- else()
- set(TSAN_ASM_SOURCES)
- endif()
- add_compiler_rt_runtime(clang_rt.tsan
- STATIC
- ARCHS ${arch}
- SOURCES ${TSAN_SOURCES} ${TSAN_ASM_SOURCES}
- $<TARGET_OBJECTS:RTInterception.${arch}>
- $<TARGET_OBJECTS:RTSanitizerCommon.${arch}>
- $<TARGET_OBJECTS:RTSanitizerCommonLibc.${arch}>
- $<TARGET_OBJECTS:RTSanitizerCommonCoverage.${arch}>
- $<TARGET_OBJECTS:RTSanitizerCommonSymbolizer.${arch}>
- $<TARGET_OBJECTS:RTUbsan.${arch}>
- ADDITIONAL_HEADERS ${TSAN_HEADERS}
- CFLAGS ${TSAN_RTL_CFLAGS}
- PARENT_TARGET tsan)
- add_compiler_rt_runtime(clang_rt.tsan_cxx
- STATIC
- ARCHS ${arch}
- SOURCES ${TSAN_CXX_SOURCES}
- $<TARGET_OBJECTS:RTUbsan_cxx.${arch}>
- ADDITIONAL_HEADERS ${TSAN_HEADERS}
- CFLAGS ${TSAN_RTL_CFLAGS}
- PARENT_TARGET tsan)
- list(APPEND TSAN_RUNTIME_LIBRARIES clang_rt.tsan-${arch}
- clang_rt.tsan_cxx-${arch})
- add_sanitizer_rt_symbols(clang_rt.tsan
- ARCHS ${arch}
- EXTRA rtl/tsan.syms.extra)
- add_sanitizer_rt_symbols(clang_rt.tsan_cxx
- ARCHS ${arch}
- EXTRA rtl/tsan.syms.extra)
- add_dependencies(tsan clang_rt.tsan-${arch}
- clang_rt.tsan_cxx-${arch}
- clang_rt.tsan-${arch}-symbols
- clang_rt.tsan_cxx-${arch}-symbols)
- endforeach()
-endif()
+# Add the actual runtime library.
+add_subdirectory(rtl)
# Build libcxx instrumented with TSan.
if(COMPILER_RT_LIBCXX_PATH AND
-#!/bin/bash
+#!/usr/bin/env bash
#
# Script that prints information about generated code in TSan runtime.
-#!/bin/bash
+#!/usr/bin/env bash
#
# Script that checks that critical functions in TSan runtime have correct number
# of push/pop/rsp instructions to verify that runtime is efficient enough.
fi
}
+# All hot functions must contain no PUSH/POP
+# and no CALLs (everything is tail-called).
for f in write1 write2 write4 write8; do
check $f rsp 1
- check $f push 2
+ check $f push 0
+ check $f pop 0
+ check $f call 0
done
for f in read1 read2 read4 read8; do
check $f rsp 1
- check $f push 3
+ check $f push 0
+ check $f pop 0
+ check $f call 0
done
for f in func_entry func_exit; do
check $f rsp 0
check $f push 0
check $f pop 0
- check $f call 1 # TraceSwitch()
+ check $f call 0
done
echo LGTM
-#!/bin/bash
+#!/usr/bin/env bash
set -u
set -e
dd_interceptors.cpp
)
-set(DD_LINKLIBS ${SANITIZER_CXX_ABI_LIBRARIES} ${SANITIZER_COMMON_LINK_LIBS})
+set(DD_LINKLIBS
+ ${COMPILER_RT_UNWINDER_LINK_LIBS}
+ ${SANITIZER_CXX_ABI_LIBRARIES}
+ ${SANITIZER_COMMON_LINK_LIBS})
append_list_if(COMPILER_RT_HAS_LIBDL dl DD_LINKLIBS)
append_list_if(COMPILER_RT_HAS_LIBRT rt DD_LINKLIBS)
if (is_bss) g_data_end = segment.end;
prev_is_data = is_data;
}
- VPrintf(1, "guessed data_start=%p data_end=%p\n", g_data_start, g_data_end);
+ VPrintf(1, "guessed data_start=0x%zx data_end=0x%zx\n", g_data_start,
+ g_data_end);
CHECK_LT(g_data_start, g_data_end);
CHECK_GE((uptr)&g_data_start, g_data_start);
CHECK_LT((uptr)&g_data_start, g_data_end);
static void ReportDeadlock(Thread *thr, DDReport *rep) {
if (rep == 0)
return;
- BlockingMutexLock lock(&ctx->report_mutex);
+ Lock lock(&ctx->report_mutex);
Printf("==============================\n");
Printf("WARNING: lock-order-inversion (potential deadlock)\n");
for (int i = 0; i < rep->n; i++) {
- Printf("Thread %d locks mutex %llu while holding mutex %llu:\n",
- rep->loop[i].thr_ctx, rep->loop[i].mtx_ctx1, rep->loop[i].mtx_ctx0);
+ Printf("Thread %lld locks mutex %llu while holding mutex %llu:\n",
+ rep->loop[i].thr_ctx, rep->loop[i].mtx_ctx1, rep->loop[i].mtx_ctx0);
PrintStackTrace(thr, rep->loop[i].stk[1]);
if (rep->loop[i].stk[0]) {
Printf("Mutex %llu was acquired here:\n",
typedef DDFlags Flags;
-struct Mutex {
+struct UserMutex {
DDMutex dd;
};
u32 Unwind() override;
};
-typedef AddrHashMap<Mutex, 31051> MutexHashMap;
+typedef AddrHashMap<UserMutex, 31051> MutexHashMap;
struct Context {
DDetector *dd;
- BlockingMutex report_mutex;
+ Mutex report_mutex;
MutexHashMap mutex_map;
};
type ^
tsan_go.cpp ^
..\rtl\tsan_interface_atomic.cpp ^
- ..\rtl\tsan_clock.cpp ^
..\rtl\tsan_flags.cpp ^
..\rtl\tsan_md5.cpp ^
..\rtl\tsan_report.cpp ^
..\rtl\tsan_rtl.cpp ^
+ ..\rtl\tsan_rtl_access.cpp ^
..\rtl\tsan_rtl_mutex.cpp ^
..\rtl\tsan_rtl_report.cpp ^
..\rtl\tsan_rtl_thread.cpp ^
..\rtl\tsan_suppressions.cpp ^
..\rtl\tsan_sync.cpp ^
..\rtl\tsan_stack_trace.cpp ^
+ ..\rtl\tsan_vector_clock.cpp ^
..\..\sanitizer_common\sanitizer_allocator.cpp ^
..\..\sanitizer_common\sanitizer_common.cpp ^
..\..\sanitizer_common\sanitizer_flags.cpp ^
..\rtl\tsan_platform_windows.cpp ^
..\..\sanitizer_common\sanitizer_win.cpp ^
..\..\sanitizer_common\sanitizer_deadlock_detector1.cpp ^
+ ..\..\sanitizer_common\sanitizer_stack_store.cpp ^
..\..\sanitizer_common\sanitizer_stackdepot.cpp ^
- ..\..\sanitizer_common\sanitizer_persistent_allocator.cpp ^
..\..\sanitizer_common\sanitizer_flag_parser.cpp ^
..\..\sanitizer_common\sanitizer_symbolizer.cpp ^
..\..\sanitizer_common\sanitizer_termination.cpp ^
-Wno-format ^
-Wno-maybe-uninitialized ^
-DSANITIZER_DEBUG=0 ^
+ -DSANITIZER_WINDOWS=1 ^
-O3 ^
-fomit-frame-pointer ^
- -std=c++14
+ -msse3 ^
+ -std=c++17
+
+rem "-msse3" used above to ensure continued support of older
+rem cpus (for now), see https://github.com/golang/go/issues/53743.
SRCS="
tsan_go.cpp
- ../rtl/tsan_clock.cpp
../rtl/tsan_external.cpp
../rtl/tsan_flags.cpp
../rtl/tsan_interface_atomic.cpp
../rtl/tsan_md5.cpp
../rtl/tsan_report.cpp
../rtl/tsan_rtl.cpp
+ ../rtl/tsan_rtl_access.cpp
../rtl/tsan_rtl_mutex.cpp
../rtl/tsan_rtl_report.cpp
../rtl/tsan_rtl_thread.cpp
../rtl/tsan_stack_trace.cpp
../rtl/tsan_suppressions.cpp
../rtl/tsan_sync.cpp
+ ../rtl/tsan_vector_clock.cpp
../../sanitizer_common/sanitizer_allocator.cpp
../../sanitizer_common/sanitizer_common.cpp
../../sanitizer_common/sanitizer_common_libcdep.cpp
../../sanitizer_common/sanitizer_flags.cpp
../../sanitizer_common/sanitizer_libc.cpp
../../sanitizer_common/sanitizer_mutex.cpp
- ../../sanitizer_common/sanitizer_persistent_allocator.cpp
../../sanitizer_common/sanitizer_printf.cpp
../../sanitizer_common/sanitizer_suppressions.cpp
../../sanitizer_common/sanitizer_thread_registry.cpp
+ ../../sanitizer_common/sanitizer_stack_store.cpp
../../sanitizer_common/sanitizer_stackdepot.cpp
../../sanitizer_common/sanitizer_stacktrace.cpp
../../sanitizer_common/sanitizer_symbolizer.cpp
ARCHCFLAGS="-m64 -mcpu=power8 -fno-function-sections"
elif [ "`uname -a | grep x86_64`" != "" ]; then
SUFFIX="linux_amd64"
- ARCHCFLAGS="-m64 -msse3"
- OSCFLAGS="$OSCFLAGS -ffreestanding -Wno-unused-const-variable -Werror -Wno-unknown-warning-option"
+ if [ "$GOAMD64" = "v3" ]; then
+ ARCHCFLAGS="-m64 -msse4.2"
+ else
+ ARCHCFLAGS="-m64 -msse3"
+ fi
+ OSCFLAGS="$OSCFLAGS -ffreestanding -Wno-unused-const-variable -Wno-unknown-warning-option"
elif [ "`uname -a | grep aarch64`" != "" ]; then
SUFFIX="linux_arm64"
ARCHCFLAGS=""
cat $F
done > $DIR/gotsan.cpp
-FLAGS=" -I../rtl -I../.. -I../../sanitizer_common -I../../../include -std=c++14 -Wall -fno-exceptions -fno-rtti -DSANITIZER_GO=1 -DSANITIZER_DEADLOCK_DETECTOR_VERSION=2 $OSCFLAGS $ARCHCFLAGS $EXTRA_CFLAGS"
+FLAGS=" -I../rtl -I../.. -I../../sanitizer_common -I../../../include -std=c++17 -Wall -fno-exceptions -fno-rtti -DSANITIZER_GO=1 -DSANITIZER_DEADLOCK_DETECTOR_VERSION=2 $OSCFLAGS $ARCHCFLAGS $EXTRA_CFLAGS"
DEBUG_FLAGS="$FLAGS -DSANITIZER_DEBUG=1 -g"
FLAGS="$FLAGS -DSANITIZER_DEBUG=0 -O3 -fomit-frame-pointer"
//
//===----------------------------------------------------------------------===//
//
-// Sanity test for Go runtime.
+// Test for Go runtime.
//
//===----------------------------------------------------------------------===//
return false;
}
-void *internal_alloc(MBlockType typ, uptr sz) {
- return InternalAlloc(sz);
-}
+void *Alloc(uptr sz) { return InternalAlloc(sz); }
-void internal_free(void *p) {
- InternalFree(p);
-}
+void FreeImpl(void *p) { InternalFree(p); }
// Callback into Go.
static void (*go_runtime_cb)(uptr cmd, void *ctx);
MBlock *b = ctx->metamap.GetBlock(cbctx.start);
if (!b)
return 0;
- ReportLocation *loc = ReportLocation::New(ReportLocationHeap);
+ auto *loc = New<ReportLocation>();
+ loc->type = ReportLocationHeap;
loc->heap_chunk_start = cbctx.start;
loc->heap_chunk_size = b->siz;
loc->tid = b->tid;
loc->stack = SymbolizeStackId(b->stk);
return loc;
} else {
- ReportLocation *loc = ReportLocation::New(ReportLocationGlobal);
+ auto *loc = New<ReportLocation>();
+ loc->type = ReportLocationGlobal;
loc->global.name = internal_strdup(cbctx.name ? cbctx.name : "??");
loc->global.file = internal_strdup(cbctx.file ? cbctx.file : "??");
loc->global.line = cbctx.line;
extern "C" {
static ThreadState *AllocGoroutine() {
- ThreadState *thr = (ThreadState*)internal_alloc(MBlockThreadContex,
- sizeof(ThreadState));
+ auto *thr = (ThreadState *)Alloc(sizeof(ThreadState));
internal_memset(thr, 0, sizeof(*thr));
return thr;
}
}
void __tsan_read(ThreadState *thr, void *addr, void *pc) {
- MemoryRead(thr, (uptr)pc, (uptr)addr, kSizeLog1);
+ MemoryAccess(thr, (uptr)pc, (uptr)addr, 1, kAccessRead);
}
void __tsan_read_pc(ThreadState *thr, void *addr, uptr callpc, uptr pc) {
if (callpc != 0)
FuncEntry(thr, callpc);
- MemoryRead(thr, (uptr)pc, (uptr)addr, kSizeLog1);
+ MemoryAccess(thr, (uptr)pc, (uptr)addr, 1, kAccessRead);
if (callpc != 0)
FuncExit(thr);
}
void __tsan_write(ThreadState *thr, void *addr, void *pc) {
- MemoryWrite(thr, (uptr)pc, (uptr)addr, kSizeLog1);
+ MemoryAccess(thr, (uptr)pc, (uptr)addr, 1, kAccessWrite);
}
void __tsan_write_pc(ThreadState *thr, void *addr, uptr callpc, uptr pc) {
if (callpc != 0)
FuncEntry(thr, callpc);
- MemoryWrite(thr, (uptr)pc, (uptr)addr, kSizeLog1);
+ MemoryAccess(thr, (uptr)pc, (uptr)addr, 1, kAccessWrite);
if (callpc != 0)
FuncExit(thr);
}
CHECK(inited);
if (thr && pc)
ctx->metamap.AllocBlock(thr, pc, p, sz);
- MemoryResetRange(0, 0, (uptr)p, sz);
+ MemoryResetRange(thr, pc, (uptr)p, sz);
}
void __tsan_free(uptr p, uptr sz) {
- ctx->metamap.FreeRange(get_cur_proc(), p, sz);
+ ctx->metamap.FreeRange(get_cur_proc(), p, sz, false);
}
void __tsan_go_start(ThreadState *parent, ThreadState **pthr, void *pc) {
ThreadState *thr = AllocGoroutine();
*pthr = thr;
- int goid = ThreadCreate(parent, (uptr)pc, 0, true);
+ Tid goid = ThreadCreate(parent, (uptr)pc, 0, true);
ThreadStart(thr, goid, 0, ThreadType::Regular);
}
void __tsan_go_end(ThreadState *thr) {
ThreadFinish(thr);
- internal_free(thr);
+ Free(thr);
}
void __tsan_proc_create(Processor **pproc) {
Release(thr, 0, (uptr)addr);
}
-void __tsan_finalizer_goroutine(ThreadState *thr) {
- AcquireGlobal(thr, 0);
-}
+void __tsan_finalizer_goroutine(ThreadState *thr) { AcquireGlobal(thr); }
void __tsan_mutex_before_lock(ThreadState *thr, uptr addr, uptr write) {
if (write)
ThreadIgnoreSyncBegin(thr, 0);
}
-void __tsan_go_ignore_sync_end(ThreadState *thr) {
- ThreadIgnoreSyncEnd(thr, 0);
-}
+void __tsan_go_ignore_sync_end(ThreadState *thr) { ThreadIgnoreSyncEnd(thr); }
void __tsan_report_count(u64 *pn) {
Lock lock(&ctx->report_mtx);
--- /dev/null
+include_directories(../..)
+
+set(TSAN_RTL_CFLAGS ${TSAN_CFLAGS})
+append_list_if(COMPILER_RT_HAS_MSSE4_2_FLAG -msse4.2 TSAN_RTL_CFLAGS)
+append_list_if(SANITIZER_LIMIT_FRAME_SIZE -Wframe-larger-than=530
+ TSAN_RTL_CFLAGS)
+append_list_if(COMPILER_RT_HAS_WGLOBAL_CONSTRUCTORS_FLAG -Wglobal-constructors
+ TSAN_RTL_CFLAGS)
+append_list_if(COMPILER_RT_INTERCEPT_LIBDISPATCH ${COMPILER_RT_LIBDISPATCH_CFLAGS}
+ TSAN_RTL_CFLAGS)
+
+set(TSAN_RTL_DYNAMIC_CFLAGS ${TSAN_RTL_CFLAGS})
+list(REMOVE_ITEM TSAN_RTL_DYNAMIC_CFLAGS -fPIE)
+
+set(TSAN_DYNAMIC_LINK_LIBS
+ ${COMPILER_RT_UNWINDER_LINK_LIBS}
+ ${SANITIZER_CXX_ABI_LIBRARIES}
+ ${SANITIZER_COMMON_LINK_LIBS})
+
+append_list_if(COMPILER_RT_HAS_LIBDL dl TSAN_DYNAMIC_LINK_LIBS)
+append_list_if(COMPILER_RT_HAS_LIBM m TSAN_DYNAMIC_LINK_LIBS)
+append_list_if(COMPILER_RT_HAS_LIBPTHREAD pthread TSAN_DYNAMIC_LINK_LIBS)
+
+set(TSAN_SOURCES
+ tsan_debugging.cpp
+ tsan_external.cpp
+ tsan_fd.cpp
+ tsan_flags.cpp
+ tsan_ignoreset.cpp
+ tsan_interceptors_posix.cpp
+ tsan_interface.cpp
+ tsan_interface_ann.cpp
+ tsan_interface_atomic.cpp
+ tsan_interface_java.cpp
+ tsan_malloc_mac.cpp
+ tsan_md5.cpp
+ tsan_mman.cpp
+ tsan_mutexset.cpp
+ tsan_report.cpp
+ tsan_rtl.cpp
+ tsan_rtl_access.cpp
+ tsan_rtl_mutex.cpp
+ tsan_rtl_proc.cpp
+ tsan_rtl_report.cpp
+ tsan_rtl_thread.cpp
+ tsan_stack_trace.cpp
+ tsan_suppressions.cpp
+ tsan_symbolize.cpp
+ tsan_sync.cpp
+ tsan_vector_clock.cpp
+ )
+
+set(TSAN_CXX_SOURCES
+ tsan_new_delete.cpp
+ )
+
+set(TSAN_PREINIT_SOURCES
+ tsan_preinit.cpp
+ )
+
+if(APPLE)
+ list(APPEND TSAN_SOURCES
+ tsan_interceptors_mac.cpp
+ tsan_interceptors_mach_vm.cpp
+ tsan_platform_mac.cpp
+ tsan_platform_posix.cpp
+ )
+elseif(UNIX)
+ # Assume Linux
+ list(APPEND TSAN_SOURCES
+ tsan_platform_linux.cpp
+ tsan_platform_posix.cpp
+ )
+endif()
+
+if(COMPILER_RT_INTERCEPT_LIBDISPATCH)
+ list(APPEND TSAN_SOURCES
+ tsan_interceptors_libdispatch.cpp
+ )
+endif()
+
+set(TSAN_HEADERS
+ tsan_defs.h
+ tsan_dense_alloc.h
+ tsan_fd.h
+ tsan_flags.h
+ tsan_flags.inc
+ tsan_ignoreset.h
+ tsan_ilist.h
+ tsan_interceptors.h
+ tsan_interface.h
+ tsan_interface.inc
+ tsan_interface_ann.h
+ tsan_interface_java.h
+ tsan_mman.h
+ tsan_mutexset.h
+ tsan_platform.h
+ tsan_ppc_regs.h
+ tsan_report.h
+ tsan_rtl.h
+ tsan_shadow.h
+ tsan_stack_trace.h
+ tsan_suppressions.h
+ tsan_symbolize.h
+ tsan_sync.h
+ tsan_trace.h
+ tsan_vector_clock.h
+ )
+
+set(TSAN_RUNTIME_LIBRARIES)
+add_compiler_rt_component(tsan)
+
+if("${CMAKE_C_FLAGS}" MATCHES "-Wno-(error=)?unused-command-line-argument")
+ set(EXTRA_CFLAGS "-Wno-error=unused-command-line-argument ${EXTRA_CFLAGS}")
+endif()
+
+if(APPLE)
+ # Ideally we would check the SDK version for the actual platform we are
+ # building for here. To make our lifes easier we assume the host SDK setup is
+ # sane and use the macOS SDK version as a proxy for aligned SDKs.
+ find_darwin_sdk_version(macosx_sdk_version "macosx")
+ if ("${macosx_sdk_version}" VERSION_LESS 10.12)
+ message(FATAL_ERROR "Building the TSan runtime requires at least macOS SDK 10.12 (or aligned SDK on other platforms)")
+ endif()
+
+ add_asm_sources(TSAN_ASM_SOURCES
+ tsan_rtl_amd64.S
+ tsan_rtl_aarch64.S
+ )
+
+ set(TSAN_LINK_LIBS ${SANITIZER_COMMON_LINK_LIBS})
+
+ add_weak_symbols("ubsan" WEAK_SYMBOL_LINK_FLAGS)
+ add_weak_symbols("sanitizer_common" WEAK_SYMBOL_LINK_FLAGS)
+
+ add_compiler_rt_runtime(clang_rt.tsan
+ SHARED
+ OS ${TSAN_SUPPORTED_OS}
+ ARCHS ${TSAN_SUPPORTED_ARCH}
+ SOURCES ${TSAN_SOURCES} ${TSAN_CXX_SOURCES} ${TSAN_ASM_SOURCES}
+ ADDITIONAL_HEADERS ${TSAN_HEADERS}
+ OBJECT_LIBS RTInterception
+ RTSanitizerCommon
+ RTSanitizerCommonLibc
+ RTSanitizerCommonCoverage
+ RTSanitizerCommonSymbolizer
+ RTUbsan
+ CFLAGS ${TSAN_RTL_CFLAGS}
+ LINK_FLAGS ${SANITIZER_COMMON_LINK_FLAGS} ${WEAK_SYMBOL_LINK_FLAGS}
+ LINK_LIBS ${TSAN_LINK_LIBS} objc
+ PARENT_TARGET tsan)
+ add_compiler_rt_object_libraries(RTTsan_dynamic
+ OS ${TSAN_SUPPORTED_OS}
+ ARCHS ${TSAN_SUPPORTED_ARCH}
+ SOURCES ${TSAN_SOURCES} ${TSAN_CXX_SOURCES} ${TSAN_ASM_SOURCES}
+ ADDITIONAL_HEADERS ${TSAN_HEADERS}
+ CFLAGS ${TSAN_RTL_CFLAGS})
+
+ # Build and check Go runtime.
+ set(BUILDGO_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/../go/buildgo.sh)
+ add_custom_target(GotsanRuntimeCheck
+ COMMAND env "CC=${CMAKE_C_COMPILER} ${OSX_SYSROOT_FLAG}"
+ EXTRA_CFLAGS=${EXTRA_CFLAGS}
+ IN_TMPDIR=1 SILENT=1 ${BUILDGO_SCRIPT}
+ DEPENDS tsan ${BUILDGO_SCRIPT}
+ WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../go
+ COMMENT "Checking TSan Go runtime..."
+ VERBATIM)
+ set_target_properties(GotsanRuntimeCheck PROPERTIES FOLDER "Compiler-RT Misc")
+else()
+ foreach(arch ${TSAN_SUPPORTED_ARCH})
+ if(arch STREQUAL "x86_64")
+ add_asm_sources(TSAN_ASM_SOURCES
+ tsan_rtl_amd64.S
+ )
+ # Check for Go runtime.
+ set(BUILDGO_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/../go/buildgo.sh)
+ add_custom_target(GotsanRuntimeCheck
+ COMMAND env "CC=${CMAKE_C_COMPILER} ${CMAKE_C_COMPILER_ARG1}"
+ EXTRA_CFLAGS=${EXTRA_CFLAGS}
+ IN_TMPDIR=1 SILENT=1 ${BUILDGO_SCRIPT}
+ DEPENDS clang_rt.tsan-${arch} ${BUILDGO_SCRIPT}
+ WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../go
+ COMMENT "Checking TSan Go runtime..."
+ VERBATIM)
+ elseif(arch STREQUAL "aarch64")
+ add_asm_sources(TSAN_ASM_SOURCES
+ tsan_rtl_aarch64.S
+ )
+ # Check for Go runtime.
+ set(BUILDGO_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/../go/buildgo.sh)
+ add_custom_target(GotsanRuntimeCheck
+ COMMAND env "CC=${CMAKE_C_COMPILER} ${CMAKE_C_COMPILER_ARG1}"
+ EXTRA_CFLAGS=${EXTRA_CFLAGS}
+ IN_TMPDIR=1 SILENT=1 ${BUILDGO_SCRIPT}
+ DEPENDS clang_rt.tsan-${arch} ${BUILDGO_SCRIPT}
+ WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../go
+ COMMENT "Checking TSan Go runtime..."
+ VERBATIM)
+ elseif(arch MATCHES "powerpc64|powerpc64le")
+ add_asm_sources(TSAN_ASM_SOURCES
+ tsan_rtl_ppc64.S
+ )
+ # Check for Go runtime.
+ set(BUILDGO_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/../go/buildgo.sh)
+ add_custom_target(GotsanRuntimeCheck
+ COMMAND env "CC=${CMAKE_C_COMPILER} ${CMAKE_C_COMPILER_ARG1}"
+ EXTRA_CFLAGS=${EXTRA_CFLAGS}
+ IN_TMPDIR=1 SILENT=1 ${BUILDGO_SCRIPT}
+ DEPENDS clang_rt.tsan-${arch} ${BUILDGO_SCRIPT}
+ WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../go
+ COMMENT "Checking TSan Go runtime..."
+ VERBATIM)
+ elseif(arch MATCHES "loongarch64")
+ add_asm_sources(TSAN_ASM_SOURCES
+ tsan_rtl_loongarch64.S
+ )
+ elseif(arch MATCHES "mips64|mips64le")
+ add_asm_sources(TSAN_ASM_SOURCES
+ tsan_rtl_mips64.S
+ )
+ elseif(arch MATCHES "s390x")
+ add_asm_sources(TSAN_ASM_SOURCES
+ tsan_rtl_s390x.S
+ )
+ # Check for Go runtime.
+ set(BUILDGO_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/../go/buildgo.sh)
+ add_custom_target(GotsanRuntimeCheck
+ COMMAND env "CC=${CMAKE_C_COMPILER} ${CMAKE_C_COMPILER_ARG1}"
+ EXTRA_CFLAGS=${EXTRA_CFLAGS}
+ IN_TMPDIR=1 SILENT=1 ${BUILDGO_SCRIPT}
+ DEPENDS clang_rt.tsan-${arch} ${BUILDGO_SCRIPT}
+ WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../go
+ COMMENT "Checking TSan Go runtime..."
+ VERBATIM)
+ else()
+ set(TSAN_ASM_SOURCES)
+ endif()
+ add_compiler_rt_runtime(clang_rt.tsan
+ STATIC
+ ARCHS ${arch}
+ SOURCES ${TSAN_SOURCES} ${TSAN_ASM_SOURCES} ${TSAN_PREINIT_SOURCES}
+ $<TARGET_OBJECTS:RTInterception.${arch}>
+ $<TARGET_OBJECTS:RTSanitizerCommon.${arch}>
+ $<TARGET_OBJECTS:RTSanitizerCommonLibc.${arch}>
+ $<TARGET_OBJECTS:RTSanitizerCommonCoverage.${arch}>
+ $<TARGET_OBJECTS:RTSanitizerCommonSymbolizer.${arch}>
+ $<TARGET_OBJECTS:RTUbsan.${arch}>
+ ADDITIONAL_HEADERS ${TSAN_HEADERS}
+ CFLAGS ${TSAN_RTL_CFLAGS}
+ PARENT_TARGET tsan)
+ add_compiler_rt_runtime(clang_rt.tsan_cxx
+ STATIC
+ ARCHS ${arch}
+ SOURCES ${TSAN_CXX_SOURCES}
+ $<TARGET_OBJECTS:RTUbsan_cxx.${arch}>
+ ADDITIONAL_HEADERS ${TSAN_HEADERS}
+ CFLAGS ${TSAN_RTL_CFLAGS}
+ PARENT_TARGET tsan)
+ list(APPEND TSAN_RUNTIME_LIBRARIES clang_rt.tsan-${arch}
+ clang_rt.tsan_cxx-${arch})
+ add_compiler_rt_runtime(clang_rt.tsan
+ SHARED
+ ARCHS ${arch}
+ SOURCES ${TSAN_SOURCES} ${TSAN_ASM_SOURCES}
+ $<TARGET_OBJECTS:RTInterception.${arch}>
+ $<TARGET_OBJECTS:RTSanitizerCommon.${arch}>
+ $<TARGET_OBJECTS:RTSanitizerCommonLibc.${arch}>
+ $<TARGET_OBJECTS:RTSanitizerCommonCoverage.${arch}>
+ $<TARGET_OBJECTS:RTSanitizerCommonSymbolizer.${arch}>
+ $<TARGET_OBJECTS:RTUbsan.${arch}>
+ ADDITIONAL_HEADERS ${TSAN_HEADERS}
+ CFLAGS ${TSAN_RTL_DYNAMIC_CFLAGS}
+ DEFS SANITIZER_SHARED
+ LINK_LIBS ${TSAN_DYNAMIC_LINK_LIBS}
+ LINK_FLAGS ${SANITIZER_COMMON_LINK_FLAGS}
+ PARENT_TARGET tsan)
+ add_sanitizer_rt_symbols(clang_rt.tsan
+ ARCHS ${arch}
+ EXTRA tsan.syms.extra)
+ add_sanitizer_rt_symbols(clang_rt.tsan_cxx
+ ARCHS ${arch}
+ EXTRA tsan.syms.extra)
+ add_dependencies(tsan clang_rt.tsan-${arch}
+ clang_rt.tsan_cxx-${arch}
+ clang_rt.tsan-${arch}-symbols
+ clang_rt.tsan_cxx-${arch}-symbols)
+ endforeach()
+endif()
+
+
__tsan_unaligned*
__tsan_release
__tsan_acquire
+__tsan_memcpy
+__tsan_memmove
+__tsan_memset
__tsan_mutex_create
__tsan_mutex_destroy
__tsan_mutex_pre_lock
ReportMutex *mutex = rep->mutexes[idx];
*mutex_id = mutex->id;
*addr = (void *)mutex->addr;
- *destroyed = mutex->destroyed;
+ *destroyed = false;
if (mutex->stack) CopyTrace(mutex->stack->frames, trace, trace_size);
return 1;
}
const char *region_kind = nullptr;
if (name && name_size > 0) name[0] = 0;
- if (IsMetaMem(addr)) {
+ if (IsMetaMem(reinterpret_cast<u32 *>(addr))) {
region_kind = "meta shadow";
- } else if (IsShadowMem(addr)) {
+ } else if (IsShadowMem(reinterpret_cast<RawShadow *>(addr))) {
region_kind = "shadow";
} else {
bool is_stack = false;
} else {
// TODO(kuba.brecka): We should not lock. This is supposed to be called
// from within the debugger when other threads are stopped.
- ctx->thread_registry->Lock();
+ ctx->thread_registry.Lock();
ThreadContext *tctx = IsThreadStackOrTls(addr, &is_stack);
- ctx->thread_registry->Unlock();
+ ctx->thread_registry.Unlock();
if (tctx) {
region_kind = is_stack ? "stack" : "tls";
} else {
*thread_id = b->tid;
// No locking. This is supposed to be called from within the debugger when
// other threads are stopped.
- ThreadContextBase *tctx = ctx->thread_registry->GetThreadLocked(b->tid);
+ ThreadContextBase *tctx = ctx->thread_registry.GetThreadLocked(b->tid);
*os_id = tctx->os_id;
StackTrace stack = StackDepotGet(b->stk);
#include "sanitizer_common/sanitizer_mutex.h"
#include "ubsan/ubsan_platform.h"
+#ifndef TSAN_VECTORIZE
+# define TSAN_VECTORIZE __SSE4_2__
+#endif
+
+#if TSAN_VECTORIZE
+// <emmintrin.h> transitively includes <stdlib.h>,
+// and it's prohibited to include std headers into tsan runtime.
+// So we do this dirty trick.
+# define _MM_MALLOC_H_INCLUDED
+# define __MM_MALLOC_H
+# include <emmintrin.h>
+# include <smmintrin.h>
+# define VECTOR_ALIGNED ALIGNED(16)
+typedef __m128i m128;
+#else
+# define VECTOR_ALIGNED
+#endif
+
// Setup defaults for compile definitions.
#ifndef TSAN_NO_HISTORY
# define TSAN_NO_HISTORY 0
namespace __tsan {
-const int kClkBits = 42;
-const unsigned kMaxTidReuse = (1 << (64 - kClkBits)) - 1;
+constexpr uptr kByteBits = 8;
-struct ClockElem {
- u64 epoch : kClkBits;
- u64 reused : 64 - kClkBits; // tid reuse count
-};
+// Thread slot ID.
+enum class Sid : u8 {};
+constexpr uptr kThreadSlotCount = 256;
+constexpr Sid kFreeSid = static_cast<Sid>(255);
-struct ClockBlock {
- static const uptr kSize = 512;
- static const uptr kTableSize = kSize / sizeof(u32);
- static const uptr kClockCount = kSize / sizeof(ClockElem);
- static const uptr kRefIdx = kTableSize - 1;
- static const uptr kBlockIdx = kTableSize - 2;
+// Abstract time unit, vector clock element.
+enum class Epoch : u16 {};
+constexpr uptr kEpochBits = 14;
+constexpr Epoch kEpochZero = static_cast<Epoch>(0);
+constexpr Epoch kEpochOver = static_cast<Epoch>(1 << kEpochBits);
+constexpr Epoch kEpochLast = static_cast<Epoch>((1 << kEpochBits) - 1);
- union {
- u32 table[kTableSize];
- ClockElem clock[kClockCount];
- };
+inline Epoch EpochInc(Epoch epoch) {
+ return static_cast<Epoch>(static_cast<u16>(epoch) + 1);
+}
- ClockBlock() {
- }
-};
+inline bool EpochOverflow(Epoch epoch) { return epoch == kEpochOver; }
-const int kTidBits = 13;
-// Reduce kMaxTid by kClockCount because one slot in ClockBlock table is
-// occupied by reference counter, so total number of elements we can store
-// in SyncClock is kClockCount * (kTableSize - 1).
-const unsigned kMaxTid = (1 << kTidBits) - ClockBlock::kClockCount;
-#if !SANITIZER_GO
-const unsigned kMaxTidInClock = kMaxTid * 2; // This includes msb 'freed' bit.
-#else
-const unsigned kMaxTidInClock = kMaxTid; // Go does not track freed memory.
-#endif
const uptr kShadowStackSize = 64 * 1024;
// Count of shadow values in a shadow cell.
// That many user bytes are mapped onto a single shadow cell.
const uptr kShadowCell = 8;
-// Size of a single shadow value (u64).
-const uptr kShadowSize = 8;
+// Single shadow value.
+enum class RawShadow : u32 {};
+const uptr kShadowSize = sizeof(RawShadow);
// Shadow memory is kShadowMultiplier times larger than user memory.
const uptr kShadowMultiplier = kShadowSize * kShadowCnt / kShadowCell;
// Size of a single meta shadow value (u32).
const uptr kMetaShadowSize = 4;
+// All addresses and PCs are assumed to be compressable to that many bits.
+const uptr kCompressedAddrBits = 44;
+
#if TSAN_NO_HISTORY
const bool kCollectHistory = false;
#else
struct Processor;
struct ThreadState;
class ThreadContext;
+struct TidSlot;
struct Context;
struct ReportStack;
class ReportDesc;
class RegionAlloc;
+struct Trace;
+struct TracePart;
+
+typedef uptr AccessType;
+
+enum : AccessType {
+ kAccessWrite = 0,
+ kAccessRead = 1 << 0,
+ kAccessAtomic = 1 << 1,
+ kAccessVptr = 1 << 2, // read or write of an object virtual table pointer
+ kAccessFree = 1 << 3, // synthetic memory access during memory freeing
+ kAccessExternalPC = 1 << 4, // access PC can have kExternalPCBit set
+ kAccessCheckOnly = 1 << 5, // check for races, but don't store
+ kAccessNoRodata = 1 << 6, // don't check for .rodata marker
+ kAccessSlotLocked = 1 << 7, // memory access with TidSlot locked
+};
// Descriptor of user's memory block.
struct MBlock {
u64 siz : 48;
u64 tag : 16;
- u32 stk;
- u16 tid;
+ StackID stk;
+ Tid tid;
};
COMPILER_CHECK(sizeof(MBlock) == 16);
// as 16-bit values, see tsan_defs.h.
};
-enum MutexType {
- MutexTypeTrace = MutexLastCommon,
- MutexTypeReport,
+enum {
+ MutexTypeReport = MutexLastCommon,
MutexTypeSyncVar,
MutexTypeAnnotations,
MutexTypeAtExit,
MutexTypeFired,
MutexTypeRacy,
MutexTypeGlobalProc,
+ MutexTypeInternalAlloc,
+ MutexTypeTrace,
+ MutexTypeSlot,
+ MutexTypeSlots,
};
} // namespace __tsan
static_assert(sizeof(T) > sizeof(IndexT),
"it doesn't make sense to use dense alloc");
- explicit DenseSlabAlloc(LinkerInitialized, const char *name) {
- freelist_ = 0;
- fillpos_ = 0;
- name_ = name;
- }
+ DenseSlabAlloc(LinkerInitialized, const char *name) : name_(name) {}
explicit DenseSlabAlloc(const char *name)
: DenseSlabAlloc(LINKER_INITIALIZED, name) {
}
void FlushCache(Cache *c) {
- SpinMutexLock lock(&mtx_);
- while (c->pos) {
- IndexT idx = c->cache[--c->pos];
- *(IndexT*)Map(idx) = freelist_;
- freelist_ = idx;
- }
+ while (c->pos) Drain(c);
}
void InitCache(Cache *c) {
internal_memset(c->cache, 0, sizeof(c->cache));
}
+ uptr AllocatedMemory() const {
+ return atomic_load_relaxed(&fillpos_) * kL2Size * sizeof(T);
+ }
+
+ template <typename Func>
+ void ForEach(Func func) {
+ Lock lock(&mtx_);
+ uptr fillpos = atomic_load_relaxed(&fillpos_);
+ for (uptr l1 = 0; l1 < fillpos; l1++) {
+ for (IndexT l2 = l1 == 0 ? 1 : 0; l2 < kL2Size; l2++) func(&map_[l1][l2]);
+ }
+ }
+
private:
T *map_[kL1Size];
- SpinMutex mtx_;
- IndexT freelist_;
- uptr fillpos_;
- const char *name_;
-
- void Refill(Cache *c) {
- SpinMutexLock lock(&mtx_);
- if (freelist_ == 0) {
- if (fillpos_ == kL1Size) {
- Printf("ThreadSanitizer: %s overflow (%zu*%zu). Dying.\n",
- name_, kL1Size, kL2Size);
- Die();
- }
- VPrintf(2, "ThreadSanitizer: growing %s: %zu out of %zu*%zu\n",
- name_, fillpos_, kL1Size, kL2Size);
- T *batch = (T*)MmapOrDie(kL2Size * sizeof(T), name_);
- // Reserve 0 as invalid index.
- IndexT start = fillpos_ == 0 ? 1 : 0;
- for (IndexT i = start; i < kL2Size; i++) {
- new(batch + i) T;
- *(IndexT*)(batch + i) = i + 1 + fillpos_ * kL2Size;
- }
- *(IndexT*)(batch + kL2Size - 1) = 0;
- freelist_ = fillpos_ * kL2Size + start;
- map_[fillpos_++] = batch;
- }
- for (uptr i = 0; i < Cache::kSize / 2 && freelist_ != 0; i++) {
- IndexT idx = freelist_;
+ Mutex mtx_;
+ // The freelist is organized as a lock-free stack of batches of nodes.
+ // The stack itself uses Block::next links, while the batch within each
+ // stack node uses Block::batch links.
+ // Low 32-bits of freelist_ is the node index, top 32-bits is ABA-counter.
+ atomic_uint64_t freelist_ = {0};
+ atomic_uintptr_t fillpos_ = {0};
+ const char *const name_;
+
+ struct Block {
+ IndexT next;
+ IndexT batch;
+ };
+
+ Block *MapBlock(IndexT idx) { return reinterpret_cast<Block *>(Map(idx)); }
+
+ static constexpr u64 kCounterInc = 1ull << 32;
+ static constexpr u64 kCounterMask = ~(kCounterInc - 1);
+
+ NOINLINE void Refill(Cache *c) {
+ // Pop 1 batch of nodes from the freelist.
+ IndexT idx;
+ u64 xchg;
+ u64 cmp = atomic_load(&freelist_, memory_order_acquire);
+ do {
+ idx = static_cast<IndexT>(cmp);
+ if (!idx)
+ return AllocSuperBlock(c);
+ Block *ptr = MapBlock(idx);
+ xchg = ptr->next | (cmp & kCounterMask);
+ } while (!atomic_compare_exchange_weak(&freelist_, &cmp, xchg,
+ memory_order_acq_rel));
+ // Unpack it into c->cache.
+ while (idx) {
c->cache[c->pos++] = idx;
- freelist_ = *(IndexT*)Map(idx);
+ idx = MapBlock(idx)->batch;
}
}
- void Drain(Cache *c) {
- SpinMutexLock lock(&mtx_);
- for (uptr i = 0; i < Cache::kSize / 2; i++) {
+ NOINLINE void Drain(Cache *c) {
+ // Build a batch of at most Cache::kSize / 2 nodes linked by Block::batch.
+ IndexT head_idx = 0;
+ for (uptr i = 0; i < Cache::kSize / 2 && c->pos; i++) {
IndexT idx = c->cache[--c->pos];
- *(IndexT*)Map(idx) = freelist_;
- freelist_ = idx;
+ Block *ptr = MapBlock(idx);
+ ptr->batch = head_idx;
+ head_idx = idx;
+ }
+ // Push it onto the freelist stack.
+ Block *head = MapBlock(head_idx);
+ u64 xchg;
+ u64 cmp = atomic_load(&freelist_, memory_order_acquire);
+ do {
+ head->next = static_cast<IndexT>(cmp);
+ xchg = head_idx | (cmp & kCounterMask) + kCounterInc;
+ } while (!atomic_compare_exchange_weak(&freelist_, &cmp, xchg,
+ memory_order_acq_rel));
+ }
+
+ NOINLINE void AllocSuperBlock(Cache *c) {
+ Lock lock(&mtx_);
+ uptr fillpos = atomic_load_relaxed(&fillpos_);
+ if (fillpos == kL1Size) {
+ Printf("ThreadSanitizer: %s overflow (%zu*%zu). Dying.\n", name_, kL1Size,
+ kL2Size);
+ Die();
+ }
+ VPrintf(2, "ThreadSanitizer: growing %s: %zu out of %zu*%zu\n", name_,
+ fillpos, kL1Size, kL2Size);
+ T *batch = (T *)MmapOrDie(kL2Size * sizeof(T), name_);
+ map_[fillpos] = batch;
+ // Reserve 0 as invalid index.
+ for (IndexT i = fillpos ? 0 : 1; i < kL2Size; i++) {
+ new (batch + i) T;
+ c->cache[c->pos++] = i + fillpos * kL2Size;
+ if (c->pos == Cache::kSize)
+ Drain(c);
}
+ atomic_store_relaxed(&fillpos_, fillpos + 1);
+ CHECK(c->pos);
}
};
# define DISPATCH_NOESCAPE
#endif
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
# define SANITIZER_WEAK_IMPORT extern "C" __attribute((weak_import))
#else
# define SANITIZER_WEAK_IMPORT extern "C" __attribute((weak))
//
//===----------------------------------------------------------------------===//
#include "tsan_rtl.h"
-#include "tsan_interceptors.h"
#include "sanitizer_common/sanitizer_ptrauth.h"
+#if !SANITIZER_GO
+# include "tsan_interceptors.h"
+#endif
+
namespace __tsan {
#define CALLERPC ((uptr)__builtin_return_address(0))
#if !SANITIZER_GO
-typedef void(*AccessFunc)(ThreadState *, uptr, uptr, int);
-void ExternalAccess(void *addr, uptr caller_pc, void *tag, AccessFunc access) {
+void ExternalAccess(void *addr, uptr caller_pc, void *tag, AccessType typ) {
CHECK_LT(tag, atomic_load(&used_tags, memory_order_relaxed));
ThreadState *thr = cur_thread();
if (caller_pc) FuncEntry(thr, caller_pc);
InsertShadowStackFrameForTag(thr, (uptr)tag);
bool in_ignored_lib;
- if (!caller_pc || !libignore()->IsIgnored(caller_pc, &in_ignored_lib)) {
- access(thr, CALLERPC, (uptr)addr, kSizeLog1);
- }
+ if (!caller_pc || !libignore()->IsIgnored(caller_pc, &in_ignored_lib))
+ MemoryAccess(thr, CALLERPC, (uptr)addr, 1, typ);
FuncExit(thr);
if (caller_pc) FuncExit(thr);
}
header = internal_strdup(header);
char *old_header =
(char *)atomic_exchange(header_ptr, (uptr)header, memory_order_seq_cst);
- if (old_header) internal_free(old_header);
+ Free(old_header);
}
SANITIZER_INTERFACE_ATTRIBUTE
SANITIZER_INTERFACE_ATTRIBUTE
void __tsan_external_read(void *addr, void *caller_pc, void *tag) {
- ExternalAccess(addr, STRIP_PAC_PC(caller_pc), tag, MemoryRead);
+ ExternalAccess(addr, STRIP_PAC_PC(caller_pc), tag, kAccessRead);
}
SANITIZER_INTERFACE_ATTRIBUTE
void __tsan_external_write(void *addr, void *caller_pc, void *tag) {
- ExternalAccess(addr, STRIP_PAC_PC(caller_pc), tag, MemoryWrite);
+ ExternalAccess(addr, STRIP_PAC_PC(caller_pc), tag, kAccessWrite);
}
} // extern "C"
//===----------------------------------------------------------------------===//
#include "tsan_fd.h"
-#include "tsan_rtl.h"
+
#include <sanitizer_common/sanitizer_atomic.h>
+#include "tsan_interceptors.h"
+#include "tsan_rtl.h"
+
namespace __tsan {
const int kTableSizeL1 = 1024;
struct FdDesc {
FdSync *sync;
- int creation_tid;
- u32 creation_stack;
+ // This is used to establish write -> epoll_wait synchronization
+ // where epoll_wait receives notification about the write.
+ atomic_uintptr_t aux_sync; // FdSync*
+ Tid creation_tid;
+ StackID creation_stack;
+ bool closed;
};
struct FdContext {
unref(thr, pc, d->sync);
d->sync = 0;
}
+ unref(thr, pc,
+ reinterpret_cast<FdSync *>(
+ atomic_load(&d->aux_sync, memory_order_relaxed)));
+ atomic_store(&d->aux_sync, 0, memory_order_relaxed);
if (flags()->io_sync == 0) {
unref(thr, pc, s);
} else if (flags()->io_sync == 1) {
}
d->creation_tid = thr->tid;
d->creation_stack = CurrentStackId(thr, pc);
+ d->closed = false;
+ // This prevents false positives on fd_close_norace3.cpp test.
+ // The mechanics of the false positive are not completely clear,
+ // but it happens only if global reset is enabled (flush_memory_ms=1)
+ // and may be related to lost writes during asynchronous MADV_DONTNEED.
+ SlotLocker locker(thr);
if (write) {
// To catch races between fd usage and open.
MemoryRangeImitateWrite(thr, pc, (uptr)d, 8);
} else {
// See the dup-related comment in FdClose.
- MemoryRead(thr, pc, (uptr)d, kSizeLog8);
+ MemoryAccess(thr, pc, (uptr)d, 8, kAccessRead | kAccessSlotLocked);
}
}
}
}
-bool FdLocation(uptr addr, int *fd, int *tid, u32 *stack) {
+bool FdLocation(uptr addr, int *fd, Tid *tid, StackID *stack, bool *closed) {
for (int l1 = 0; l1 < kTableSizeL1; l1++) {
FdDesc *tab = (FdDesc*)atomic_load(&fdctx.tab[l1], memory_order_relaxed);
if (tab == 0)
*fd = l1 * kTableSizeL1 + l2;
*tid = d->creation_tid;
*stack = d->creation_stack;
+ *closed = d->closed;
return true;
}
}
FdDesc *d = fddesc(thr, pc, fd);
FdSync *s = d->sync;
DPrintf("#%d: FdAcquire(%d) -> %p\n", thr->tid, fd, s);
- MemoryRead(thr, pc, (uptr)d, kSizeLog8);
+ MemoryAccess(thr, pc, (uptr)d, 8, kAccessRead);
if (s)
Acquire(thr, pc, (uptr)s);
}
FdDesc *d = fddesc(thr, pc, fd);
FdSync *s = d->sync;
DPrintf("#%d: FdRelease(%d) -> %p\n", thr->tid, fd, s);
- MemoryRead(thr, pc, (uptr)d, kSizeLog8);
+ MemoryAccess(thr, pc, (uptr)d, 8, kAccessRead);
if (s)
Release(thr, pc, (uptr)s);
+ if (uptr aux_sync = atomic_load(&d->aux_sync, memory_order_acquire))
+ Release(thr, pc, aux_sync);
}
void FdAccess(ThreadState *thr, uptr pc, int fd) {
if (bogusfd(fd))
return;
FdDesc *d = fddesc(thr, pc, fd);
- MemoryRead(thr, pc, (uptr)d, kSizeLog8);
+ MemoryAccess(thr, pc, (uptr)d, 8, kAccessRead);
}
void FdClose(ThreadState *thr, uptr pc, int fd, bool write) {
if (bogusfd(fd))
return;
FdDesc *d = fddesc(thr, pc, fd);
- if (write) {
- // To catch races between fd usage and close.
- MemoryWrite(thr, pc, (uptr)d, kSizeLog8);
- } else {
- // This path is used only by dup2/dup3 calls.
- // We do read instead of write because there is a number of legitimate
- // cases where write would lead to false positives:
- // 1. Some software dups a closed pipe in place of a socket before closing
- // the socket (to prevent races actually).
- // 2. Some daemons dup /dev/null in place of stdin/stdout.
- // On the other hand we have not seen cases when write here catches real
- // bugs.
- MemoryRead(thr, pc, (uptr)d, kSizeLog8);
+ {
+ // Need to lock the slot to make MemoryAccess and MemoryResetRange atomic
+ // with respect to global reset. See the comment in MemoryRangeFreed.
+ SlotLocker locker(thr);
+ if (!MustIgnoreInterceptor(thr)) {
+ if (write) {
+ // To catch races between fd usage and close.
+ MemoryAccess(thr, pc, (uptr)d, 8,
+ kAccessWrite | kAccessCheckOnly | kAccessSlotLocked);
+ } else {
+ // This path is used only by dup2/dup3 calls.
+ // We do read instead of write because there is a number of legitimate
+ // cases where write would lead to false positives:
+ // 1. Some software dups a closed pipe in place of a socket before
+ // closing
+ // the socket (to prevent races actually).
+ // 2. Some daemons dup /dev/null in place of stdin/stdout.
+ // On the other hand we have not seen cases when write here catches real
+ // bugs.
+ MemoryAccess(thr, pc, (uptr)d, 8,
+ kAccessRead | kAccessCheckOnly | kAccessSlotLocked);
+ }
+ }
+ // We need to clear it, because if we do not intercept any call out there
+ // that creates fd, we will hit false postives.
+ MemoryResetRange(thr, pc, (uptr)d, 8);
}
- // We need to clear it, because if we do not intercept any call out there
- // that creates fd, we will hit false postives.
- MemoryResetRange(thr, pc, (uptr)d, 8);
unref(thr, pc, d->sync);
d->sync = 0;
- d->creation_tid = 0;
- d->creation_stack = 0;
+ unref(thr, pc,
+ reinterpret_cast<FdSync *>(
+ atomic_load(&d->aux_sync, memory_order_relaxed)));
+ atomic_store(&d->aux_sync, 0, memory_order_relaxed);
+ d->closed = true;
+ d->creation_tid = thr->tid;
+ d->creation_stack = CurrentStackId(thr, pc);
}
void FdFileCreate(ThreadState *thr, uptr pc, int fd) {
return;
// Ignore the case when user dups not yet connected socket.
FdDesc *od = fddesc(thr, pc, oldfd);
- MemoryRead(thr, pc, (uptr)od, kSizeLog8);
+ MemoryAccess(thr, pc, (uptr)od, 8, kAccessRead);
FdClose(thr, pc, newfd, write);
init(thr, pc, newfd, ref(od->sync), write);
}
init(thr, pc, fd, allocsync(thr, pc));
}
+void FdPollAdd(ThreadState *thr, uptr pc, int epfd, int fd) {
+ DPrintf("#%d: FdPollAdd(%d, %d)\n", thr->tid, epfd, fd);
+ if (bogusfd(epfd) || bogusfd(fd))
+ return;
+ FdDesc *d = fddesc(thr, pc, fd);
+ // Associate fd with epoll fd only once.
+ // While an fd can be associated with multiple epolls at the same time,
+ // or with different epolls during different phases of lifetime,
+ // synchronization semantics (and examples) of this are unclear.
+ // So we don't support this for now.
+ // If we change the association, it will also create lifetime management
+ // problem for FdRelease which accesses the aux_sync.
+ if (atomic_load(&d->aux_sync, memory_order_relaxed))
+ return;
+ FdDesc *epd = fddesc(thr, pc, epfd);
+ FdSync *s = epd->sync;
+ if (!s)
+ return;
+ uptr cmp = 0;
+ if (atomic_compare_exchange_strong(
+ &d->aux_sync, &cmp, reinterpret_cast<uptr>(s), memory_order_release))
+ ref(s);
+}
+
void FdSocketCreate(ThreadState *thr, uptr pc, int fd) {
DPrintf("#%d: FdSocketCreate(%d)\n", thr->tid, fd);
if (bogusfd(fd))
void FdSignalCreate(ThreadState *thr, uptr pc, int fd);
void FdInotifyCreate(ThreadState *thr, uptr pc, int fd);
void FdPollCreate(ThreadState *thr, uptr pc, int fd);
+void FdPollAdd(ThreadState *thr, uptr pc, int epfd, int fd);
void FdSocketCreate(ThreadState *thr, uptr pc, int fd);
void FdSocketAccept(ThreadState *thr, uptr pc, int fd, int newfd);
void FdSocketConnecting(ThreadState *thr, uptr pc, int fd);
void FdSocketConnect(ThreadState *thr, uptr pc, int fd);
-bool FdLocation(uptr addr, int *fd, int *tid, u32 *stack);
+bool FdLocation(uptr addr, int *fd, Tid *tid, StackID *stack, bool *closed);
void FdOnFork(ThreadState *thr, uptr pc);
uptr File2addr(const char *path);
// Override some common flags defaults.
CommonFlags cf;
cf.CopyFrom(*common_flags());
+ cf.external_symbolizer_path = GetEnv("TSAN_SYMBOLIZER_PATH");
cf.allow_addr2line = true;
if (SANITIZER_GO) {
// Does not work as expected for Go: runtime handles SIGABRT and crashes.
ubsan_parser.ParseStringFromEnv("UBSAN_OPTIONS");
#endif
- // Sanity check.
+ // Check flags.
if (!f->report_bugs) {
f->report_thread_leaks = false;
f->report_destroy_locked = false;
if (common_flags()->help) parser.PrintFlagDescriptions();
- if (f->history_size < 0 || f->history_size > 7) {
- Printf("ThreadSanitizer: incorrect value for history_size"
- " (must be [0..7])\n");
- Die();
- }
-
if (f->io_sync < 0 || f->io_sync > 2) {
Printf("ThreadSanitizer: incorrect value for io_sync"
" (must be [0..2])\n");
TSAN_FLAG(bool, suppress_equal_stacks, true,
"Suppress a race report if we've already output another race report "
"with the same stack.")
-TSAN_FLAG(bool, suppress_equal_addresses, true,
- "Suppress a race report if we've already output another race report "
- "on the same address.")
-
TSAN_FLAG(bool, report_bugs, true,
"Turns off bug reporting entirely (useful for benchmarking).")
TSAN_FLAG(bool, report_thread_leaks, true, "Report thread leaks at exit?")
bool, force_seq_cst_atomics, false,
"If set, all atomics are effectively sequentially consistent (seq_cst), "
"regardless of what user actually specified.")
-TSAN_FLAG(bool, print_benign, false, "Print matched \"benign\" races at exit.")
+TSAN_FLAG(bool, force_background_thread, false,
+ "If set, eagerly launch a background thread for memory reclamation "
+ "instead of waiting for a user call to pthread_create.")
TSAN_FLAG(bool, halt_on_error, false, "Exit after first reported error.")
TSAN_FLAG(int, atexit_sleep_ms, 1000,
"Sleep in main thread before exiting for that many ms "
"Stops on start until __tsan_resume() is called (for debugging).")
TSAN_FLAG(bool, running_on_valgrind, false,
"Controls whether RunningOnValgrind() returns true or false.")
-// There are a lot of goroutines in Go, so we use smaller history.
TSAN_FLAG(
- int, history_size, SANITIZER_GO ? 1 : 3,
- "Per-thread history size, controls how many previous memory accesses "
- "are remembered per thread. Possible values are [0..7]. "
- "history_size=0 amounts to 32K memory accesses. Each next value doubles "
- "the amount of memory accesses, up to history_size=7 that amounts to "
- "4M memory accesses. The default value is 2 (128K memory accesses).")
+ uptr, history_size, 0,
+ "Per-thread history size,"
+ " controls how many extra previous memory accesses are remembered per thread.")
TSAN_FLAG(int, io_sync, 1,
"Controls level of synchronization implied by IO operations. "
"0 - no synchronization "
TSAN_FLAG(bool, die_after_fork, true,
"Die after multi-threaded fork if the child creates new threads.")
TSAN_FLAG(const char *, suppressions, "", "Suppressions file name.")
-TSAN_FLAG(bool, ignore_interceptors_accesses, SANITIZER_MAC ? true : false,
+TSAN_FLAG(bool, ignore_interceptors_accesses, SANITIZER_APPLE ? true : false,
"Ignore reads and writes from all interceptors.")
-TSAN_FLAG(bool, ignore_noninstrumented_modules, SANITIZER_MAC ? true : false,
+TSAN_FLAG(bool, ignore_noninstrumented_modules, SANITIZER_APPLE ? true : false,
"Interceptors should only detect races when called from instrumented "
"modules.")
TSAN_FLAG(bool, shared_ptr_interceptor, true,
"Track atomic reference counting in libc++ shared_ptr and weak_ptr.")
+TSAN_FLAG(bool, print_full_thread_history, false,
+ "If set, prints thread creation stacks for the threads involved in "
+ "the report and their ancestors up to the main thread.")
: size_() {
}
-void IgnoreSet::Add(u32 stack_id) {
+void IgnoreSet::Add(StackID stack_id) {
if (size_ == kMaxSize)
return;
for (uptr i = 0; i < size_; i++) {
stacks_[size_++] = stack_id;
}
-void IgnoreSet::Reset() {
- size_ = 0;
-}
-
-uptr IgnoreSet::Size() const {
- return size_;
-}
-
-u32 IgnoreSet::At(uptr i) const {
+StackID IgnoreSet::At(uptr i) const {
CHECK_LT(i, size_);
CHECK_LE(size_, kMaxSize);
return stacks_[i];
class IgnoreSet {
public:
- static const uptr kMaxSize = 16;
-
IgnoreSet();
- void Add(u32 stack_id);
- void Reset();
- uptr Size() const;
- u32 At(uptr i) const;
+ void Add(StackID stack_id);
+ void Reset() { size_ = 0; }
+ uptr Size() const { return size_; }
+ StackID At(uptr i) const;
private:
+ static constexpr uptr kMaxSize = 16;
uptr size_;
- u32 stacks_[kMaxSize];
+ StackID stacks_[kMaxSize];
};
} // namespace __tsan
--- /dev/null
+//===-- tsan_ilist.h --------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#ifndef TSAN_ILIST_H
+#define TSAN_ILIST_H
+
+#include "sanitizer_common/sanitizer_internal_defs.h"
+
+namespace __tsan {
+
+class INode {
+ public:
+ INode() = default;
+
+ private:
+ INode* next_ = nullptr;
+ INode* prev_ = nullptr;
+
+ template <typename Base, INode Base::*Node, typename Elem>
+ friend class IList;
+ INode(const INode&) = delete;
+ void operator=(const INode&) = delete;
+};
+
+// Intrusive doubly-linked list.
+//
+// The node class (MyNode) needs to include "INode foo" field,
+// then the list can be declared as IList<MyNode, &MyNode::foo>.
+// This design allows to link MyNode into multiple lists using
+// different INode fields.
+// The optional Elem template argument allows to specify node MDT
+// (most derived type) if it's different from MyNode.
+template <typename Base, INode Base::*Node, typename Elem = Base>
+class IList {
+ public:
+ IList();
+
+ void PushFront(Elem* e);
+ void PushBack(Elem* e);
+ void Remove(Elem* e);
+
+ Elem* PopFront();
+ Elem* PopBack();
+ Elem* Front();
+ Elem* Back();
+
+ // Prev links point towards front of the queue.
+ Elem* Prev(Elem* e);
+ // Next links point towards back of the queue.
+ Elem* Next(Elem* e);
+
+ uptr Size() const;
+ bool Empty() const;
+ bool Queued(Elem* e) const;
+
+ private:
+ INode node_;
+ uptr size_ = 0;
+
+ void Push(Elem* e, INode* after);
+ static INode* ToNode(Elem* e);
+ static Elem* ToElem(INode* n);
+
+ IList(const IList&) = delete;
+ void operator=(const IList&) = delete;
+};
+
+template <typename Base, INode Base::*Node, typename Elem>
+IList<Base, Node, Elem>::IList() {
+ node_.next_ = node_.prev_ = &node_;
+}
+
+template <typename Base, INode Base::*Node, typename Elem>
+void IList<Base, Node, Elem>::PushFront(Elem* e) {
+ Push(e, &node_);
+}
+
+template <typename Base, INode Base::*Node, typename Elem>
+void IList<Base, Node, Elem>::PushBack(Elem* e) {
+ Push(e, node_.prev_);
+}
+
+template <typename Base, INode Base::*Node, typename Elem>
+void IList<Base, Node, Elem>::Push(Elem* e, INode* after) {
+ INode* n = ToNode(e);
+ DCHECK_EQ(n->next_, nullptr);
+ DCHECK_EQ(n->prev_, nullptr);
+ INode* next = after->next_;
+ n->next_ = next;
+ n->prev_ = after;
+ next->prev_ = n;
+ after->next_ = n;
+ size_++;
+}
+
+template <typename Base, INode Base::*Node, typename Elem>
+void IList<Base, Node, Elem>::Remove(Elem* e) {
+ INode* n = ToNode(e);
+ INode* next = n->next_;
+ INode* prev = n->prev_;
+ DCHECK(next);
+ DCHECK(prev);
+ DCHECK(size_);
+ next->prev_ = prev;
+ prev->next_ = next;
+ n->prev_ = n->next_ = nullptr;
+ size_--;
+}
+
+template <typename Base, INode Base::*Node, typename Elem>
+Elem* IList<Base, Node, Elem>::PopFront() {
+ Elem* e = Front();
+ if (e)
+ Remove(e);
+ return e;
+}
+
+template <typename Base, INode Base::*Node, typename Elem>
+Elem* IList<Base, Node, Elem>::PopBack() {
+ Elem* e = Back();
+ if (e)
+ Remove(e);
+ return e;
+}
+
+template <typename Base, INode Base::*Node, typename Elem>
+Elem* IList<Base, Node, Elem>::Front() {
+ return size_ ? ToElem(node_.next_) : nullptr;
+}
+
+template <typename Base, INode Base::*Node, typename Elem>
+Elem* IList<Base, Node, Elem>::Back() {
+ return size_ ? ToElem(node_.prev_) : nullptr;
+}
+
+template <typename Base, INode Base::*Node, typename Elem>
+Elem* IList<Base, Node, Elem>::Prev(Elem* e) {
+ INode* n = ToNode(e);
+ DCHECK(n->prev_);
+ return n->prev_ != &node_ ? ToElem(n->prev_) : nullptr;
+}
+
+template <typename Base, INode Base::*Node, typename Elem>
+Elem* IList<Base, Node, Elem>::Next(Elem* e) {
+ INode* n = ToNode(e);
+ DCHECK(n->next_);
+ return n->next_ != &node_ ? ToElem(n->next_) : nullptr;
+}
+
+template <typename Base, INode Base::*Node, typename Elem>
+uptr IList<Base, Node, Elem>::Size() const {
+ return size_;
+}
+
+template <typename Base, INode Base::*Node, typename Elem>
+bool IList<Base, Node, Elem>::Empty() const {
+ return size_ == 0;
+}
+
+template <typename Base, INode Base::*Node, typename Elem>
+bool IList<Base, Node, Elem>::Queued(Elem* e) const {
+ INode* n = ToNode(e);
+ DCHECK_EQ(!n->next_, !n->prev_);
+ return n->next_;
+}
+
+template <typename Base, INode Base::*Node, typename Elem>
+INode* IList<Base, Node, Elem>::ToNode(Elem* e) {
+ return &(e->*Node);
+}
+
+template <typename Base, INode Base::*Node, typename Elem>
+Elem* IList<Base, Node, Elem>::ToElem(INode* n) {
+ return static_cast<Elem*>(reinterpret_cast<Base*>(
+ reinterpret_cast<uptr>(n) -
+ reinterpret_cast<uptr>(&(reinterpret_cast<Elem*>(0)->*Node))));
+}
+
+} // namespace __tsan
+
+#endif
public:
ScopedInterceptor(ThreadState *thr, const char *fname, uptr pc);
~ScopedInterceptor();
- void DisableIgnores();
- void EnableIgnores();
+ void DisableIgnores() {
+ if (UNLIKELY(ignoring_))
+ DisableIgnoresImpl();
+ }
+ void EnableIgnores() {
+ if (UNLIKELY(ignoring_))
+ EnableIgnoresImpl();
+ }
+
private:
ThreadState *const thr_;
- const uptr pc_;
- bool in_ignored_lib_;
- bool ignoring_;
+ bool in_ignored_lib_ = false;
+ bool in_blocking_func_ = false;
+ bool ignoring_ = false;
+
+ void DisableIgnoresImpl();
+ void EnableIgnoresImpl();
};
LibIgnore *libignore();
#if !SANITIZER_GO
inline bool in_symbolizer() {
- cur_thread_init();
- return UNLIKELY(cur_thread()->in_symbolizer);
+ return UNLIKELY(cur_thread_init()->in_symbolizer);
}
#endif
+inline bool MustIgnoreInterceptor(ThreadState *thr) {
+ return !thr->is_inited || thr->ignore_interceptors || thr->in_ignored_lib;
+}
+
} // namespace __tsan
-#define SCOPED_INTERCEPTOR_RAW(func, ...) \
- cur_thread_init(); \
- ThreadState *thr = cur_thread(); \
- const uptr caller_pc = GET_CALLER_PC(); \
- ScopedInterceptor si(thr, #func, caller_pc); \
- const uptr pc = GET_CURRENT_PC(); \
- (void)pc; \
- /**/
-
-#define SCOPED_TSAN_INTERCEPTOR(func, ...) \
- SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \
- if (REAL(func) == 0) { \
+#define SCOPED_INTERCEPTOR_RAW(func, ...) \
+ ThreadState *thr = cur_thread_init(); \
+ ScopedInterceptor si(thr, #func, GET_CALLER_PC()); \
+ UNUSED const uptr pc = GET_CURRENT_PC();
+
+#ifdef __powerpc64__
+// Debugging of crashes on powerpc after commit:
+// c80604f7a3 ("tsan: remove real func check from interceptors")
+// Somehow replacing if with DCHECK leads to strange failures in:
+// SanitizerCommon-tsan-powerpc64le-Linux :: Linux/ptrace.cpp
+// https://lab.llvm.org/buildbot/#/builders/105
+// https://lab.llvm.org/buildbot/#/builders/121
+// https://lab.llvm.org/buildbot/#/builders/57
+# define CHECK_REAL_FUNC(func) \
+ if (REAL(func) == 0) { \
Report("FATAL: ThreadSanitizer: failed to intercept %s\n", #func); \
- Die(); \
- } \
- if (!thr->is_inited || thr->ignore_interceptors || thr->in_ignored_lib) \
- return REAL(func)(__VA_ARGS__); \
-/**/
+ Die(); \
+ }
+#else
+# define CHECK_REAL_FUNC(func) DCHECK(REAL(func))
+#endif
+
+#define SCOPED_TSAN_INTERCEPTOR(func, ...) \
+ SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \
+ CHECK_REAL_FUNC(func); \
+ if (MustIgnoreInterceptor(thr)) \
+ return REAL(func)(__VA_ARGS__);
#define SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START() \
si.DisableIgnores();
#define TSAN_INTERCEPTOR(ret, func, ...) INTERCEPTOR(ret, func, __VA_ARGS__)
+#if SANITIZER_FREEBSD
+# define TSAN_INTERCEPTOR_FREEBSD_ALIAS(ret, func, ...) \
+ TSAN_INTERCEPTOR(ret, _pthread_##func, __VA_ARGS__) \
+ ALIAS(WRAPPER_NAME(pthread_##func));
+#else
+# define TSAN_INTERCEPTOR_FREEBSD_ALIAS(ret, func, ...)
+#endif
+
#if SANITIZER_NETBSD
# define TSAN_INTERCEPTOR_NETBSD_ALIAS(ret, func, ...) \
TSAN_INTERCEPTOR(ret, __libc_##func, __VA_ARGS__) \
#include "BlocksRuntime/Block.h"
#include "tsan_dispatch_defs.h"
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
# include <Availability.h>
#endif
// dispatch_async_and_wait() and friends were introduced in macOS 10.14.
// Linking of these interceptors fails when using an older SDK.
-#if !SANITIZER_MAC || defined(__MAC_10_14)
+#if !SANITIZER_APPLE || defined(__MAC_10_14)
// macOS 10.14 is greater than our minimal deployment target. To ensure we
// generate a weak reference so the TSan dylib continues to work on older
// systems, we need to forward declare the intercepted functions as "weak
//===----------------------------------------------------------------------===//
#include "sanitizer_common/sanitizer_platform.h"
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
#include "interception/interception.h"
#include "tsan_interceptors.h"
if (h.created()) {
ThreadIgnoreBegin(thr, pc);
*h = (uptr) user_alloc(thr, pc, /*size=*/1);
- ThreadIgnoreEnd(thr, pc);
+ ThreadIgnoreEnd(thr);
}
return *h;
}
{
SCOPED_INTERCEPTOR_RAW(swapcontext, oucp, ucp);
}
- // Bacause of swapcontext() semantics we have no option but to copy its
- // impementation here
+ // Because of swapcontext() semantics we have no option but to copy its
+ // implementation here
if (!oucp || !ucp) {
errno = EINVAL;
return -1;
} // namespace __tsan
-#endif // SANITIZER_MAC
+#endif // SANITIZER_APPLE
using namespace __tsan;
-#if SANITIZER_FREEBSD || SANITIZER_MAC
+#if SANITIZER_FREEBSD || SANITIZER_APPLE
#define stdout __stdoutp
#define stderr __stderrp
#endif
#define PTHREAD_ABI_BASE "GLIBC_2.3.2"
#elif defined(__aarch64__) || SANITIZER_PPC64V2
#define PTHREAD_ABI_BASE "GLIBC_2.17"
+#elif SANITIZER_LOONGARCH64
+#define PTHREAD_ABI_BASE "GLIBC_2.36"
#endif
extern "C" int pthread_attr_init(void *attr);
DECLARE_REAL(int, fflush, __sanitizer_FILE *fp)
DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, uptr size)
DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr)
+extern "C" int pthread_equal(void *t1, void *t2);
extern "C" void *pthread_self();
extern "C" void _exit(int status);
#if !SANITIZER_NETBSD
extern "C" int fileno_unlocked(void *stream);
extern "C" int dirfd(void *dirp);
#endif
-#if SANITIZER_GLIBC
-extern "C" int mallopt(int param, int value);
-#endif
#if SANITIZER_NETBSD
extern __sanitizer_FILE __sF[];
#else
extern __sanitizer_FILE *stdout, *stderr;
#endif
-#if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD
+#if !SANITIZER_FREEBSD && !SANITIZER_APPLE && !SANITIZER_NETBSD
const int PTHREAD_MUTEX_RECURSIVE = 1;
const int PTHREAD_MUTEX_RECURSIVE_NP = 1;
#else
const int PTHREAD_MUTEX_RECURSIVE = 2;
const int PTHREAD_MUTEX_RECURSIVE_NP = 2;
#endif
-#if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD
+#if !SANITIZER_FREEBSD && !SANITIZER_APPLE && !SANITIZER_NETBSD
const int EPOLL_CTL_ADD = 1;
#endif
const int SIGILL = 4;
const int SIGSEGV = 11;
const int SIGPIPE = 13;
const int SIGTERM = 15;
-#if defined(__mips__) || SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD
+#if defined(__mips__) || SANITIZER_FREEBSD || SANITIZER_APPLE || SANITIZER_NETBSD
const int SIGBUS = 10;
const int SIGSYS = 12;
#else
const int SIGBUS = 7;
const int SIGSYS = 31;
#endif
+const int SI_TIMER = -2;
void *const MAP_FAILED = (void*)-1;
#if SANITIZER_NETBSD
const int PTHREAD_BARRIER_SERIAL_THREAD = 1234567;
-#elif !SANITIZER_MAC
+#elif !SANITIZER_APPLE
const int PTHREAD_BARRIER_SERIAL_THREAD = -1;
#endif
const int MAP_FIXED = 0x10;
# define F_TLOCK 2 /* Test and lock a region for exclusive use. */
# define F_TEST 3 /* Test a region for other processes locks. */
-#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD
+#if SANITIZER_FREEBSD || SANITIZER_APPLE || SANITIZER_NETBSD
const int SA_SIGINFO = 0x40;
const int SIG_SETMASK = 3;
#elif defined(__mips__)
#endif
#define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED \
- (cur_thread_init(), !cur_thread()->is_inited)
+ (!cur_thread_init()->is_inited)
namespace __tsan {
struct SignalDesc {
bool armed;
- bool sigaction;
__sanitizer_siginfo siginfo;
ucontext_t ctx;
};
struct ThreadSignalContext {
int int_signal_send;
- atomic_uintptr_t in_blocking_func;
- atomic_uintptr_t have_pending_signals;
SignalDesc pending_signals[kSigCount];
// emptyset and oldset are too big for stack.
__sanitizer_sigset_t emptyset;
__sanitizer_sigset_t oldset;
};
+void EnterBlockingFunc(ThreadState *thr) {
+ for (;;) {
+ // The order is important to not delay a signal infinitely if it's
+ // delivered right before we set in_blocking_func. Note: we can't call
+ // ProcessPendingSignals when in_blocking_func is set, or we can handle
+ // a signal synchronously when we are already handling a signal.
+ atomic_store(&thr->in_blocking_func, 1, memory_order_relaxed);
+ if (atomic_load(&thr->pending_signals, memory_order_relaxed) == 0)
+ break;
+ atomic_store(&thr->in_blocking_func, 0, memory_order_relaxed);
+ ProcessPendingSignals(thr);
+ }
+}
+
// The sole reason tsan wraps atexit callbacks is to establish synchronization
// between callback setup and callback execution.
struct AtExitCtx {
void (*f)();
void *arg;
+ uptr pc;
};
// InterceptorContext holds all global data required for interceptors.
// in a single cache line if possible (it's accessed in every interceptor).
ALIGNED(64) LibIgnore libignore;
__sanitizer_sigaction sigactions[kSigCount];
-#if !SANITIZER_MAC && !SANITIZER_NETBSD
+#if !SANITIZER_APPLE && !SANITIZER_NETBSD
unsigned finalize_key;
#endif
} // namespace __tsan
static ThreadSignalContext *SigCtx(ThreadState *thr) {
- ThreadSignalContext *ctx = (ThreadSignalContext*)thr->signal_ctx;
+ // This function may be called reentrantly if it is interrupted by a signal
+ // handler. Use CAS to handle the race.
+ uptr ctx = atomic_load(&thr->signal_ctx, memory_order_relaxed);
if (ctx == 0 && !thr->is_dead) {
- ctx = (ThreadSignalContext*)MmapOrDie(sizeof(*ctx), "ThreadSignalContext");
- MemoryResetRange(thr, (uptr)&SigCtx, (uptr)ctx, sizeof(*ctx));
- thr->signal_ctx = ctx;
+ uptr pctx =
+ (uptr)MmapOrDie(sizeof(ThreadSignalContext), "ThreadSignalContext");
+ MemoryResetRange(thr, (uptr)&SigCtx, pctx, sizeof(ThreadSignalContext));
+ if (atomic_compare_exchange_strong(&thr->signal_ctx, &ctx, pctx,
+ memory_order_relaxed)) {
+ ctx = pctx;
+ } else {
+ UnmapOrDie((ThreadSignalContext *)pctx, sizeof(ThreadSignalContext));
+ }
}
- return ctx;
+ return (ThreadSignalContext *)ctx;
}
ScopedInterceptor::ScopedInterceptor(ThreadState *thr, const char *fname,
uptr pc)
- : thr_(thr), pc_(pc), in_ignored_lib_(false), ignoring_(false) {
- Initialize(thr);
+ : thr_(thr) {
+ LazyInitialize(thr);
+ if (UNLIKELY(atomic_load(&thr->in_blocking_func, memory_order_relaxed))) {
+ // pthread_join is marked as blocking, but it's also known to call other
+ // intercepted functions (mmap, free). If we don't reset in_blocking_func
+ // we can get deadlocks and memory corruptions if we deliver a synchronous
+ // signal inside of an mmap/free interceptor.
+ // So reset it and restore it back in the destructor.
+ // See https://github.com/google/sanitizers/issues/1540
+ atomic_store(&thr->in_blocking_func, 0, memory_order_relaxed);
+ in_blocking_func_ = true;
+ }
if (!thr_->is_inited) return;
if (!thr_->ignore_interceptors) FuncEntry(thr, pc);
DPrintf("#%d: intercept %s()\n", thr_->tid, fname);
ScopedInterceptor::~ScopedInterceptor() {
if (!thr_->is_inited) return;
DisableIgnores();
+ if (UNLIKELY(in_blocking_func_))
+ EnterBlockingFunc(thr_);
if (!thr_->ignore_interceptors) {
ProcessPendingSignals(thr_);
FuncExit(thr_);
}
}
-void ScopedInterceptor::EnableIgnores() {
- if (ignoring_) {
- ThreadIgnoreBegin(thr_, pc_, /*save_stack=*/false);
- if (flags()->ignore_noninstrumented_modules) thr_->suppress_reports++;
- if (in_ignored_lib_) {
- DCHECK(!thr_->in_ignored_lib);
- thr_->in_ignored_lib = true;
- }
+NOINLINE
+void ScopedInterceptor::EnableIgnoresImpl() {
+ ThreadIgnoreBegin(thr_, 0);
+ if (flags()->ignore_noninstrumented_modules)
+ thr_->suppress_reports++;
+ if (in_ignored_lib_) {
+ DCHECK(!thr_->in_ignored_lib);
+ thr_->in_ignored_lib = true;
}
}
-void ScopedInterceptor::DisableIgnores() {
- if (ignoring_) {
- ThreadIgnoreEnd(thr_, pc_);
- if (flags()->ignore_noninstrumented_modules) thr_->suppress_reports--;
- if (in_ignored_lib_) {
- DCHECK(thr_->in_ignored_lib);
- thr_->in_ignored_lib = false;
- }
+NOINLINE
+void ScopedInterceptor::DisableIgnoresImpl() {
+ ThreadIgnoreEnd(thr_);
+ if (flags()->ignore_noninstrumented_modules)
+ thr_->suppress_reports--;
+ if (in_ignored_lib_) {
+ DCHECK(thr_->in_ignored_lib);
+ thr_->in_ignored_lib = false;
}
}
#define TSAN_INTERCEPT(func) INTERCEPT_FUNCTION(func)
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD
+# define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func)
+#else
+# define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION_VER(func, ver)
+#endif
#if SANITIZER_FREEBSD
-# define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func)
-# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func)
-# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func)
-#elif SANITIZER_NETBSD
-# define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func)
-# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func) \
- INTERCEPT_FUNCTION(__libc_##func)
-# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func) \
- INTERCEPT_FUNCTION(__libc_thr_##func)
+# define TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(func) \
+ INTERCEPT_FUNCTION(_pthread_##func)
#else
-# define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION_VER(func, ver)
-# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func)
-# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func)
+# define TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(func)
+#endif
+#if SANITIZER_NETBSD
+# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func) \
+ INTERCEPT_FUNCTION(__libc_##func)
+# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func) \
+ INTERCEPT_FUNCTION(__libc_thr_##func)
+#else
+# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func)
+# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func)
#endif
#define READ_STRING_OF_LEN(thr, pc, s, len, n) \
struct BlockingCall {
explicit BlockingCall(ThreadState *thr)
- : thr(thr)
- , ctx(SigCtx(thr)) {
- for (;;) {
- atomic_store(&ctx->in_blocking_func, 1, memory_order_relaxed);
- if (atomic_load(&ctx->have_pending_signals, memory_order_relaxed) == 0)
- break;
- atomic_store(&ctx->in_blocking_func, 0, memory_order_relaxed);
- ProcessPendingSignals(thr);
- }
+ : thr(thr) {
+ EnterBlockingFunc(thr);
// When we are in a "blocking call", we process signals asynchronously
// (right when they arrive). In this context we do not expect to be
// executing any user/runtime code. The known interceptor sequence when
~BlockingCall() {
thr->ignore_interceptors--;
- atomic_store(&ctx->in_blocking_func, 0, memory_order_relaxed);
+ atomic_store(&thr->in_blocking_func, 0, memory_order_relaxed);
}
ThreadState *thr;
- ThreadSignalContext *ctx;
};
TSAN_INTERCEPTOR(unsigned, sleep, unsigned sec) {
return BLOCK_REAL(pause)(fake);
}
-static void at_exit_wrapper() {
+// Note: we specifically call the function in such strange way
+// with "installed_at" because in reports it will appear between
+// callback frames and the frame that installed the callback.
+static void at_exit_callback_installed_at() {
AtExitCtx *ctx;
{
// Ensure thread-safety.
interceptor_ctx()->AtExitStack.PopBack();
}
- Acquire(cur_thread(), (uptr)0, (uptr)ctx);
+ ThreadState *thr = cur_thread();
+ Acquire(thr, ctx->pc, (uptr)ctx);
+ FuncEntry(thr, ctx->pc);
((void(*)())ctx->f)();
- InternalFree(ctx);
+ FuncExit(thr);
+ Free(ctx);
}
-static void cxa_at_exit_wrapper(void *arg) {
- Acquire(cur_thread(), 0, (uptr)arg);
+static void cxa_at_exit_callback_installed_at(void *arg) {
+ ThreadState *thr = cur_thread();
AtExitCtx *ctx = (AtExitCtx*)arg;
+ Acquire(thr, ctx->pc, (uptr)arg);
+ FuncEntry(thr, ctx->pc);
((void(*)(void *arg))ctx->f)(ctx->arg);
- InternalFree(ctx);
+ FuncExit(thr);
+ Free(ctx);
}
static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(),
// We want to setup the atexit callback even if we are in ignored lib
// or after fork.
SCOPED_INTERCEPTOR_RAW(atexit, f);
- return setup_at_exit_wrapper(thr, pc, (void(*)())f, 0, 0);
+ return setup_at_exit_wrapper(thr, GET_CALLER_PC(), (void (*)())f, 0, 0);
}
#endif
if (in_symbolizer())
return 0;
SCOPED_TSAN_INTERCEPTOR(__cxa_atexit, f, arg, dso);
- return setup_at_exit_wrapper(thr, pc, (void(*)())f, arg, dso);
+ return setup_at_exit_wrapper(thr, GET_CALLER_PC(), (void (*)())f, arg, dso);
}
static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(),
void *arg, void *dso) {
- AtExitCtx *ctx = (AtExitCtx*)InternalAlloc(sizeof(AtExitCtx));
+ auto *ctx = New<AtExitCtx>();
ctx->f = f;
ctx->arg = arg;
+ ctx->pc = pc;
Release(thr, pc, (uptr)ctx);
// Memory allocation in __cxa_atexit will race with free during exit,
// because we do not see synchronization around atexit callback list.
// due to atexit_mu held on exit from the calloc interceptor.
ScopedIgnoreInterceptors ignore;
- res = REAL(__cxa_atexit)((void (*)(void *a))at_exit_wrapper, 0, 0);
+ res = REAL(__cxa_atexit)((void (*)(void *a))at_exit_callback_installed_at,
+ 0, 0);
// Push AtExitCtx on the top of the stack of callback functions
if (!res) {
interceptor_ctx()->AtExitStack.PushBack(ctx);
}
} else {
- res = REAL(__cxa_atexit)(cxa_at_exit_wrapper, ctx, dso);
+ res = REAL(__cxa_atexit)(cxa_at_exit_callback_installed_at, ctx, dso);
}
- ThreadIgnoreEnd(thr, pc);
+ ThreadIgnoreEnd(thr);
return res;
}
-#if !SANITIZER_MAC && !SANITIZER_NETBSD
-static void on_exit_wrapper(int status, void *arg) {
+#if !SANITIZER_APPLE && !SANITIZER_NETBSD
+static void on_exit_callback_installed_at(int status, void *arg) {
ThreadState *thr = cur_thread();
- uptr pc = 0;
- Acquire(thr, pc, (uptr)arg);
AtExitCtx *ctx = (AtExitCtx*)arg;
+ Acquire(thr, ctx->pc, (uptr)arg);
+ FuncEntry(thr, ctx->pc);
((void(*)(int status, void *arg))ctx->f)(status, ctx->arg);
- InternalFree(ctx);
+ FuncExit(thr);
+ Free(ctx);
}
TSAN_INTERCEPTOR(int, on_exit, void(*f)(int, void*), void *arg) {
if (in_symbolizer())
return 0;
SCOPED_TSAN_INTERCEPTOR(on_exit, f, arg);
- AtExitCtx *ctx = (AtExitCtx*)InternalAlloc(sizeof(AtExitCtx));
+ auto *ctx = New<AtExitCtx>();
ctx->f = (void(*)())f;
ctx->arg = arg;
+ ctx->pc = GET_CALLER_PC();
Release(thr, pc, (uptr)ctx);
// Memory allocation in __cxa_atexit will race with free during exit,
// because we do not see synchronization around atexit callback list.
ThreadIgnoreBegin(thr, pc);
- int res = REAL(on_exit)(on_exit_wrapper, ctx);
- ThreadIgnoreEnd(thr, pc);
+ int res = REAL(on_exit)(on_exit_callback_installed_at, ctx);
+ ThreadIgnoreEnd(thr);
return res;
}
#define TSAN_MAYBE_INTERCEPT_ON_EXIT TSAN_INTERCEPT(on_exit)
buf->shadow_stack_pos = thr->shadow_stack_pos;
ThreadSignalContext *sctx = SigCtx(thr);
buf->int_signal_send = sctx ? sctx->int_signal_send : 0;
- buf->in_blocking_func = sctx ?
- atomic_load(&sctx->in_blocking_func, memory_order_relaxed) :
- false;
+ buf->in_blocking_func = atomic_load(&thr->in_blocking_func, memory_order_relaxed);
buf->in_signal_handler = atomic_load(&thr->in_signal_handler,
memory_order_relaxed);
}
while (thr->shadow_stack_pos > buf->shadow_stack_pos)
FuncExit(thr);
ThreadSignalContext *sctx = SigCtx(thr);
- if (sctx) {
+ if (sctx)
sctx->int_signal_send = buf->int_signal_send;
- atomic_store(&sctx->in_blocking_func, buf->in_blocking_func,
- memory_order_relaxed);
- }
+ atomic_store(&thr->in_blocking_func, buf->in_blocking_func,
+ memory_order_relaxed);
atomic_store(&thr->in_signal_handler, buf->in_signal_handler,
memory_order_relaxed);
JmpBufGarbageCollect(thr, buf->sp - 1); // do not collect buf->sp
}
// FIXME: put everything below into a common extern "C" block?
-extern "C" void __tsan_setjmp(uptr sp) {
- cur_thread_init();
- SetJmp(cur_thread(), sp);
-}
+extern "C" void __tsan_setjmp(uptr sp) { SetJmp(cur_thread_init(), sp); }
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
TSAN_INTERCEPTOR(int, setjmp, void *env);
TSAN_INTERCEPTOR(int, _setjmp, void *env);
TSAN_INTERCEPTOR(int, sigsetjmp, void *env);
-#else // SANITIZER_MAC
+#else // SANITIZER_APPLE
#if SANITIZER_NETBSD
#define setjmp_symname __setjmp14
#if !SANITIZER_NETBSD
DEFINE_REAL(int, __sigsetjmp, void *env)
#endif
-#endif // SANITIZER_MAC
+#endif // SANITIZER_APPLE
#if SANITIZER_NETBSD
#define longjmp_symname __longjmp14
}
#endif
-#if !SANITIZER_MAC
+#if !SANITIZER_APPLE
TSAN_INTERCEPTOR(void*, malloc, uptr size) {
if (in_symbolizer())
return InternalAlloc(size);
#define TSAN_MAYBE_INTERCEPT_MEMALIGN
#endif
-#if !SANITIZER_MAC
+#if !SANITIZER_APPLE
TSAN_INTERCEPTOR(void*, aligned_alloc, uptr align, uptr sz) {
if (in_symbolizer())
return InternalAlloc(sz, nullptr, align);
#define TSAN_MAYBE_INTERCEPT_PVALLOC
#endif
-#if !SANITIZER_MAC
+#if !SANITIZER_APPLE
TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) {
if (in_symbolizer()) {
void *p = InternalAlloc(sz, nullptr, align);
}
#endif
+// Both __cxa_guard_acquire and pthread_once 0-initialize
+// the object initially. pthread_once does not have any
+// other ABI requirements. __cxa_guard_acquire assumes
+// that any non-0 value in the first byte means that
+// initialization is completed. Contents of the remaining
+// bytes are up to us.
+constexpr u32 kGuardInit = 0;
+constexpr u32 kGuardDone = 1;
+constexpr u32 kGuardRunning = 1 << 16;
+constexpr u32 kGuardWaiter = 1 << 17;
+
+static int guard_acquire(ThreadState *thr, uptr pc, atomic_uint32_t *g,
+ bool blocking_hooks = true) {
+ if (blocking_hooks)
+ OnPotentiallyBlockingRegionBegin();
+ auto on_exit = at_scope_exit([blocking_hooks] {
+ if (blocking_hooks)
+ OnPotentiallyBlockingRegionEnd();
+ });
+
+ for (;;) {
+ u32 cmp = atomic_load(g, memory_order_acquire);
+ if (cmp == kGuardInit) {
+ if (atomic_compare_exchange_strong(g, &cmp, kGuardRunning,
+ memory_order_relaxed))
+ return 1;
+ } else if (cmp == kGuardDone) {
+ if (!thr->in_ignored_lib)
+ Acquire(thr, pc, (uptr)g);
+ return 0;
+ } else {
+ if ((cmp & kGuardWaiter) ||
+ atomic_compare_exchange_strong(g, &cmp, cmp | kGuardWaiter,
+ memory_order_relaxed))
+ FutexWait(g, cmp | kGuardWaiter);
+ }
+ }
+}
+
+static void guard_release(ThreadState *thr, uptr pc, atomic_uint32_t *g,
+ u32 v) {
+ if (!thr->in_ignored_lib)
+ Release(thr, pc, (uptr)g);
+ u32 old = atomic_exchange(g, v, memory_order_release);
+ if (old & kGuardWaiter)
+ FutexWake(g, 1 << 30);
+}
+
// __cxa_guard_acquire and friends need to be intercepted in a special way -
// regular interceptors will break statically-linked libstdc++. Linux
// interceptors are especially defined as weak functions (so that they don't
// these interceptors with INTERFACE_ATTRIBUTE.
// On OS X, we don't support statically linking, so we just use a regular
// interceptor.
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
#define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR
#else
#define STDCXX_INTERCEPTOR(rettype, name, ...) \
// Used in thread-safe function static initialization.
STDCXX_INTERCEPTOR(int, __cxa_guard_acquire, atomic_uint32_t *g) {
SCOPED_INTERCEPTOR_RAW(__cxa_guard_acquire, g);
- OnPotentiallyBlockingRegionBegin();
- auto on_exit = at_scope_exit(&OnPotentiallyBlockingRegionEnd);
- for (;;) {
- u32 cmp = atomic_load(g, memory_order_acquire);
- if (cmp == 0) {
- if (atomic_compare_exchange_strong(g, &cmp, 1<<16, memory_order_relaxed))
- return 1;
- } else if (cmp == 1) {
- Acquire(thr, pc, (uptr)g);
- return 0;
- } else {
- internal_sched_yield();
- }
- }
+ return guard_acquire(thr, pc, g);
}
STDCXX_INTERCEPTOR(void, __cxa_guard_release, atomic_uint32_t *g) {
SCOPED_INTERCEPTOR_RAW(__cxa_guard_release, g);
- Release(thr, pc, (uptr)g);
- atomic_store(g, 1, memory_order_release);
+ guard_release(thr, pc, g, kGuardDone);
}
STDCXX_INTERCEPTOR(void, __cxa_guard_abort, atomic_uint32_t *g) {
SCOPED_INTERCEPTOR_RAW(__cxa_guard_abort, g);
- atomic_store(g, 0, memory_order_relaxed);
+ guard_release(thr, pc, g, kGuardInit);
}
namespace __tsan {
}
void PlatformCleanUpThreadState(ThreadState *thr) {
- ThreadSignalContext *sctx = thr->signal_ctx;
+ ThreadSignalContext *sctx = (ThreadSignalContext *)atomic_load(
+ &thr->signal_ctx, memory_order_relaxed);
if (sctx) {
- thr->signal_ctx = 0;
+ atomic_store(&thr->signal_ctx, 0, memory_order_relaxed);
UnmapOrDie(sctx, sizeof(*sctx));
}
}
} // namespace __tsan
-#if !SANITIZER_MAC && !SANITIZER_NETBSD && !SANITIZER_FREEBSD
+#if !SANITIZER_APPLE && !SANITIZER_NETBSD && !SANITIZER_FREEBSD
static void thread_finalize(void *v) {
uptr iter = (uptr)v;
if (iter > 1) {
struct ThreadParam {
void* (*callback)(void *arg);
void *param;
- atomic_uintptr_t tid;
+ Tid tid;
+ Semaphore created;
+ Semaphore started;
};
extern "C" void *__tsan_thread_start_func(void *arg) {
ThreadParam *p = (ThreadParam*)arg;
void* (*callback)(void *arg) = p->callback;
void *param = p->param;
- int tid = 0;
{
- cur_thread_init();
- ThreadState *thr = cur_thread();
+ ThreadState *thr = cur_thread_init();
// Thread-local state is not initialized yet.
ScopedIgnoreInterceptors ignore;
-#if !SANITIZER_MAC && !SANITIZER_NETBSD && !SANITIZER_FREEBSD
+#if !SANITIZER_APPLE && !SANITIZER_NETBSD && !SANITIZER_FREEBSD
ThreadIgnoreBegin(thr, 0);
if (pthread_setspecific(interceptor_ctx()->finalize_key,
(void *)GetPthreadDestructorIterations())) {
Printf("ThreadSanitizer: failed to set thread key\n");
Die();
}
- ThreadIgnoreEnd(thr, 0);
+ ThreadIgnoreEnd(thr);
#endif
- while ((tid = atomic_load(&p->tid, memory_order_acquire)) == 0)
- internal_sched_yield();
+ p->created.Wait();
Processor *proc = ProcCreate();
ProcWire(proc, thr);
- ThreadStart(thr, tid, GetTid(), ThreadType::Regular);
- atomic_store(&p->tid, 0, memory_order_release);
+ ThreadStart(thr, p->tid, GetTid(), ThreadType::Regular);
+ p->started.Post();
}
void *res = callback(param);
// Prevent the callback from being tail called,
"fork is not supported. Dying (set die_after_fork=0 to override)\n");
Die();
} else {
- VPrintf(1, "ThreadSanitizer: starting new threads after multi-threaded "
- "fork is not supported (pid %d). Continuing because of "
- "die_after_fork=0, but you are on your own\n", internal_getpid());
+ VPrintf(1,
+ "ThreadSanitizer: starting new threads after multi-threaded "
+ "fork is not supported (pid %lu). Continuing because of "
+ "die_after_fork=0, but you are on your own\n",
+ internal_getpid());
}
}
__sanitizer_pthread_attr_t myattr;
ThreadParam p;
p.callback = callback;
p.param = param;
- atomic_store(&p.tid, 0, memory_order_relaxed);
+ p.tid = kMainTid;
int res = -1;
{
// Otherwise we see false positives in pthread stack manipulation.
ScopedIgnoreInterceptors ignore;
ThreadIgnoreBegin(thr, pc);
res = REAL(pthread_create)(th, attr, __tsan_thread_start_func, &p);
- ThreadIgnoreEnd(thr, pc);
+ ThreadIgnoreEnd(thr);
}
if (res == 0) {
- int tid = ThreadCreate(thr, pc, *(uptr*)th, IsStateDetached(detached));
- CHECK_NE(tid, 0);
+ p.tid = ThreadCreate(thr, pc, *(uptr *)th, IsStateDetached(detached));
+ CHECK_NE(p.tid, kMainTid);
// Synchronization on p.tid serves two purposes:
// 1. ThreadCreate must finish before the new thread starts.
// Otherwise the new thread can call pthread_detach, but the pthread_t
// 2. ThreadStart must finish before this thread continues.
// Otherwise, this thread can call pthread_detach and reset thr->sync
// before the new thread got a chance to acquire from it in ThreadStart.
- atomic_store(&p.tid, tid, memory_order_release);
- while (atomic_load(&p.tid, memory_order_acquire) != 0)
- internal_sched_yield();
+ p.created.Post();
+ p.started.Wait();
}
if (attr == &myattr)
pthread_attr_destroy(&myattr);
TSAN_INTERCEPTOR(int, pthread_join, void *th, void **ret) {
SCOPED_INTERCEPTOR_RAW(pthread_join, th, ret);
- int tid = ThreadConsumeTid(thr, pc, (uptr)th);
+ Tid tid = ThreadConsumeTid(thr, pc, (uptr)th);
ThreadIgnoreBegin(thr, pc);
int res = BLOCK_REAL(pthread_join)(th, ret);
- ThreadIgnoreEnd(thr, pc);
+ ThreadIgnoreEnd(thr);
if (res == 0) {
ThreadJoin(thr, pc, tid);
}
TSAN_INTERCEPTOR(int, pthread_detach, void *th) {
SCOPED_INTERCEPTOR_RAW(pthread_detach, th);
- int tid = ThreadConsumeTid(thr, pc, (uptr)th);
+ Tid tid = ThreadConsumeTid(thr, pc, (uptr)th);
int res = REAL(pthread_detach)(th);
if (res == 0) {
ThreadDetach(thr, pc, tid);
TSAN_INTERCEPTOR(void, pthread_exit, void *retval) {
{
SCOPED_INTERCEPTOR_RAW(pthread_exit, retval);
-#if !SANITIZER_MAC && !SANITIZER_ANDROID
+#if !SANITIZER_APPLE && !SANITIZER_ANDROID
CHECK_EQ(thr, &cur_thread_placeholder);
#endif
}
#if SANITIZER_LINUX
TSAN_INTERCEPTOR(int, pthread_tryjoin_np, void *th, void **ret) {
SCOPED_INTERCEPTOR_RAW(pthread_tryjoin_np, th, ret);
- int tid = ThreadConsumeTid(thr, pc, (uptr)th);
+ Tid tid = ThreadConsumeTid(thr, pc, (uptr)th);
ThreadIgnoreBegin(thr, pc);
int res = REAL(pthread_tryjoin_np)(th, ret);
- ThreadIgnoreEnd(thr, pc);
+ ThreadIgnoreEnd(thr);
if (res == 0)
ThreadJoin(thr, pc, tid);
else
TSAN_INTERCEPTOR(int, pthread_timedjoin_np, void *th, void **ret,
const struct timespec *abstime) {
SCOPED_INTERCEPTOR_RAW(pthread_timedjoin_np, th, ret, abstime);
- int tid = ThreadConsumeTid(thr, pc, (uptr)th);
+ Tid tid = ThreadConsumeTid(thr, pc, (uptr)th);
ThreadIgnoreBegin(thr, pc);
int res = BLOCK_REAL(pthread_timedjoin_np)(th, ret, abstime);
- ThreadIgnoreEnd(thr, pc);
+ ThreadIgnoreEnd(thr);
if (res == 0)
ThreadJoin(thr, pc, tid);
else
// tsan code. Also ScopedInterceptor and BlockingCall destructors won't run
// since the thread is cancelled, so we have to manually execute them
// (the thread still can run some user code due to pthread_cleanup_push).
- ThreadSignalContext *ctx = SigCtx(thr);
- CHECK_EQ(atomic_load(&ctx->in_blocking_func, memory_order_relaxed), 1);
- atomic_store(&ctx->in_blocking_func, 0, memory_order_relaxed);
+ CHECK_EQ(atomic_load(&thr->in_blocking_func, memory_order_relaxed), 1);
+ atomic_store(&thr->in_blocking_func, 0, memory_order_relaxed);
MutexPostLock(thr, pc, (uptr)m, MutexFlagDoPreLockOnPostLock);
// Undo BlockingCall ctor effects.
thr->ignore_interceptors--;
#define TSAN_MAYBE_PTHREAD_COND_CLOCKWAIT
#endif
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
INTERCEPTOR(int, pthread_cond_timedwait_relative_np, void *c, void *m,
void *reltime) {
void *cond = init_cond(c);
return res;
}
+TSAN_INTERCEPTOR(int, pthread_mutex_lock, void *m) {
+ SCOPED_TSAN_INTERCEPTOR(pthread_mutex_lock, m);
+ MutexPreLock(thr, pc, (uptr)m);
+ int res = REAL(pthread_mutex_lock)(m);
+ if (res == errno_EOWNERDEAD)
+ MutexRepair(thr, pc, (uptr)m);
+ if (res == 0 || res == errno_EOWNERDEAD)
+ MutexPostLock(thr, pc, (uptr)m);
+ if (res == errno_EINVAL)
+ MutexInvalidAccess(thr, pc, (uptr)m);
+ return res;
+}
+
TSAN_INTERCEPTOR(int, pthread_mutex_trylock, void *m) {
SCOPED_TSAN_INTERCEPTOR(pthread_mutex_trylock, m);
int res = REAL(pthread_mutex_trylock)(m);
return res;
}
-#if !SANITIZER_MAC
+#if !SANITIZER_APPLE
TSAN_INTERCEPTOR(int, pthread_mutex_timedlock, void *m, void *abstime) {
SCOPED_TSAN_INTERCEPTOR(pthread_mutex_timedlock, m, abstime);
int res = REAL(pthread_mutex_timedlock)(m, abstime);
}
#endif
-#if !SANITIZER_MAC
+TSAN_INTERCEPTOR(int, pthread_mutex_unlock, void *m) {
+ SCOPED_TSAN_INTERCEPTOR(pthread_mutex_unlock, m);
+ MutexUnlock(thr, pc, (uptr)m);
+ int res = REAL(pthread_mutex_unlock)(m);
+ if (res == errno_EINVAL)
+ MutexInvalidAccess(thr, pc, (uptr)m);
+ return res;
+}
+
+#if SANITIZER_GLIBC
+# if !__GLIBC_PREREQ(2, 34)
+// glibc 2.34 applies a non-default version for the two functions. They are no
+// longer expected to be intercepted by programs.
+TSAN_INTERCEPTOR(int, __pthread_mutex_lock, void *m) {
+ SCOPED_TSAN_INTERCEPTOR(__pthread_mutex_lock, m);
+ MutexPreLock(thr, pc, (uptr)m);
+ int res = REAL(__pthread_mutex_lock)(m);
+ if (res == errno_EOWNERDEAD)
+ MutexRepair(thr, pc, (uptr)m);
+ if (res == 0 || res == errno_EOWNERDEAD)
+ MutexPostLock(thr, pc, (uptr)m);
+ if (res == errno_EINVAL)
+ MutexInvalidAccess(thr, pc, (uptr)m);
+ return res;
+}
+
+TSAN_INTERCEPTOR(int, __pthread_mutex_unlock, void *m) {
+ SCOPED_TSAN_INTERCEPTOR(__pthread_mutex_unlock, m);
+ MutexUnlock(thr, pc, (uptr)m);
+ int res = REAL(__pthread_mutex_unlock)(m);
+ if (res == errno_EINVAL)
+ MutexInvalidAccess(thr, pc, (uptr)m);
+ return res;
+}
+# endif
+#endif
+
+#if !SANITIZER_APPLE
TSAN_INTERCEPTOR(int, pthread_spin_init, void *m, int pshared) {
SCOPED_TSAN_INTERCEPTOR(pthread_spin_init, m, pshared);
int res = REAL(pthread_spin_init)(m, pshared);
return res;
}
-#if !SANITIZER_MAC
+#if !SANITIZER_APPLE
TSAN_INTERCEPTOR(int, pthread_rwlock_timedrdlock, void *m, void *abstime) {
SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedrdlock, m, abstime);
int res = REAL(pthread_rwlock_timedrdlock)(m, abstime);
return res;
}
-#if !SANITIZER_MAC
+#if !SANITIZER_APPLE
TSAN_INTERCEPTOR(int, pthread_rwlock_timedwrlock, void *m, void *abstime) {
SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedwrlock, m, abstime);
int res = REAL(pthread_rwlock_timedwrlock)(m, abstime);
return res;
}
-#if !SANITIZER_MAC
+#if !SANITIZER_APPLE
TSAN_INTERCEPTOR(int, pthread_barrier_init, void *b, void *a, unsigned count) {
SCOPED_TSAN_INTERCEPTOR(pthread_barrier_init, b, a, count);
- MemoryWrite(thr, pc, (uptr)b, kSizeLog1);
+ MemoryAccess(thr, pc, (uptr)b, 1, kAccessWrite);
int res = REAL(pthread_barrier_init)(b, a, count);
return res;
}
TSAN_INTERCEPTOR(int, pthread_barrier_destroy, void *b) {
SCOPED_TSAN_INTERCEPTOR(pthread_barrier_destroy, b);
- MemoryWrite(thr, pc, (uptr)b, kSizeLog1);
+ MemoryAccess(thr, pc, (uptr)b, 1, kAccessWrite);
int res = REAL(pthread_barrier_destroy)(b);
return res;
}
TSAN_INTERCEPTOR(int, pthread_barrier_wait, void *b) {
SCOPED_TSAN_INTERCEPTOR(pthread_barrier_wait, b);
Release(thr, pc, (uptr)b);
- MemoryRead(thr, pc, (uptr)b, kSizeLog1);
+ MemoryAccess(thr, pc, (uptr)b, 1, kAccessRead);
int res = REAL(pthread_barrier_wait)(b);
- MemoryRead(thr, pc, (uptr)b, kSizeLog1);
+ MemoryAccess(thr, pc, (uptr)b, 1, kAccessRead);
if (res == 0 || res == PTHREAD_BARRIER_SERIAL_THREAD) {
Acquire(thr, pc, (uptr)b);
}
return errno_EINVAL;
atomic_uint32_t *a;
- if (SANITIZER_MAC)
+ if (SANITIZER_APPLE)
a = static_cast<atomic_uint32_t*>((void *)((char *)o + sizeof(long_t)));
else if (SANITIZER_NETBSD)
a = static_cast<atomic_uint32_t*>
else
a = static_cast<atomic_uint32_t*>(o);
- u32 v = atomic_load(a, memory_order_acquire);
- if (v == 0 && atomic_compare_exchange_strong(a, &v, 1,
- memory_order_relaxed)) {
+ // Mac OS X appears to use pthread_once() where calling BlockingRegion hooks
+ // result in crashes due to too little stack space.
+ if (guard_acquire(thr, pc, a, !SANITIZER_APPLE)) {
(*f)();
- if (!thr->in_ignored_lib)
- Release(thr, pc, (uptr)o);
- atomic_store(a, 2, memory_order_release);
- } else {
- while (v != 2) {
- internal_sched_yield();
- v = atomic_load(a, memory_order_acquire);
- }
- if (!thr->in_ignored_lib)
- Acquire(thr, pc, (uptr)o);
+ guard_release(thr, pc, a, kGuardDone);
}
return 0;
}
-#if SANITIZER_LINUX && !SANITIZER_ANDROID
+#if SANITIZER_GLIBC
TSAN_INTERCEPTOR(int, __fxstat, int version, int fd, void *buf) {
SCOPED_TSAN_INTERCEPTOR(__fxstat, version, fd, buf);
if (fd > 0)
#endif
TSAN_INTERCEPTOR(int, fstat, int fd, void *buf) {
-#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_ANDROID || SANITIZER_NETBSD
- SCOPED_TSAN_INTERCEPTOR(fstat, fd, buf);
+#if SANITIZER_GLIBC
+ SCOPED_TSAN_INTERCEPTOR(__fxstat, 0, fd, buf);
if (fd > 0)
FdAccess(thr, pc, fd);
- return REAL(fstat)(fd, buf);
+ return REAL(__fxstat)(0, fd, buf);
#else
- SCOPED_TSAN_INTERCEPTOR(__fxstat, 0, fd, buf);
+ SCOPED_TSAN_INTERCEPTOR(fstat, fd, buf);
if (fd > 0)
FdAccess(thr, pc, fd);
- return REAL(__fxstat)(0, fd, buf);
+ return REAL(fstat)(fd, buf);
#endif
}
-#if SANITIZER_LINUX && !SANITIZER_ANDROID
+#if SANITIZER_GLIBC
TSAN_INTERCEPTOR(int, __fxstat64, int version, int fd, void *buf) {
SCOPED_TSAN_INTERCEPTOR(__fxstat64, version, fd, buf);
if (fd > 0)
#define TSAN_MAYBE_INTERCEPT___FXSTAT64
#endif
-#if SANITIZER_LINUX && !SANITIZER_ANDROID
+#if SANITIZER_GLIBC
TSAN_INTERCEPTOR(int, fstat64, int fd, void *buf) {
SCOPED_TSAN_INTERCEPTOR(__fxstat64, 0, fd, buf);
if (fd > 0)
return newfd2;
}
-#if !SANITIZER_MAC
+#if !SANITIZER_APPLE
TSAN_INTERCEPTOR(int, dup3, int oldfd, int newfd, int flags) {
SCOPED_TSAN_INTERCEPTOR(dup3, oldfd, newfd, flags);
int newfd2 = REAL(dup3)(oldfd, newfd, flags);
#if SANITIZER_LINUX
TSAN_INTERCEPTOR(int, signalfd, int fd, void *mask, int flags) {
- SCOPED_TSAN_INTERCEPTOR(signalfd, fd, mask, flags);
- if (fd >= 0)
- FdClose(thr, pc, fd);
+ SCOPED_INTERCEPTOR_RAW(signalfd, fd, mask, flags);
+ FdClose(thr, pc, fd);
fd = REAL(signalfd)(fd, mask, flags);
- if (fd >= 0)
+ if (!MustIgnoreInterceptor(thr))
FdSignalCreate(thr, pc, fd);
return fd;
}
}
TSAN_INTERCEPTOR(int, close, int fd) {
- SCOPED_TSAN_INTERCEPTOR(close, fd);
- if (fd >= 0)
+ SCOPED_INTERCEPTOR_RAW(close, fd);
+ if (!in_symbolizer())
FdClose(thr, pc, fd);
return REAL(close)(fd);
}
#if SANITIZER_LINUX
TSAN_INTERCEPTOR(int, __close, int fd) {
- SCOPED_TSAN_INTERCEPTOR(__close, fd);
- if (fd >= 0)
- FdClose(thr, pc, fd);
+ SCOPED_INTERCEPTOR_RAW(__close, fd);
+ FdClose(thr, pc, fd);
return REAL(__close)(fd);
}
#define TSAN_MAYBE_INTERCEPT___CLOSE TSAN_INTERCEPT(__close)
// glibc guts
#if SANITIZER_LINUX && !SANITIZER_ANDROID
TSAN_INTERCEPTOR(void, __res_iclose, void *state, bool free_addr) {
- SCOPED_TSAN_INTERCEPTOR(__res_iclose, state, free_addr);
+ SCOPED_INTERCEPTOR_RAW(__res_iclose, state, free_addr);
int fds[64];
int cnt = ExtractResolvFDs(state, fds, ARRAY_SIZE(fds));
- for (int i = 0; i < cnt; i++) {
- if (fds[i] > 0)
- FdClose(thr, pc, fds[i]);
- }
+ for (int i = 0; i < cnt; i++) FdClose(thr, pc, fds[i]);
REAL(__res_iclose)(state, free_addr);
}
#define TSAN_MAYBE_INTERCEPT___RES_ICLOSE TSAN_INTERCEPT(__res_iclose)
return res;
}
-#if !SANITIZER_MAC
+#if !SANITIZER_APPLE
TSAN_INTERCEPTOR(int, pipe2, int *pipefd, int flags) {
SCOPED_TSAN_INTERCEPTOR(pipe2, pipefd, flags);
int res = REAL(pipe2)(pipefd, flags);
}
TSAN_INTERCEPTOR(int, closedir, void *dirp) {
- SCOPED_TSAN_INTERCEPTOR(closedir, dirp);
+ SCOPED_INTERCEPTOR_RAW(closedir, dirp);
if (dirp) {
int fd = dirfd(dirp);
FdClose(thr, pc, fd);
FdAccess(thr, pc, epfd);
if (epfd >= 0 && fd >= 0)
FdAccess(thr, pc, fd);
- if (op == EPOLL_CTL_ADD && epfd >= 0)
+ if (op == EPOLL_CTL_ADD && epfd >= 0) {
+ FdPollAdd(thr, pc, epfd, fd);
FdRelease(thr, pc, epfd);
+ }
int res = REAL(epoll_ctl)(epfd, op, fd, ev);
return res;
}
return res;
}
-#define TSAN_MAYBE_INTERCEPT_EPOLL \
- TSAN_INTERCEPT(epoll_create); \
- TSAN_INTERCEPT(epoll_create1); \
- TSAN_INTERCEPT(epoll_ctl); \
- TSAN_INTERCEPT(epoll_wait); \
- TSAN_INTERCEPT(epoll_pwait)
+TSAN_INTERCEPTOR(int, epoll_pwait2, int epfd, void *ev, int cnt, void *timeout,
+ void *sigmask) {
+ SCOPED_INTERCEPTOR_RAW(epoll_pwait2, epfd, ev, cnt, timeout, sigmask);
+ // This function is new and may not be present in libc and/or kernel.
+ // Since we effectively add it to libc (as will be probed by the program
+ // using dlsym or a weak function pointer) we need to handle the case
+ // when it's not present in the actual libc.
+ if (!REAL(epoll_pwait2)) {
+ errno = errno_ENOSYS;
+ return -1;
+ }
+ if (MustIgnoreInterceptor(thr))
+ REAL(epoll_pwait2)(epfd, ev, cnt, timeout, sigmask);
+ if (epfd >= 0)
+ FdAccess(thr, pc, epfd);
+ int res = BLOCK_REAL(epoll_pwait2)(epfd, ev, cnt, timeout, sigmask);
+ if (res > 0 && epfd >= 0)
+ FdAcquire(thr, pc, epfd);
+ return res;
+}
+
+# define TSAN_MAYBE_INTERCEPT_EPOLL \
+ TSAN_INTERCEPT(epoll_create); \
+ TSAN_INTERCEPT(epoll_create1); \
+ TSAN_INTERCEPT(epoll_ctl); \
+ TSAN_INTERCEPT(epoll_wait); \
+ TSAN_INTERCEPT(epoll_pwait); \
+ TSAN_INTERCEPT(epoll_pwait2)
#else
#define TSAN_MAYBE_INTERCEPT_EPOLL
#endif
namespace __tsan {
+static void ReportErrnoSpoiling(ThreadState *thr, uptr pc, int sig) {
+ VarSizeStackTrace stack;
+ // StackTrace::GetNestInstructionPc(pc) is used because return address is
+ // expected, OutputReport() will undo this.
+ ObtainCurrentStack(thr, StackTrace::GetNextInstructionPc(pc), &stack);
+ ThreadRegistryLock l(&ctx->thread_registry);
+ ScopedReport rep(ReportTypeErrnoInSignal);
+ rep.SetSigNum(sig);
+ if (!IsFiredSuppression(ctx, ReportTypeErrnoInSignal, stack)) {
+ rep.AddStack(stack, true);
+ OutputReport(thr, rep);
+ }
+}
+
static void CallUserSignalHandler(ThreadState *thr, bool sync, bool acquire,
- bool sigact, int sig,
- __sanitizer_siginfo *info, void *uctx) {
+ int sig, __sanitizer_siginfo *info,
+ void *uctx) {
+ CHECK(thr->slot);
__sanitizer_sigaction *sigactions = interceptor_ctx()->sigactions;
if (acquire)
Acquire(thr, 0, (uptr)&sigactions[sig]);
// Signals are generally asynchronous, so if we receive a signals when
// ignores are enabled we should disable ignores. This is critical for sync
- // and interceptors, because otherwise we can miss syncronization and report
+ // and interceptors, because otherwise we can miss synchronization and report
// false races.
int ignore_reads_and_writes = thr->ignore_reads_and_writes;
int ignore_interceptors = thr->ignore_interceptors;
int ignore_sync = thr->ignore_sync;
+ // For symbolizer we only process SIGSEGVs synchronously
+ // (bug in symbolizer or in tsan). But we want to reset
+ // in_symbolizer to fail gracefully. Symbolizer and user code
+ // use different memory allocators, so if we don't reset
+ // in_symbolizer we can get memory allocated with one being
+ // feed with another, which can cause more crashes.
+ int in_symbolizer = thr->in_symbolizer;
if (!ctx->after_multithreaded_fork) {
thr->ignore_reads_and_writes = 0;
thr->fast_state.ClearIgnoreBit();
thr->ignore_interceptors = 0;
thr->ignore_sync = 0;
+ thr->in_symbolizer = 0;
}
// Ensure that the handler does not spoil errno.
const int saved_errno = errno;
// This code races with sigaction. Be careful to not read sa_sigaction twice.
// Also need to remember pc for reporting before the call,
// because the handler can reset it.
- volatile uptr pc =
- sigact ? (uptr)sigactions[sig].sigaction : (uptr)sigactions[sig].handler;
+ volatile uptr pc = (sigactions[sig].sa_flags & SA_SIGINFO)
+ ? (uptr)sigactions[sig].sigaction
+ : (uptr)sigactions[sig].handler;
if (pc != sig_dfl && pc != sig_ign) {
- if (sigact)
- ((__sanitizer_sigactionhandler_ptr)pc)(sig, info, uctx);
- else
- ((__sanitizer_sighandler_ptr)pc)(sig);
+ // The callback can be either sa_handler or sa_sigaction.
+ // They have different signatures, but we assume that passing
+ // additional arguments to sa_handler works and is harmless.
+ ((__sanitizer_sigactionhandler_ptr)pc)(sig, info, uctx);
}
if (!ctx->after_multithreaded_fork) {
thr->ignore_reads_and_writes = ignore_reads_and_writes;
thr->fast_state.SetIgnoreBit();
thr->ignore_interceptors = ignore_interceptors;
thr->ignore_sync = ignore_sync;
+ thr->in_symbolizer = in_symbolizer;
}
// We do not detect errno spoiling for SIGTERM,
// because some SIGTERM handlers do spoil errno but reraise SIGTERM,
// from rtl_generic_sighandler) we have not yet received the reraised
// signal; and it looks too fragile to intercept all ways to reraise a signal.
if (ShouldReport(thr, ReportTypeErrnoInSignal) && !sync && sig != SIGTERM &&
- errno != 99) {
- VarSizeStackTrace stack;
- // StackTrace::GetNestInstructionPc(pc) is used because return address is
- // expected, OutputReport() will undo this.
- ObtainCurrentStack(thr, StackTrace::GetNextInstructionPc(pc), &stack);
- ThreadRegistryLock l(ctx->thread_registry);
- ScopedReport rep(ReportTypeErrnoInSignal);
- if (!IsFiredSuppression(ctx, ReportTypeErrnoInSignal, stack)) {
- rep.AddStack(stack, true);
- OutputReport(thr, rep);
- }
- }
+ errno != 99)
+ ReportErrnoSpoiling(thr, pc, sig);
errno = saved_errno;
}
-void ProcessPendingSignals(ThreadState *thr) {
+void ProcessPendingSignalsImpl(ThreadState *thr) {
+ atomic_store(&thr->pending_signals, 0, memory_order_relaxed);
ThreadSignalContext *sctx = SigCtx(thr);
- if (sctx == 0 ||
- atomic_load(&sctx->have_pending_signals, memory_order_relaxed) == 0)
+ if (sctx == 0)
return;
- atomic_store(&sctx->have_pending_signals, 0, memory_order_relaxed);
atomic_fetch_add(&thr->in_signal_handler, 1, memory_order_relaxed);
internal_sigfillset(&sctx->emptyset);
int res = REAL(pthread_sigmask)(SIG_SETMASK, &sctx->emptyset, &sctx->oldset);
SignalDesc *signal = &sctx->pending_signals[sig];
if (signal->armed) {
signal->armed = false;
- CallUserSignalHandler(thr, false, true, signal->sigaction, sig,
- &signal->siginfo, &signal->ctx);
+ CallUserSignalHandler(thr, false, true, sig, &signal->siginfo,
+ &signal->ctx);
}
}
res = REAL(pthread_sigmask)(SIG_SETMASK, &sctx->oldset, 0);
} // namespace __tsan
-static bool is_sync_signal(ThreadSignalContext *sctx, int sig) {
+static bool is_sync_signal(ThreadSignalContext *sctx, int sig,
+ __sanitizer_siginfo *info) {
+ // If we are sending signal to ourselves, we must process it now.
+ if (sctx && sig == sctx->int_signal_send)
+ return true;
+#if SANITIZER_HAS_SIGINFO
+ // POSIX timers can be configured to send any kind of signal; however, it
+ // doesn't make any sense to consider a timer signal as synchronous!
+ if (info->si_code == SI_TIMER)
+ return false;
+#endif
return sig == SIGSEGV || sig == SIGBUS || sig == SIGILL || sig == SIGTRAP ||
- sig == SIGABRT || sig == SIGFPE || sig == SIGPIPE || sig == SIGSYS ||
- // If we are sending signal to ourselves, we must process it now.
- (sctx && sig == sctx->int_signal_send);
+ sig == SIGABRT || sig == SIGFPE || sig == SIGPIPE || sig == SIGSYS;
}
-void ALWAYS_INLINE rtl_generic_sighandler(bool sigact, int sig,
- __sanitizer_siginfo *info,
- void *ctx) {
- cur_thread_init();
- ThreadState *thr = cur_thread();
+void sighandler(int sig, __sanitizer_siginfo *info, void *ctx) {
+ ThreadState *thr = cur_thread_init();
ThreadSignalContext *sctx = SigCtx(thr);
if (sig < 0 || sig >= kSigCount) {
VPrintf(1, "ThreadSanitizer: ignoring signal %d\n", sig);
return;
}
// Don't mess with synchronous signals.
- const bool sync = is_sync_signal(sctx, sig);
+ const bool sync = is_sync_signal(sctx, sig, info);
if (sync ||
// If we are in blocking function, we can safely process it now
// (but check if we are in a recursive interceptor,
// i.e. pthread_join()->munmap()).
- (sctx && atomic_load(&sctx->in_blocking_func, memory_order_relaxed))) {
+ atomic_load(&thr->in_blocking_func, memory_order_relaxed)) {
atomic_fetch_add(&thr->in_signal_handler, 1, memory_order_relaxed);
- if (sctx && atomic_load(&sctx->in_blocking_func, memory_order_relaxed)) {
- atomic_store(&sctx->in_blocking_func, 0, memory_order_relaxed);
- CallUserSignalHandler(thr, sync, true, sigact, sig, info, ctx);
- atomic_store(&sctx->in_blocking_func, 1, memory_order_relaxed);
+ if (atomic_load(&thr->in_blocking_func, memory_order_relaxed)) {
+ atomic_store(&thr->in_blocking_func, 0, memory_order_relaxed);
+ CallUserSignalHandler(thr, sync, true, sig, info, ctx);
+ atomic_store(&thr->in_blocking_func, 1, memory_order_relaxed);
} else {
// Be very conservative with when we do acquire in this case.
// It's unsafe to do acquire in async handlers, because ThreadState
// SIGSYS looks relatively safe -- it's synchronous and can actually
// need some global state.
bool acq = (sig == SIGSYS);
- CallUserSignalHandler(thr, sync, acq, sigact, sig, info, ctx);
+ CallUserSignalHandler(thr, sync, acq, sig, info, ctx);
}
atomic_fetch_add(&thr->in_signal_handler, -1, memory_order_relaxed);
return;
SignalDesc *signal = &sctx->pending_signals[sig];
if (signal->armed == false) {
signal->armed = true;
- signal->sigaction = sigact;
- if (info)
- internal_memcpy(&signal->siginfo, info, sizeof(*info));
- if (ctx)
- internal_memcpy(&signal->ctx, ctx, sizeof(signal->ctx));
- atomic_store(&sctx->have_pending_signals, 1, memory_order_relaxed);
+ internal_memcpy(&signal->siginfo, info, sizeof(*info));
+ internal_memcpy(&signal->ctx, ctx, sizeof(signal->ctx));
+ atomic_store(&thr->pending_signals, 1, memory_order_relaxed);
}
}
-static void rtl_sighandler(int sig) {
- rtl_generic_sighandler(false, sig, 0, 0);
-}
-
-static void rtl_sigaction(int sig, __sanitizer_siginfo *info, void *ctx) {
- rtl_generic_sighandler(true, sig, info, ctx);
-}
-
TSAN_INTERCEPTOR(int, raise, int sig) {
SCOPED_TSAN_INTERCEPTOR(raise, sig);
ThreadSignalContext *sctx = SigCtx(thr);
ThreadSignalContext *sctx = SigCtx(thr);
CHECK_NE(sctx, 0);
int prev = sctx->int_signal_send;
- if (tid == pthread_self()) {
+ bool self = pthread_equal(tid, pthread_self());
+ if (self)
sctx->int_signal_send = sig;
- }
int res = REAL(pthread_kill)(tid, sig);
- if (tid == pthread_self()) {
+ if (self) {
CHECK_EQ(sctx->int_signal_send, sig);
sctx->int_signal_send = prev;
}
// inside of getaddrinfo. So ignore memory accesses.
ThreadIgnoreBegin(thr, pc);
int res = REAL(getaddrinfo)(node, service, hints, rv);
- ThreadIgnoreEnd(thr, pc);
+ ThreadIgnoreEnd(thr);
return res;
}
return;
ThreadState *thr = cur_thread();
const uptr pc = StackTrace::GetCurrentPc();
- ForkChildAfter(thr, pc);
+ ForkChildAfter(thr, pc, true);
FdOnFork(thr, pc);
}
+#if !SANITIZER_IOS
TSAN_INTERCEPTOR(int, vfork, int fake) {
// Some programs (e.g. openjdk) call close for all file descriptors
// in the child process. Under tsan it leads to false positives, because
// Instead we simply turn vfork into fork.
return WRAP(fork)(fake);
}
+#endif
-#if !SANITIZER_MAC && !SANITIZER_ANDROID
+#if SANITIZER_LINUX
+TSAN_INTERCEPTOR(int, clone, int (*fn)(void *), void *stack, int flags,
+ void *arg, int *parent_tid, void *tls, pid_t *child_tid) {
+ SCOPED_INTERCEPTOR_RAW(clone, fn, stack, flags, arg, parent_tid, tls,
+ child_tid);
+ struct Arg {
+ int (*fn)(void *);
+ void *arg;
+ };
+ auto wrapper = +[](void *p) -> int {
+ auto *thr = cur_thread();
+ uptr pc = GET_CURRENT_PC();
+ // Start the background thread for fork, but not for clone.
+ // For fork we did this always and it's known to work (or user code has
+ // adopted). But if we do this for the new clone interceptor some code
+ // (sandbox2) fails. So model we used to do for years and don't start the
+ // background thread after clone.
+ ForkChildAfter(thr, pc, false);
+ FdOnFork(thr, pc);
+ auto *arg = static_cast<Arg *>(p);
+ return arg->fn(arg->arg);
+ };
+ ForkBefore(thr, pc);
+ Arg arg_wrapper = {fn, arg};
+ int pid = REAL(clone)(wrapper, stack, flags, &arg_wrapper, parent_tid, tls,
+ child_tid);
+ ForkParentAfter(thr, pc);
+ return pid;
+}
+#endif
+
+#if !SANITIZER_APPLE && !SANITIZER_ANDROID
typedef int (*dl_iterate_phdr_cb_t)(__sanitizer_dl_phdr_info *info, SIZE_T size,
void *data);
struct dl_iterate_phdr_data {
};
static bool IsAppNotRodata(uptr addr) {
- return IsAppMem(addr) && *(u64*)MemToShadow(addr) != kShadowRodata;
+ return IsAppMem(addr) && *MemToShadow(addr) != Shadow::kRodata;
}
static int dl_iterate_phdr_cb(__sanitizer_dl_phdr_info *info, SIZE_T size,
struct TsanInterceptorContext {
ThreadState *thr;
- const uptr caller_pc;
const uptr pc;
};
-#if !SANITIZER_MAC
+#if !SANITIZER_APPLE
static void HandleRecvmsg(ThreadState *thr, uptr pc,
__sanitizer_msghdr *msg) {
int fds[64];
((TsanInterceptorContext *) ctx)->pc, (uptr) ptr, size, \
false)
-#define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \
- SCOPED_TSAN_INTERCEPTOR(func, __VA_ARGS__); \
- TsanInterceptorContext _ctx = {thr, caller_pc, pc}; \
- ctx = (void *)&_ctx; \
- (void) ctx;
+#define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \
+ SCOPED_TSAN_INTERCEPTOR(func, __VA_ARGS__); \
+ TsanInterceptorContext _ctx = {thr, pc}; \
+ ctx = (void *)&_ctx; \
+ (void)ctx;
#define COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, func, ...) \
SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \
- TsanInterceptorContext _ctx = {thr, caller_pc, pc}; \
+ TsanInterceptorContext _ctx = {thr, pc}; \
ctx = (void *)&_ctx; \
- (void) ctx;
+ (void)ctx;
#define COMMON_INTERCEPTOR_FILE_OPEN(ctx, file, path) \
if (path) \
#define COMMON_INTERCEPTOR_FILE_CLOSE(ctx, file) \
if (file) { \
int fd = fileno_unlocked(file); \
- if (fd >= 0) FdClose(thr, pc, fd); \
+ FdClose(thr, pc, fd); \
}
+#define COMMON_INTERCEPTOR_DLOPEN(filename, flag) \
+ ({ \
+ CheckNoDeepBind(filename, flag); \
+ ThreadIgnoreBegin(thr, 0); \
+ void *res = REAL(dlopen)(filename, flag); \
+ ThreadIgnoreEnd(thr); \
+ res; \
+ })
+
#define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, handle) \
libignore()->OnLibraryLoaded(filename)
#define COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name) \
ThreadSetName(((TsanInterceptorContext *) ctx)->thr, name)
-#define COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name) \
- __tsan::ctx->thread_registry->SetThreadNameByUserId(thread, name)
+#define COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name) \
+ if (pthread_equal(pthread_self(), reinterpret_cast<void *>(thread))) \
+ COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name); \
+ else \
+ __tsan::ctx->thread_registry.SetThreadNameByUserId(thread, name)
#define COMMON_INTERCEPTOR_BLOCK_REAL(name) BLOCK_REAL(name)
#define COMMON_INTERCEPTOR_ON_EXIT(ctx) \
OnExit(((TsanInterceptorContext *) ctx)->thr)
-#define COMMON_INTERCEPTOR_MUTEX_PRE_LOCK(ctx, m) \
- MutexPreLock(((TsanInterceptorContext *)ctx)->thr, \
- ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
-
-#define COMMON_INTERCEPTOR_MUTEX_POST_LOCK(ctx, m) \
- MutexPostLock(((TsanInterceptorContext *)ctx)->thr, \
- ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
-
-#define COMMON_INTERCEPTOR_MUTEX_UNLOCK(ctx, m) \
- MutexUnlock(((TsanInterceptorContext *)ctx)->thr, \
- ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
-
-#define COMMON_INTERCEPTOR_MUTEX_REPAIR(ctx, m) \
- MutexRepair(((TsanInterceptorContext *)ctx)->thr, \
- ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
-
-#define COMMON_INTERCEPTOR_MUTEX_INVALID(ctx, m) \
- MutexInvalidAccess(((TsanInterceptorContext *)ctx)->thr, \
- ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
-
#define COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, sz, prot, flags, fd, \
off) \
do { \
off); \
} while (false)
-#if !SANITIZER_MAC
+#if !SANITIZER_APPLE
#define COMMON_INTERCEPTOR_HANDLE_RECVMSG(ctx, msg) \
HandleRecvmsg(((TsanInterceptorContext *)ctx)->thr, \
((TsanInterceptorContext *)ctx)->pc, msg)
int sigaction_impl(int sig, const __sanitizer_sigaction *act,
__sanitizer_sigaction *old) {
// Note: if we call REAL(sigaction) directly for any reason without proxying
- // the signal handler through rtl_sigaction, very bad things will happen.
+ // the signal handler through sighandler, very bad things will happen.
// The handler will run synchronously and corrupt tsan per-thread state.
SCOPED_INTERCEPTOR_RAW(sigaction, sig, act, old);
if (sig <= 0 || sig >= kSigCount) {
sigactions[sig].sa_flags = *(volatile int const *)&act->sa_flags;
internal_memcpy(&sigactions[sig].sa_mask, &act->sa_mask,
sizeof(sigactions[sig].sa_mask));
-#if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD
+#if !SANITIZER_FREEBSD && !SANITIZER_APPLE && !SANITIZER_NETBSD
sigactions[sig].sa_restorer = act->sa_restorer;
#endif
internal_memcpy(&newact, act, sizeof(newact));
internal_sigfillset(&newact.sa_mask);
- if ((uptr)act->handler != sig_ign && (uptr)act->handler != sig_dfl) {
- if (newact.sa_flags & SA_SIGINFO)
- newact.sigaction = rtl_sigaction;
- else
- newact.handler = rtl_sighandler;
+ if ((act->sa_flags & SA_SIGINFO) ||
+ ((uptr)act->handler != sig_ign && (uptr)act->handler != sig_dfl)) {
+ newact.sa_flags |= SA_SIGINFO;
+ newact.sigaction = sighandler;
}
ReleaseStore(thr, pc, (uptr)&sigactions[sig]);
act = &newact;
}
int res = REAL(sigaction)(sig, act, old);
- if (res == 0 && old) {
- uptr cb = (uptr)old->sigaction;
- if (cb == (uptr)rtl_sigaction || cb == (uptr)rtl_sighandler) {
- internal_memcpy(old, &old_stored, sizeof(*old));
- }
- }
+ if (res == 0 && old && old->sigaction == sighandler)
+ internal_memcpy(old, &old_stored, sizeof(*old));
return res;
}
return old.handler;
}
-#define TSAN_SYSCALL() \
+#define TSAN_SYSCALL() \
ThreadState *thr = cur_thread(); \
- if (thr->ignore_interceptors) \
- return; \
- ScopedSyscall scoped_syscall(thr) \
-/**/
+ if (thr->ignore_interceptors) \
+ return; \
+ ScopedSyscall scoped_syscall(thr)
struct ScopedSyscall {
ThreadState *thr;
- explicit ScopedSyscall(ThreadState *thr)
- : thr(thr) {
- Initialize(thr);
- }
+ explicit ScopedSyscall(ThreadState *thr) : thr(thr) { LazyInitialize(thr); }
~ScopedSyscall() {
ProcessPendingSignals(thr);
}
};
-#if !SANITIZER_FREEBSD && !SANITIZER_MAC
+#if !SANITIZER_FREEBSD && !SANITIZER_APPLE
static void syscall_access_range(uptr pc, uptr p, uptr s, bool write) {
TSAN_SYSCALL();
MemoryAccessRange(thr, pc, p, s, write);
static USED void syscall_acquire(uptr pc, uptr addr) {
TSAN_SYSCALL();
Acquire(thr, pc, addr);
- DPrintf("syscall_acquire(%p)\n", addr);
+ DPrintf("syscall_acquire(0x%zx))\n", addr);
}
static USED void syscall_release(uptr pc, uptr addr) {
TSAN_SYSCALL();
- DPrintf("syscall_release(%p)\n", addr);
+ DPrintf("syscall_release(0x%zx)\n", addr);
Release(thr, pc, addr);
}
static void syscall_fd_close(uptr pc, int fd) {
- TSAN_SYSCALL();
+ auto *thr = cur_thread();
FdClose(thr, pc, fd);
}
static USED void syscall_fd_acquire(uptr pc, int fd) {
TSAN_SYSCALL();
FdAcquire(thr, pc, fd);
- DPrintf("syscall_fd_acquire(%p)\n", fd);
+ DPrintf("syscall_fd_acquire(%d)\n", fd);
}
static USED void syscall_fd_release(uptr pc, int fd) {
TSAN_SYSCALL();
- DPrintf("syscall_fd_release(%p)\n", fd);
+ DPrintf("syscall_fd_release(%d)\n", fd);
FdRelease(thr, pc, fd);
}
ThreadState *thr = cur_thread();
if (pid == 0) {
// child
- ForkChildAfter(thr, pc);
+ ForkChildAfter(thr, pc, true);
FdOnFork(thr, pc);
} else if (pid > 0) {
// parent
#define TSAN_MAYBE_INTERCEPT_THR_EXIT
#endif
+TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, cond_init, void *c, void *a)
+TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, cond_destroy, void *c)
+TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, cond_signal, void *c)
+TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, cond_broadcast, void *c)
+TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, cond_wait, void *c, void *m)
+TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, mutex_init, void *m, void *a)
+TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, mutex_destroy, void *m)
+TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, mutex_lock, void *m)
+TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, mutex_trylock, void *m)
+TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, mutex_unlock, void *m)
+TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_init, void *l, void *a)
+TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_destroy, void *l)
+TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_rdlock, void *l)
+TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_tryrdlock, void *l)
+TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_wrlock, void *l)
+TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_trywrlock, void *l)
+TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_unlock, void *l)
+TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, once, void *o, void (*i)())
+TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, sigmask, int f, void *n, void *o)
+
TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_init, void *c, void *a)
TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_signal, void *c)
TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_broadcast, void *c)
TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_destroy, void *c)
TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_init, void *m, void *a)
TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_destroy, void *m)
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_lock, void *m)
TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_trylock, void *m)
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_unlock, void *m)
TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_init, void *m, void *a)
TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_destroy, void *m)
TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_rdlock, void *m)
Die();
}
-#if !SANITIZER_MAC && !SANITIZER_ANDROID
+#if !SANITIZER_APPLE && !SANITIZER_ANDROID
static void unreachable() {
Report("FATAL: ThreadSanitizer: unreachable called\n");
Die();
SANITIZER_WEAK_ATTRIBUTE void InitializeLibdispatchInterceptors() {}
void InitializeInterceptors() {
-#if !SANITIZER_MAC
+#if !SANITIZER_APPLE
// We need to setup it early, because functions like dlsym() can call it.
REAL(memset) = internal_memset;
REAL(memcpy) = internal_memcpy;
#endif
- // Instruct libc malloc to consume less memory.
-#if SANITIZER_GLIBC
- mallopt(1, 0); // M_MXFAST
- mallopt(-3, 32*1024); // M_MMAP_THRESHOLD
-#endif
-
new(interceptor_ctx()) InterceptorContext();
InitializeCommonInterceptors();
InitializeSignalInterceptors();
InitializeLibdispatchInterceptors();
-#if !SANITIZER_MAC
+#if !SANITIZER_APPLE
// We can not use TSAN_INTERCEPT to get setjmp addr,
// because it does &setjmp and setjmp is not present in some versions of libc.
using __interception::InterceptFunction;
TSAN_INTERCEPT(pthread_mutex_init);
TSAN_INTERCEPT(pthread_mutex_destroy);
+ TSAN_INTERCEPT(pthread_mutex_lock);
TSAN_INTERCEPT(pthread_mutex_trylock);
TSAN_INTERCEPT(pthread_mutex_timedlock);
+ TSAN_INTERCEPT(pthread_mutex_unlock);
+#if SANITIZER_GLIBC
+# if !__GLIBC_PREREQ(2, 34)
+ TSAN_INTERCEPT(__pthread_mutex_lock);
+ TSAN_INTERCEPT(__pthread_mutex_unlock);
+# endif
+#endif
TSAN_INTERCEPT(pthread_spin_init);
TSAN_INTERCEPT(pthread_spin_destroy);
TSAN_INTERCEPT(fork);
TSAN_INTERCEPT(vfork);
+#if SANITIZER_LINUX
+ TSAN_INTERCEPT(clone);
+#endif
#if !SANITIZER_ANDROID
TSAN_INTERCEPT(dl_iterate_phdr);
#endif
TSAN_MAYBE_INTERCEPT__LWP_EXIT;
TSAN_MAYBE_INTERCEPT_THR_EXIT;
-#if !SANITIZER_MAC && !SANITIZER_ANDROID
+#if !SANITIZER_APPLE && !SANITIZER_ANDROID
// Need to setup it, because interceptors check that the function is resolved.
// But atexit is emitted directly into the module, so can't be resolved.
REAL(atexit) = (int(*)(void(*)()))unreachable;
Die();
}
-#if !SANITIZER_MAC && !SANITIZER_NETBSD && !SANITIZER_FREEBSD
+#if !SANITIZER_APPLE && !SANITIZER_NETBSD && !SANITIZER_FREEBSD
if (pthread_key_create(&interceptor_ctx()->finalize_key, &thread_finalize)) {
Printf("ThreadSanitizer: failed to create thread key\n");
Die();
}
#endif
+ TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(cond_init);
+ TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(cond_destroy);
+ TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(cond_signal);
+ TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(cond_broadcast);
+ TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(cond_wait);
+ TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(mutex_init);
+ TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(mutex_destroy);
+ TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(mutex_lock);
+ TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(mutex_trylock);
+ TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(mutex_unlock);
+ TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_init);
+ TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_destroy);
+ TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_rdlock);
+ TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_tryrdlock);
+ TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_wrlock);
+ TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_trywrlock);
+ TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_unlock);
+ TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(once);
+ TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(sigmask);
+
TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_init);
TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_signal);
TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_broadcast);
TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_destroy);
TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_init);
TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_destroy);
+ TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_lock);
TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_trylock);
+ TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_unlock);
TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_init);
TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_destroy);
TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_rdlock);
// Note that no_sanitize_thread attribute does not turn off atomic interception
// so attaching it to the function defined in user code does not help.
// That's why we now have what we have.
-extern "C" SANITIZER_INTERFACE_ATTRIBUTE
-void __tsan_testonly_barrier_init(u64 *barrier, u32 count) {
- if (count >= (1 << 8)) {
- Printf("barrier_init: count is too large (%d)\n", count);
- Die();
+constexpr u32 kBarrierThreadBits = 10;
+constexpr u32 kBarrierThreads = 1 << kBarrierThreadBits;
+
+extern "C" {
+
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_testonly_barrier_init(
+ atomic_uint32_t *barrier, u32 num_threads) {
+ if (num_threads >= kBarrierThreads) {
+ Printf("barrier_init: count is too large (%d)\n", num_threads);
+ Die();
}
- // 8 lsb is thread count, the remaining are count of entered threads.
- *barrier = count;
+ // kBarrierThreadBits lsb is thread count,
+ // the remaining are count of entered threads.
+ atomic_store(barrier, num_threads, memory_order_relaxed);
}
-extern "C" SANITIZER_INTERFACE_ATTRIBUTE
-void __tsan_testonly_barrier_wait(u64 *barrier) {
- unsigned old = __atomic_fetch_add(barrier, 1 << 8, __ATOMIC_RELAXED);
- unsigned old_epoch = (old >> 8) / (old & 0xff);
+static u32 barrier_epoch(u32 value) {
+ return (value >> kBarrierThreadBits) / (value & (kBarrierThreads - 1));
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_testonly_barrier_wait(
+ atomic_uint32_t *barrier) {
+ u32 old = atomic_fetch_add(barrier, kBarrierThreads, memory_order_relaxed);
+ u32 old_epoch = barrier_epoch(old);
+ if (barrier_epoch(old + kBarrierThreads) != old_epoch) {
+ FutexWake(barrier, (1 << 30));
+ return;
+ }
for (;;) {
- unsigned cur = __atomic_load_n(barrier, __ATOMIC_RELAXED);
- unsigned cur_epoch = (cur >> 8) / (cur & 0xff);
- if (cur_epoch != old_epoch)
+ u32 cur = atomic_load(barrier, memory_order_relaxed);
+ if (barrier_epoch(cur) != old_epoch)
return;
- internal_sched_yield();
+ FutexWait(barrier, cur);
}
}
+
+void *__tsan_memcpy(void *dst, const void *src, uptr size) {
+ void *ctx;
+#if PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE
+ COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, dst, src, size);
+#else
+ COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size);
+#endif
+}
+
+void *__tsan_memset(void *dst, int c, uptr size) {
+ void *ctx;
+ COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, c, size);
+}
+
+void *__tsan_memmove(void *dst, const void *src, uptr size) {
+ void *ctx;
+ COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size);
+}
+}
using namespace __tsan;
-void __tsan_init() {
- cur_thread_init();
- Initialize(cur_thread());
-}
+void __tsan_init() { Initialize(cur_thread_init()); }
void __tsan_flush_memory() {
FlushShadowMemory();
}
-void __tsan_read16(void *addr) {
- MemoryRead(cur_thread(), CALLERPC, (uptr)addr, kSizeLog8);
- MemoryRead(cur_thread(), CALLERPC, (uptr)addr + 8, kSizeLog8);
-}
-
-void __tsan_write16(void *addr) {
- MemoryWrite(cur_thread(), CALLERPC, (uptr)addr, kSizeLog8);
- MemoryWrite(cur_thread(), CALLERPC, (uptr)addr + 8, kSizeLog8);
-}
-
void __tsan_read16_pc(void *addr, void *pc) {
- MemoryRead(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, kSizeLog8);
- MemoryRead(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr + 8, kSizeLog8);
+ uptr pc_no_pac = STRIP_PAC_PC(pc);
+ ThreadState *thr = cur_thread();
+ MemoryAccess(thr, pc_no_pac, (uptr)addr, 8, kAccessRead);
+ MemoryAccess(thr, pc_no_pac, (uptr)addr + 8, 8, kAccessRead);
}
void __tsan_write16_pc(void *addr, void *pc) {
- MemoryWrite(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, kSizeLog8);
- MemoryWrite(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr + 8, kSizeLog8);
+ uptr pc_no_pac = STRIP_PAC_PC(pc);
+ ThreadState *thr = cur_thread();
+ MemoryAccess(thr, pc_no_pac, (uptr)addr, 8, kAccessWrite);
+ MemoryAccess(thr, pc_no_pac, (uptr)addr + 8, 8, kAccessWrite);
}
// __tsan_unaligned_read/write calls are emitted by compiler.
-void __tsan_unaligned_read2(const void *addr) {
- UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 2, false, false);
-}
-
-void __tsan_unaligned_read4(const void *addr) {
- UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 4, false, false);
-}
-
-void __tsan_unaligned_read8(const void *addr) {
- UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, false, false);
-}
-
void __tsan_unaligned_read16(const void *addr) {
- UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 16, false, false);
-}
-
-void __tsan_unaligned_write2(void *addr) {
- UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 2, true, false);
-}
-
-void __tsan_unaligned_write4(void *addr) {
- UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 4, true, false);
-}
-
-void __tsan_unaligned_write8(void *addr) {
- UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, true, false);
+ uptr pc = CALLERPC;
+ ThreadState *thr = cur_thread();
+ UnalignedMemoryAccess(thr, pc, (uptr)addr, 8, kAccessRead);
+ UnalignedMemoryAccess(thr, pc, (uptr)addr + 8, 8, kAccessRead);
}
void __tsan_unaligned_write16(void *addr) {
- UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 16, true, false);
+ uptr pc = CALLERPC;
+ ThreadState *thr = cur_thread();
+ UnalignedMemoryAccess(thr, pc, (uptr)addr, 8, kAccessWrite);
+ UnalignedMemoryAccess(thr, pc, (uptr)addr + 8, 8, kAccessWrite);
}
-// __sanitizer_unaligned_load/store are for user instrumentation.
-
extern "C" {
-SANITIZER_INTERFACE_ATTRIBUTE
-u16 __sanitizer_unaligned_load16(const uu16 *addr) {
- __tsan_unaligned_read2(addr);
- return *addr;
-}
-
-SANITIZER_INTERFACE_ATTRIBUTE
-u32 __sanitizer_unaligned_load32(const uu32 *addr) {
- __tsan_unaligned_read4(addr);
- return *addr;
-}
-
-SANITIZER_INTERFACE_ATTRIBUTE
-u64 __sanitizer_unaligned_load64(const uu64 *addr) {
- __tsan_unaligned_read8(addr);
- return *addr;
-}
-
-SANITIZER_INTERFACE_ATTRIBUTE
-void __sanitizer_unaligned_store16(uu16 *addr, u16 v) {
- __tsan_unaligned_write2(addr);
- *addr = v;
-}
-
-SANITIZER_INTERFACE_ATTRIBUTE
-void __sanitizer_unaligned_store32(uu32 *addr, u32 v) {
- __tsan_unaligned_write4(addr);
- *addr = v;
-}
-
-SANITIZER_INTERFACE_ATTRIBUTE
-void __sanitizer_unaligned_store64(uu64 *addr, u64 v) {
- __tsan_unaligned_write8(addr);
- *addr = v;
-}
-
SANITIZER_INTERFACE_ATTRIBUTE
void *__tsan_get_current_fiber() {
return cur_thread();
SANITIZER_INTERFACE_ATTRIBUTE
void __tsan_vptr_update(void **vptr_p, void *new_val);
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__tsan_memcpy(void *dest, const void *src, uptr count);
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__tsan_memset(void *dest, int ch, uptr count);
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__tsan_memmove(void *dest, const void *src, uptr count);
+
SANITIZER_INTERFACE_ATTRIBUTE void __tsan_func_entry(void *call_pc);
SANITIZER_INTERFACE_ATTRIBUTE void __tsan_func_exit();
void __tsan_write_range(void *addr, unsigned long size);
SANITIZER_INTERFACE_ATTRIBUTE
-void __tsan_read_range_pc(void *addr, unsigned long size, void *pc); // NOLINT
+void __tsan_read_range_pc(void *addr, unsigned long size, void *pc);
SANITIZER_INTERFACE_ATTRIBUTE
-void __tsan_write_range_pc(void *addr, unsigned long size, void *pc); // NOLINT
+void __tsan_write_range_pc(void *addr, unsigned long size, void *pc);
// User may provide function that would be called right when TSan detects
// an error. The argument 'report' is an opaque pointer that can be used to
void __tsan_go_atomic64_compare_exchange(ThreadState *thr, uptr cpc, uptr pc,
u8 *a);
-SANITIZER_INTERFACE_ATTRIBUTE
-void __tsan_on_initialize();
-
-SANITIZER_INTERFACE_ATTRIBUTE
-int __tsan_on_finalize(int failed);
-
} // extern "C"
} // namespace __tsan
--- /dev/null
+//===-- tsan_interface.inc --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_ptrauth.h"
+#include "tsan_interface.h"
+#include "tsan_rtl.h"
+
+#define CALLERPC ((uptr)__builtin_return_address(0))
+
+using namespace __tsan;
+
+void __tsan_read1(void *addr) {
+ MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 1, kAccessRead);
+}
+
+void __tsan_read2(void *addr) {
+ MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 2, kAccessRead);
+}
+
+void __tsan_read4(void *addr) {
+ MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 4, kAccessRead);
+}
+
+void __tsan_read8(void *addr) {
+ MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, kAccessRead);
+}
+
+void __tsan_read16(void *addr) {
+ MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr, kAccessRead);
+}
+
+void __tsan_write1(void *addr) {
+ MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 1, kAccessWrite);
+}
+
+void __tsan_write2(void *addr) {
+ MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 2, kAccessWrite);
+}
+
+void __tsan_write4(void *addr) {
+ MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 4, kAccessWrite);
+}
+
+void __tsan_write8(void *addr) {
+ MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, kAccessWrite);
+}
+
+void __tsan_write16(void *addr) {
+ MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr, kAccessWrite);
+}
+
+void __tsan_read1_pc(void *addr, void *pc) {
+ MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 1, kAccessRead | kAccessExternalPC);
+}
+
+void __tsan_read2_pc(void *addr, void *pc) {
+ MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 2, kAccessRead | kAccessExternalPC);
+}
+
+void __tsan_read4_pc(void *addr, void *pc) {
+ MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 4, kAccessRead | kAccessExternalPC);
+}
+
+void __tsan_read8_pc(void *addr, void *pc) {
+ MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 8, kAccessRead | kAccessExternalPC);
+}
+
+void __tsan_write1_pc(void *addr, void *pc) {
+ MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 1, kAccessWrite | kAccessExternalPC);
+}
+
+void __tsan_write2_pc(void *addr, void *pc) {
+ MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 2, kAccessWrite | kAccessExternalPC);
+}
+
+void __tsan_write4_pc(void *addr, void *pc) {
+ MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 4, kAccessWrite | kAccessExternalPC);
+}
+
+void __tsan_write8_pc(void *addr, void *pc) {
+ MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 8, kAccessWrite | kAccessExternalPC);
+}
+
+ALWAYS_INLINE USED void __tsan_unaligned_read2(const void *addr) {
+ UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 2, kAccessRead);
+}
+
+ALWAYS_INLINE USED void __tsan_unaligned_read4(const void *addr) {
+ UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 4, kAccessRead);
+}
+
+ALWAYS_INLINE USED void __tsan_unaligned_read8(const void *addr) {
+ UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, kAccessRead);
+}
+
+ALWAYS_INLINE USED void __tsan_unaligned_write2(void *addr) {
+ UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 2, kAccessWrite);
+}
+
+ALWAYS_INLINE USED void __tsan_unaligned_write4(void *addr) {
+ UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 4, kAccessWrite);
+}
+
+ALWAYS_INLINE USED void __tsan_unaligned_write8(void *addr) {
+ UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, kAccessWrite);
+}
+
+extern "C" {
+// __sanitizer_unaligned_load/store are for user instrumentation.
+SANITIZER_INTERFACE_ATTRIBUTE
+u16 __sanitizer_unaligned_load16(const uu16 *addr) {
+ __tsan_unaligned_read2(addr);
+ return *addr;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+u32 __sanitizer_unaligned_load32(const uu32 *addr) {
+ __tsan_unaligned_read4(addr);
+ return *addr;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+u64 __sanitizer_unaligned_load64(const uu64 *addr) {
+ __tsan_unaligned_read8(addr);
+ return *addr;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_unaligned_store16(uu16 *addr, u16 v) {
+ *addr = v;
+ __tsan_unaligned_write2(addr);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_unaligned_store32(uu32 *addr, u32 v) {
+ *addr = v;
+ __tsan_unaligned_write4(addr);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_unaligned_store64(uu64 *addr, u64 v) {
+ *addr = v;
+ __tsan_unaligned_write8(addr);
+}
+}
+
+void __tsan_vptr_update(void **vptr_p, void *new_val) {
+ if (*vptr_p == new_val)
+ return;
+ MemoryAccess(cur_thread(), CALLERPC, (uptr)vptr_p, sizeof(*vptr_p),
+ kAccessWrite | kAccessVptr);
+}
+
+void __tsan_vptr_read(void **vptr_p) {
+ MemoryAccess(cur_thread(), CALLERPC, (uptr)vptr_p, sizeof(*vptr_p),
+ kAccessRead | kAccessVptr);
+}
+
+void __tsan_func_entry(void *pc) { FuncEntry(cur_thread(), STRIP_PAC_PC(pc)); }
+
+void __tsan_func_exit() { FuncExit(cur_thread()); }
+
+void __tsan_ignore_thread_begin() { ThreadIgnoreBegin(cur_thread(), CALLERPC); }
+
+void __tsan_ignore_thread_end() { ThreadIgnoreEnd(cur_thread()); }
+
+void __tsan_read_range(void *addr, uptr size) {
+ MemoryAccessRange(cur_thread(), CALLERPC, (uptr)addr, size, false);
+}
+
+void __tsan_write_range(void *addr, uptr size) {
+ MemoryAccessRange(cur_thread(), CALLERPC, (uptr)addr, size, true);
+}
+
+void __tsan_read_range_pc(void *addr, uptr size, void *pc) {
+ MemoryAccessRange(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, size, false);
+}
+
+void __tsan_write_range_pc(void *addr, uptr size, void *pc) {
+ MemoryAccessRange(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, size, true);
+}
ThreadState *const thr_;
};
-#define SCOPED_ANNOTATION_RET(typ, ret) \
- if (!flags()->enable_annotations) \
- return ret; \
- ThreadState *thr = cur_thread(); \
- const uptr caller_pc = (uptr)__builtin_return_address(0); \
- ScopedAnnotation sa(thr, __func__, caller_pc); \
- const uptr pc = StackTrace::GetCurrentPc(); \
- (void)pc; \
-/**/
+#define SCOPED_ANNOTATION_RET(typ, ret) \
+ if (!flags()->enable_annotations) \
+ return ret; \
+ ThreadState *thr = cur_thread(); \
+ const uptr caller_pc = (uptr)__builtin_return_address(0); \
+ ScopedAnnotation sa(thr, __func__, caller_pc); \
+ const uptr pc = StackTrace::GetCurrentPc(); \
+ (void)pc;
#define SCOPED_ANNOTATION(typ) SCOPED_ANNOTATION_RET(typ, )
struct DynamicAnnContext {
Mutex mtx;
- ExpectRace expect;
ExpectRace benign;
DynamicAnnContext() : mtx(MutexTypeAnnotations) {}
return;
}
}
- race = (ExpectRace*)internal_alloc(MBlockExpectRace, sizeof(ExpectRace));
+ race = static_cast<ExpectRace *>(Alloc(sizeof(ExpectRace)));
race->addr = addr;
race->size = size;
race->file = f;
void InitializeDynamicAnnotations() {
dyn_ann_ctx = new(dyn_ann_ctx_placeholder) DynamicAnnContext;
- InitList(&dyn_ann_ctx->expect);
InitList(&dyn_ann_ctx->benign);
}
bool IsExpectedReport(uptr addr, uptr size) {
ReadLock lock(&dyn_ann_ctx->mtx);
- if (CheckContains(&dyn_ann_ctx->expect, addr, size))
- return true;
- if (CheckContains(&dyn_ann_ctx->benign, addr, size))
- return true;
- return false;
-}
-
-static void CollectMatchedBenignRaces(Vector<ExpectRace> *matched,
- int *unique_count, int *hit_count, atomic_uintptr_t ExpectRace::*counter) {
- ExpectRace *list = &dyn_ann_ctx->benign;
- for (ExpectRace *race = list->next; race != list; race = race->next) {
- (*unique_count)++;
- const uptr cnt = atomic_load_relaxed(&(race->*counter));
- if (cnt == 0)
- continue;
- *hit_count += cnt;
- uptr i = 0;
- for (; i < matched->Size(); i++) {
- ExpectRace *race0 = &(*matched)[i];
- if (race->line == race0->line
- && internal_strcmp(race->file, race0->file) == 0
- && internal_strcmp(race->desc, race0->desc) == 0) {
- atomic_fetch_add(&(race0->*counter), cnt, memory_order_relaxed);
- break;
- }
- }
- if (i == matched->Size())
- matched->PushBack(*race);
- }
-}
-
-void PrintMatchedBenignRaces() {
- Lock lock(&dyn_ann_ctx->mtx);
- int unique_count = 0;
- int hit_count = 0;
- int add_count = 0;
- Vector<ExpectRace> hit_matched;
- CollectMatchedBenignRaces(&hit_matched, &unique_count, &hit_count,
- &ExpectRace::hitcount);
- Vector<ExpectRace> add_matched;
- CollectMatchedBenignRaces(&add_matched, &unique_count, &add_count,
- &ExpectRace::addcount);
- if (hit_matched.Size()) {
- Printf("ThreadSanitizer: Matched %d \"benign\" races (pid=%d):\n",
- hit_count, (int)internal_getpid());
- for (uptr i = 0; i < hit_matched.Size(); i++) {
- Printf("%d %s:%d %s\n",
- atomic_load_relaxed(&hit_matched[i].hitcount),
- hit_matched[i].file, hit_matched[i].line, hit_matched[i].desc);
- }
- }
- if (hit_matched.Size()) {
- Printf("ThreadSanitizer: Annotated %d \"benign\" races, %d unique"
- " (pid=%d):\n",
- add_count, unique_count, (int)internal_getpid());
- for (uptr i = 0; i < add_matched.Size(); i++) {
- Printf("%d %s:%d %s\n",
- atomic_load_relaxed(&add_matched[i].addcount),
- add_matched[i].file, add_matched[i].line, add_matched[i].desc);
- }
- }
-}
-
-static void ReportMissedExpectedRace(ExpectRace *race) {
- Printf("==================\n");
- Printf("WARNING: ThreadSanitizer: missed expected data race\n");
- Printf(" %s addr=%zx %s:%d\n",
- race->desc, race->addr, race->file, race->line);
- Printf("==================\n");
+ return CheckContains(&dyn_ann_ctx->benign, addr, size);
}
} // namespace __tsan
}
void INTERFACE_ATTRIBUTE AnnotateCondVarSignal(char *f, int l, uptr cv) {
- SCOPED_ANNOTATION(AnnotateCondVarSignal);
}
void INTERFACE_ATTRIBUTE AnnotateCondVarSignalAll(char *f, int l, uptr cv) {
- SCOPED_ANNOTATION(AnnotateCondVarSignalAll);
}
void INTERFACE_ATTRIBUTE AnnotateMutexIsNotPHB(char *f, int l, uptr mu) {
- SCOPED_ANNOTATION(AnnotateMutexIsNotPHB);
}
void INTERFACE_ATTRIBUTE AnnotateCondVarWait(char *f, int l, uptr cv,
uptr lock) {
- SCOPED_ANNOTATION(AnnotateCondVarWait);
}
void INTERFACE_ATTRIBUTE AnnotateRWLockCreate(char *f, int l, uptr m) {
}
void INTERFACE_ATTRIBUTE AnnotateTraceMemory(char *f, int l, uptr mem) {
- SCOPED_ANNOTATION(AnnotateTraceMemory);
}
void INTERFACE_ATTRIBUTE AnnotateFlushState(char *f, int l) {
- SCOPED_ANNOTATION(AnnotateFlushState);
}
void INTERFACE_ATTRIBUTE AnnotateNewMemory(char *f, int l, uptr mem,
uptr size) {
- SCOPED_ANNOTATION(AnnotateNewMemory);
}
void INTERFACE_ATTRIBUTE AnnotateNoOp(char *f, int l, uptr mem) {
- SCOPED_ANNOTATION(AnnotateNoOp);
}
void INTERFACE_ATTRIBUTE AnnotateFlushExpectedRaces(char *f, int l) {
- SCOPED_ANNOTATION(AnnotateFlushExpectedRaces);
- Lock lock(&dyn_ann_ctx->mtx);
- while (dyn_ann_ctx->expect.next != &dyn_ann_ctx->expect) {
- ExpectRace *race = dyn_ann_ctx->expect.next;
- if (atomic_load_relaxed(&race->hitcount) == 0) {
- ctx->nmissed_expected++;
- ReportMissedExpectedRace(race);
- }
- race->prev->next = race->next;
- race->next->prev = race->prev;
- internal_free(race);
- }
}
void INTERFACE_ATTRIBUTE AnnotateEnableRaceDetection(
char *f, int l, int enable) {
- SCOPED_ANNOTATION(AnnotateEnableRaceDetection);
- // FIXME: Reconsider this functionality later. It may be irrelevant.
}
void INTERFACE_ATTRIBUTE AnnotateMutexIsUsedAsCondVar(
char *f, int l, uptr mu) {
- SCOPED_ANNOTATION(AnnotateMutexIsUsedAsCondVar);
}
void INTERFACE_ATTRIBUTE AnnotatePCQGet(
char *f, int l, uptr pcq) {
- SCOPED_ANNOTATION(AnnotatePCQGet);
}
void INTERFACE_ATTRIBUTE AnnotatePCQPut(
char *f, int l, uptr pcq) {
- SCOPED_ANNOTATION(AnnotatePCQPut);
}
void INTERFACE_ATTRIBUTE AnnotatePCQDestroy(
char *f, int l, uptr pcq) {
- SCOPED_ANNOTATION(AnnotatePCQDestroy);
}
void INTERFACE_ATTRIBUTE AnnotatePCQCreate(
char *f, int l, uptr pcq) {
- SCOPED_ANNOTATION(AnnotatePCQCreate);
}
void INTERFACE_ATTRIBUTE AnnotateExpectRace(
char *f, int l, uptr mem, char *desc) {
- SCOPED_ANNOTATION(AnnotateExpectRace);
- Lock lock(&dyn_ann_ctx->mtx);
- AddExpectRace(&dyn_ann_ctx->expect,
- f, l, mem, 1, desc);
- DPrintf("Add expected race: %s addr=%zx %s:%d\n", desc, mem, f, l);
}
-static void BenignRaceImpl(
- char *f, int l, uptr mem, uptr size, char *desc) {
+static void BenignRaceImpl(char *f, int l, uptr mem, uptr size, char *desc) {
Lock lock(&dyn_ann_ctx->mtx);
AddExpectRace(&dyn_ann_ctx->benign,
f, l, mem, size, desc);
DPrintf("Add benign race: %s addr=%zx %s:%d\n", desc, mem, f, l);
}
-// FIXME: Turn it off later. WTF is benign race?1?? Go talk to Hans Boehm.
void INTERFACE_ATTRIBUTE AnnotateBenignRaceSized(
char *f, int l, uptr mem, uptr size, char *desc) {
SCOPED_ANNOTATION(AnnotateBenignRaceSized);
void INTERFACE_ATTRIBUTE AnnotateIgnoreReadsEnd(char *f, int l) {
SCOPED_ANNOTATION(AnnotateIgnoreReadsEnd);
- ThreadIgnoreEnd(thr, pc);
+ ThreadIgnoreEnd(thr);
}
void INTERFACE_ATTRIBUTE AnnotateIgnoreWritesBegin(char *f, int l) {
void INTERFACE_ATTRIBUTE AnnotateIgnoreWritesEnd(char *f, int l) {
SCOPED_ANNOTATION(AnnotateIgnoreWritesEnd);
- ThreadIgnoreEnd(thr, pc);
+ ThreadIgnoreEnd(thr);
}
void INTERFACE_ATTRIBUTE AnnotateIgnoreSyncBegin(char *f, int l) {
void INTERFACE_ATTRIBUTE AnnotateIgnoreSyncEnd(char *f, int l) {
SCOPED_ANNOTATION(AnnotateIgnoreSyncEnd);
- ThreadIgnoreSyncEnd(thr, pc);
+ ThreadIgnoreSyncEnd(thr);
}
void INTERFACE_ATTRIBUTE AnnotatePublishMemoryRange(
char *f, int l, uptr addr, uptr size) {
- SCOPED_ANNOTATION(AnnotatePublishMemoryRange);
}
void INTERFACE_ATTRIBUTE AnnotateUnpublishMemoryRange(
char *f, int l, uptr addr, uptr size) {
- SCOPED_ANNOTATION(AnnotateUnpublishMemoryRange);
}
void INTERFACE_ATTRIBUTE AnnotateThreadName(
// WTFAnnotateHappensAfter(). Those are being used by Webkit to annotate
// atomic operations, which should be handled by ThreadSanitizer correctly.
void INTERFACE_ATTRIBUTE WTFAnnotateHappensBefore(char *f, int l, uptr addr) {
- SCOPED_ANNOTATION(AnnotateHappensBefore);
}
void INTERFACE_ATTRIBUTE WTFAnnotateHappensAfter(char *f, int l, uptr addr) {
- SCOPED_ANNOTATION(AnnotateHappensAfter);
}
void INTERFACE_ATTRIBUTE WTFAnnotateBenignRaceSized(
else
MutexPreLock(thr, pc, (uptr)m);
}
- ThreadIgnoreBegin(thr, pc, /*save_stack=*/false);
- ThreadIgnoreSyncBegin(thr, pc, /*save_stack=*/false);
+ ThreadIgnoreBegin(thr, 0);
+ ThreadIgnoreSyncBegin(thr, 0);
}
INTERFACE_ATTRIBUTE
void __tsan_mutex_post_lock(void *m, unsigned flagz, int rec) {
SCOPED_ANNOTATION(__tsan_mutex_post_lock);
- ThreadIgnoreSyncEnd(thr, pc);
- ThreadIgnoreEnd(thr, pc);
+ ThreadIgnoreSyncEnd(thr);
+ ThreadIgnoreEnd(thr);
if (!(flagz & MutexFlagTryLockFailed)) {
if (flagz & MutexFlagReadLock)
MutexPostReadLock(thr, pc, (uptr)m, flagz);
} else {
ret = MutexUnlock(thr, pc, (uptr)m, flagz);
}
- ThreadIgnoreBegin(thr, pc, /*save_stack=*/false);
- ThreadIgnoreSyncBegin(thr, pc, /*save_stack=*/false);
+ ThreadIgnoreBegin(thr, 0);
+ ThreadIgnoreSyncBegin(thr, 0);
return ret;
}
INTERFACE_ATTRIBUTE
void __tsan_mutex_post_unlock(void *m, unsigned flagz) {
SCOPED_ANNOTATION(__tsan_mutex_post_unlock);
- ThreadIgnoreSyncEnd(thr, pc);
- ThreadIgnoreEnd(thr, pc);
+ ThreadIgnoreSyncEnd(thr);
+ ThreadIgnoreEnd(thr);
}
INTERFACE_ATTRIBUTE
void __tsan_mutex_pre_signal(void *addr, unsigned flagz) {
SCOPED_ANNOTATION(__tsan_mutex_pre_signal);
- ThreadIgnoreBegin(thr, pc, /*save_stack=*/false);
- ThreadIgnoreSyncBegin(thr, pc, /*save_stack=*/false);
+ ThreadIgnoreBegin(thr, 0);
+ ThreadIgnoreSyncBegin(thr, 0);
}
INTERFACE_ATTRIBUTE
void __tsan_mutex_post_signal(void *addr, unsigned flagz) {
SCOPED_ANNOTATION(__tsan_mutex_post_signal);
- ThreadIgnoreSyncEnd(thr, pc);
- ThreadIgnoreEnd(thr, pc);
+ ThreadIgnoreSyncEnd(thr);
+ ThreadIgnoreEnd(thr);
}
INTERFACE_ATTRIBUTE
void __tsan_mutex_pre_divert(void *addr, unsigned flagz) {
SCOPED_ANNOTATION(__tsan_mutex_pre_divert);
// Exit from ignore region started in __tsan_mutex_pre_lock/unlock/signal.
- ThreadIgnoreSyncEnd(thr, pc);
- ThreadIgnoreEnd(thr, pc);
+ ThreadIgnoreSyncEnd(thr);
+ ThreadIgnoreEnd(thr);
}
INTERFACE_ATTRIBUTE
void __tsan_mutex_post_divert(void *addr, unsigned flagz) {
SCOPED_ANNOTATION(__tsan_mutex_post_divert);
- ThreadIgnoreBegin(thr, pc, /*save_stack=*/false);
- ThreadIgnoreSyncBegin(thr, pc, /*save_stack=*/false);
+ ThreadIgnoreBegin(thr, 0);
+ ThreadIgnoreSyncBegin(thr, 0);
}
} // extern "C"
static StaticSpinMutex mutex128;
#endif
+#if SANITIZER_DEBUG
static bool IsLoadOrder(morder mo) {
return mo == mo_relaxed || mo == mo_consume
|| mo == mo_acquire || mo == mo_seq_cst;
static bool IsStoreOrder(morder mo) {
return mo == mo_relaxed || mo == mo_release || mo == mo_seq_cst;
}
+#endif
static bool IsReleaseOrder(morder mo) {
return mo == mo_release || mo == mo_acq_rel || mo == mo_seq_cst;
}
#endif
-template<typename T>
-static int SizeLog() {
+template <typename T>
+static int AccessSize() {
if (sizeof(T) <= 1)
- return kSizeLog1;
+ return 1;
else if (sizeof(T) <= 2)
- return kSizeLog2;
+ return 2;
else if (sizeof(T) <= 4)
- return kSizeLog4;
+ return 4;
else
- return kSizeLog8;
+ return 8;
// For 16-byte atomics we also use 8-byte memory access,
// this leads to false negatives only in very obscure cases.
}
case mo_acq_rel: return memory_order_acq_rel;
case mo_seq_cst: return memory_order_seq_cst;
}
- CHECK(0);
+ DCHECK(0);
return memory_order_seq_cst;
}
#endif
template <typename T>
-static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a,
- morder mo) NO_THREAD_SAFETY_ANALYSIS {
- CHECK(IsLoadOrder(mo));
+static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a, morder mo) {
+ DCHECK(IsLoadOrder(mo));
// This fast-path is critical for performance.
// Assume the access is atomic.
if (!IsAcquireOrder(mo)) {
- MemoryReadAtomic(thr, pc, (uptr)a, SizeLog<T>());
+ MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(),
+ kAccessRead | kAccessAtomic);
return NoTsanAtomicLoad(a, mo);
}
// Don't create sync object if it does not exist yet. For example, an atomic
// pointer is initialized to nullptr and then periodically acquire-loaded.
T v = NoTsanAtomicLoad(a, mo);
- SyncVar *s = ctx->metamap.GetIfExistsAndLock((uptr)a, false);
+ SyncVar *s = ctx->metamap.GetSyncIfExists((uptr)a);
if (s) {
- AcquireImpl(thr, pc, &s->clock);
+ SlotLocker locker(thr);
+ ReadLock lock(&s->mtx);
+ thr->clock.Acquire(s->clock);
// Re-read under sync mutex because we need a consistent snapshot
// of the value and the clock we acquire.
v = NoTsanAtomicLoad(a, mo);
- s->mtx.ReadUnlock();
}
- MemoryReadAtomic(thr, pc, (uptr)a, SizeLog<T>());
+ MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), kAccessRead | kAccessAtomic);
return v;
}
template <typename T>
static void AtomicStore(ThreadState *thr, uptr pc, volatile T *a, T v,
- morder mo) NO_THREAD_SAFETY_ANALYSIS {
- CHECK(IsStoreOrder(mo));
- MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>());
+ morder mo) {
+ DCHECK(IsStoreOrder(mo));
+ MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), kAccessWrite | kAccessAtomic);
// This fast-path is critical for performance.
// Assume the access is atomic.
// Strictly saying even relaxed store cuts off release sequence,
NoTsanAtomicStore(a, v, mo);
return;
}
- __sync_synchronize();
- SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, true);
- thr->fast_state.IncrementEpoch();
- // Can't increment epoch w/o writing to the trace as well.
- TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
- ReleaseStoreImpl(thr, pc, &s->clock);
- NoTsanAtomicStore(a, v, mo);
- s->mtx.Unlock();
+ SlotLocker locker(thr);
+ {
+ auto s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false);
+ Lock lock(&s->mtx);
+ thr->clock.ReleaseStore(&s->clock);
+ NoTsanAtomicStore(a, v, mo);
+ }
+ IncrementEpoch(thr);
}
template <typename T, T (*F)(volatile T *v, T op)>
-static T AtomicRMW(ThreadState *thr, uptr pc, volatile T *a, T v,
- morder mo) NO_THREAD_SAFETY_ANALYSIS {
- MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>());
- SyncVar *s = 0;
- if (mo != mo_relaxed) {
- s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, true);
- thr->fast_state.IncrementEpoch();
- // Can't increment epoch w/o writing to the trace as well.
- TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
+static T AtomicRMW(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) {
+ MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), kAccessWrite | kAccessAtomic);
+ if (LIKELY(mo == mo_relaxed))
+ return F(a, v);
+ SlotLocker locker(thr);
+ {
+ auto s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false);
+ RWLock lock(&s->mtx, IsReleaseOrder(mo));
if (IsAcqRelOrder(mo))
- AcquireReleaseImpl(thr, pc, &s->clock);
+ thr->clock.ReleaseAcquire(&s->clock);
else if (IsReleaseOrder(mo))
- ReleaseImpl(thr, pc, &s->clock);
+ thr->clock.Release(&s->clock);
else if (IsAcquireOrder(mo))
- AcquireImpl(thr, pc, &s->clock);
+ thr->clock.Acquire(s->clock);
+ v = F(a, v);
}
- v = F(a, v);
- if (s)
- s->mtx.Unlock();
+ if (IsReleaseOrder(mo))
+ IncrementEpoch(thr);
return v;
}
}
template <typename T>
-static bool AtomicCAS(ThreadState *thr, uptr pc, volatile T *a, T *c, T v, morder mo,
- morder fmo) NO_THREAD_SAFETY_ANALYSIS {
+static bool AtomicCAS(ThreadState *thr, uptr pc, volatile T *a, T *c, T v,
+ morder mo, morder fmo) {
// 31.7.2.18: "The failure argument shall not be memory_order_release
// nor memory_order_acq_rel". LLVM (2021-05) fallbacks to Monotonic
// (mo_relaxed) when those are used.
- CHECK(IsLoadOrder(fmo));
-
- MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>());
- SyncVar *s = 0;
- bool write_lock = IsReleaseOrder(mo);
-
- if (mo != mo_relaxed || fmo != mo_relaxed)
- s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, write_lock);
-
- T cc = *c;
- T pr = func_cas(a, cc, v);
- bool success = pr == cc;
- if (!success) {
+ DCHECK(IsLoadOrder(fmo));
+
+ MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), kAccessWrite | kAccessAtomic);
+ if (LIKELY(mo == mo_relaxed && fmo == mo_relaxed)) {
+ T cc = *c;
+ T pr = func_cas(a, cc, v);
+ if (pr == cc)
+ return true;
*c = pr;
- mo = fmo;
+ return false;
}
-
- if (s) {
- thr->fast_state.IncrementEpoch();
- // Can't increment epoch w/o writing to the trace as well.
- TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
-
+ SlotLocker locker(thr);
+ bool release = IsReleaseOrder(mo);
+ bool success;
+ {
+ auto s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false);
+ RWLock lock(&s->mtx, release);
+ T cc = *c;
+ T pr = func_cas(a, cc, v);
+ success = pr == cc;
+ if (!success) {
+ *c = pr;
+ mo = fmo;
+ }
if (success && IsAcqRelOrder(mo))
- AcquireReleaseImpl(thr, pc, &s->clock);
+ thr->clock.ReleaseAcquire(&s->clock);
else if (success && IsReleaseOrder(mo))
- ReleaseImpl(thr, pc, &s->clock);
+ thr->clock.Release(&s->clock);
else if (IsAcquireOrder(mo))
- AcquireImpl(thr, pc, &s->clock);
-
- if (write_lock)
- s->mtx.Unlock();
- else
- s->mtx.ReadUnlock();
+ thr->clock.Acquire(s->clock);
}
-
+ if (success && release)
+ IncrementEpoch(thr);
return success;
}
return (morder)(mo & 0x7fff);
}
-#define SCOPED_ATOMIC(func, ...) \
- ThreadState *const thr = cur_thread(); \
- if (UNLIKELY(thr->ignore_sync || thr->ignore_interceptors)) { \
- ProcessPendingSignals(thr); \
- return NoTsanAtomic##func(__VA_ARGS__); \
- } \
- const uptr callpc = (uptr)__builtin_return_address(0); \
- uptr pc = StackTrace::GetCurrentPc(); \
- mo = convert_morder(mo); \
- ScopedAtomic sa(thr, callpc, a, mo, __func__); \
- return Atomic##func(thr, pc, __VA_ARGS__); \
-/**/
-
-class ScopedAtomic {
- public:
- ScopedAtomic(ThreadState *thr, uptr pc, const volatile void *a,
- morder mo, const char *func)
- : thr_(thr) {
- FuncEntry(thr_, pc);
- DPrintf("#%d: %s(%p, %d)\n", thr_->tid, func, a, mo);
- }
- ~ScopedAtomic() {
- ProcessPendingSignals(thr_);
- FuncExit(thr_);
- }
- private:
- ThreadState *thr_;
-};
+# define ATOMIC_IMPL(func, ...) \
+ ThreadState *const thr = cur_thread(); \
+ ProcessPendingSignals(thr); \
+ if (UNLIKELY(thr->ignore_sync || thr->ignore_interceptors)) \
+ return NoTsanAtomic##func(__VA_ARGS__); \
+ mo = convert_morder(mo); \
+ return Atomic##func(thr, GET_CALLER_PC(), __VA_ARGS__);
extern "C" {
SANITIZER_INTERFACE_ATTRIBUTE
a8 __tsan_atomic8_load(const volatile a8 *a, morder mo) {
- SCOPED_ATOMIC(Load, a, mo);
+ ATOMIC_IMPL(Load, a, mo);
}
SANITIZER_INTERFACE_ATTRIBUTE
a16 __tsan_atomic16_load(const volatile a16 *a, morder mo) {
- SCOPED_ATOMIC(Load, a, mo);
+ ATOMIC_IMPL(Load, a, mo);
}
SANITIZER_INTERFACE_ATTRIBUTE
a32 __tsan_atomic32_load(const volatile a32 *a, morder mo) {
- SCOPED_ATOMIC(Load, a, mo);
+ ATOMIC_IMPL(Load, a, mo);
}
SANITIZER_INTERFACE_ATTRIBUTE
a64 __tsan_atomic64_load(const volatile a64 *a, morder mo) {
- SCOPED_ATOMIC(Load, a, mo);
+ ATOMIC_IMPL(Load, a, mo);
}
#if __TSAN_HAS_INT128
SANITIZER_INTERFACE_ATTRIBUTE
a128 __tsan_atomic128_load(const volatile a128 *a, morder mo) {
- SCOPED_ATOMIC(Load, a, mo);
+ ATOMIC_IMPL(Load, a, mo);
}
#endif
SANITIZER_INTERFACE_ATTRIBUTE
void __tsan_atomic8_store(volatile a8 *a, a8 v, morder mo) {
- SCOPED_ATOMIC(Store, a, v, mo);
+ ATOMIC_IMPL(Store, a, v, mo);
}
SANITIZER_INTERFACE_ATTRIBUTE
void __tsan_atomic16_store(volatile a16 *a, a16 v, morder mo) {
- SCOPED_ATOMIC(Store, a, v, mo);
+ ATOMIC_IMPL(Store, a, v, mo);
}
SANITIZER_INTERFACE_ATTRIBUTE
void __tsan_atomic32_store(volatile a32 *a, a32 v, morder mo) {
- SCOPED_ATOMIC(Store, a, v, mo);
+ ATOMIC_IMPL(Store, a, v, mo);
}
SANITIZER_INTERFACE_ATTRIBUTE
void __tsan_atomic64_store(volatile a64 *a, a64 v, morder mo) {
- SCOPED_ATOMIC(Store, a, v, mo);
+ ATOMIC_IMPL(Store, a, v, mo);
}
#if __TSAN_HAS_INT128
SANITIZER_INTERFACE_ATTRIBUTE
void __tsan_atomic128_store(volatile a128 *a, a128 v, morder mo) {
- SCOPED_ATOMIC(Store, a, v, mo);
+ ATOMIC_IMPL(Store, a, v, mo);
}
#endif
SANITIZER_INTERFACE_ATTRIBUTE
a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, morder mo) {
- SCOPED_ATOMIC(Exchange, a, v, mo);
+ ATOMIC_IMPL(Exchange, a, v, mo);
}
SANITIZER_INTERFACE_ATTRIBUTE
a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, morder mo) {
- SCOPED_ATOMIC(Exchange, a, v, mo);
+ ATOMIC_IMPL(Exchange, a, v, mo);
}
SANITIZER_INTERFACE_ATTRIBUTE
a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, morder mo) {
- SCOPED_ATOMIC(Exchange, a, v, mo);
+ ATOMIC_IMPL(Exchange, a, v, mo);
}
SANITIZER_INTERFACE_ATTRIBUTE
a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, morder mo) {
- SCOPED_ATOMIC(Exchange, a, v, mo);
+ ATOMIC_IMPL(Exchange, a, v, mo);
}
#if __TSAN_HAS_INT128
SANITIZER_INTERFACE_ATTRIBUTE
a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, morder mo) {
- SCOPED_ATOMIC(Exchange, a, v, mo);
+ ATOMIC_IMPL(Exchange, a, v, mo);
}
#endif
SANITIZER_INTERFACE_ATTRIBUTE
a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, morder mo) {
- SCOPED_ATOMIC(FetchAdd, a, v, mo);
+ ATOMIC_IMPL(FetchAdd, a, v, mo);
}
SANITIZER_INTERFACE_ATTRIBUTE
a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, morder mo) {
- SCOPED_ATOMIC(FetchAdd, a, v, mo);
+ ATOMIC_IMPL(FetchAdd, a, v, mo);
}
SANITIZER_INTERFACE_ATTRIBUTE
a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, morder mo) {
- SCOPED_ATOMIC(FetchAdd, a, v, mo);
+ ATOMIC_IMPL(FetchAdd, a, v, mo);
}
SANITIZER_INTERFACE_ATTRIBUTE
a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, morder mo) {
- SCOPED_ATOMIC(FetchAdd, a, v, mo);
+ ATOMIC_IMPL(FetchAdd, a, v, mo);
}
#if __TSAN_HAS_INT128
SANITIZER_INTERFACE_ATTRIBUTE
a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, morder mo) {
- SCOPED_ATOMIC(FetchAdd, a, v, mo);
+ ATOMIC_IMPL(FetchAdd, a, v, mo);
}
#endif
SANITIZER_INTERFACE_ATTRIBUTE
a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, morder mo) {
- SCOPED_ATOMIC(FetchSub, a, v, mo);
+ ATOMIC_IMPL(FetchSub, a, v, mo);
}
SANITIZER_INTERFACE_ATTRIBUTE
a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, morder mo) {
- SCOPED_ATOMIC(FetchSub, a, v, mo);
+ ATOMIC_IMPL(FetchSub, a, v, mo);
}
SANITIZER_INTERFACE_ATTRIBUTE
a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, morder mo) {
- SCOPED_ATOMIC(FetchSub, a, v, mo);
+ ATOMIC_IMPL(FetchSub, a, v, mo);
}
SANITIZER_INTERFACE_ATTRIBUTE
a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, morder mo) {
- SCOPED_ATOMIC(FetchSub, a, v, mo);
+ ATOMIC_IMPL(FetchSub, a, v, mo);
}
#if __TSAN_HAS_INT128
SANITIZER_INTERFACE_ATTRIBUTE
a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, morder mo) {
- SCOPED_ATOMIC(FetchSub, a, v, mo);
+ ATOMIC_IMPL(FetchSub, a, v, mo);
}
#endif
SANITIZER_INTERFACE_ATTRIBUTE
a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, morder mo) {
- SCOPED_ATOMIC(FetchAnd, a, v, mo);
+ ATOMIC_IMPL(FetchAnd, a, v, mo);
}
SANITIZER_INTERFACE_ATTRIBUTE
a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, morder mo) {
- SCOPED_ATOMIC(FetchAnd, a, v, mo);
+ ATOMIC_IMPL(FetchAnd, a, v, mo);
}
SANITIZER_INTERFACE_ATTRIBUTE
a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, morder mo) {
- SCOPED_ATOMIC(FetchAnd, a, v, mo);
+ ATOMIC_IMPL(FetchAnd, a, v, mo);
}
SANITIZER_INTERFACE_ATTRIBUTE
a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, morder mo) {
- SCOPED_ATOMIC(FetchAnd, a, v, mo);
+ ATOMIC_IMPL(FetchAnd, a, v, mo);
}
#if __TSAN_HAS_INT128
SANITIZER_INTERFACE_ATTRIBUTE
a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, morder mo) {
- SCOPED_ATOMIC(FetchAnd, a, v, mo);
+ ATOMIC_IMPL(FetchAnd, a, v, mo);
}
#endif
SANITIZER_INTERFACE_ATTRIBUTE
a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, morder mo) {
- SCOPED_ATOMIC(FetchOr, a, v, mo);
+ ATOMIC_IMPL(FetchOr, a, v, mo);
}
SANITIZER_INTERFACE_ATTRIBUTE
a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, morder mo) {
- SCOPED_ATOMIC(FetchOr, a, v, mo);
+ ATOMIC_IMPL(FetchOr, a, v, mo);
}
SANITIZER_INTERFACE_ATTRIBUTE
a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, morder mo) {
- SCOPED_ATOMIC(FetchOr, a, v, mo);
+ ATOMIC_IMPL(FetchOr, a, v, mo);
}
SANITIZER_INTERFACE_ATTRIBUTE
a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, morder mo) {
- SCOPED_ATOMIC(FetchOr, a, v, mo);
+ ATOMIC_IMPL(FetchOr, a, v, mo);
}
#if __TSAN_HAS_INT128
SANITIZER_INTERFACE_ATTRIBUTE
a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, morder mo) {
- SCOPED_ATOMIC(FetchOr, a, v, mo);
+ ATOMIC_IMPL(FetchOr, a, v, mo);
}
#endif
SANITIZER_INTERFACE_ATTRIBUTE
a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, morder mo) {
- SCOPED_ATOMIC(FetchXor, a, v, mo);
+ ATOMIC_IMPL(FetchXor, a, v, mo);
}
SANITIZER_INTERFACE_ATTRIBUTE
a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, morder mo) {
- SCOPED_ATOMIC(FetchXor, a, v, mo);
+ ATOMIC_IMPL(FetchXor, a, v, mo);
}
SANITIZER_INTERFACE_ATTRIBUTE
a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, morder mo) {
- SCOPED_ATOMIC(FetchXor, a, v, mo);
+ ATOMIC_IMPL(FetchXor, a, v, mo);
}
SANITIZER_INTERFACE_ATTRIBUTE
a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, morder mo) {
- SCOPED_ATOMIC(FetchXor, a, v, mo);
+ ATOMIC_IMPL(FetchXor, a, v, mo);
}
#if __TSAN_HAS_INT128
SANITIZER_INTERFACE_ATTRIBUTE
a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, morder mo) {
- SCOPED_ATOMIC(FetchXor, a, v, mo);
+ ATOMIC_IMPL(FetchXor, a, v, mo);
}
#endif
SANITIZER_INTERFACE_ATTRIBUTE
a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, morder mo) {
- SCOPED_ATOMIC(FetchNand, a, v, mo);
+ ATOMIC_IMPL(FetchNand, a, v, mo);
}
SANITIZER_INTERFACE_ATTRIBUTE
a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, morder mo) {
- SCOPED_ATOMIC(FetchNand, a, v, mo);
+ ATOMIC_IMPL(FetchNand, a, v, mo);
}
SANITIZER_INTERFACE_ATTRIBUTE
a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, morder mo) {
- SCOPED_ATOMIC(FetchNand, a, v, mo);
+ ATOMIC_IMPL(FetchNand, a, v, mo);
}
SANITIZER_INTERFACE_ATTRIBUTE
a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, morder mo) {
- SCOPED_ATOMIC(FetchNand, a, v, mo);
+ ATOMIC_IMPL(FetchNand, a, v, mo);
}
#if __TSAN_HAS_INT128
SANITIZER_INTERFACE_ATTRIBUTE
a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, morder mo) {
- SCOPED_ATOMIC(FetchNand, a, v, mo);
+ ATOMIC_IMPL(FetchNand, a, v, mo);
}
#endif
SANITIZER_INTERFACE_ATTRIBUTE
int __tsan_atomic8_compare_exchange_strong(volatile a8 *a, a8 *c, a8 v,
morder mo, morder fmo) {
- SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
+ ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
}
SANITIZER_INTERFACE_ATTRIBUTE
int __tsan_atomic16_compare_exchange_strong(volatile a16 *a, a16 *c, a16 v,
morder mo, morder fmo) {
- SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
+ ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
}
SANITIZER_INTERFACE_ATTRIBUTE
int __tsan_atomic32_compare_exchange_strong(volatile a32 *a, a32 *c, a32 v,
morder mo, morder fmo) {
- SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
+ ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
}
SANITIZER_INTERFACE_ATTRIBUTE
int __tsan_atomic64_compare_exchange_strong(volatile a64 *a, a64 *c, a64 v,
morder mo, morder fmo) {
- SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
+ ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
}
#if __TSAN_HAS_INT128
SANITIZER_INTERFACE_ATTRIBUTE
int __tsan_atomic128_compare_exchange_strong(volatile a128 *a, a128 *c, a128 v,
morder mo, morder fmo) {
- SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
+ ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
}
#endif
SANITIZER_INTERFACE_ATTRIBUTE
int __tsan_atomic8_compare_exchange_weak(volatile a8 *a, a8 *c, a8 v,
morder mo, morder fmo) {
- SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
+ ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
}
SANITIZER_INTERFACE_ATTRIBUTE
int __tsan_atomic16_compare_exchange_weak(volatile a16 *a, a16 *c, a16 v,
morder mo, morder fmo) {
- SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
+ ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
}
SANITIZER_INTERFACE_ATTRIBUTE
int __tsan_atomic32_compare_exchange_weak(volatile a32 *a, a32 *c, a32 v,
morder mo, morder fmo) {
- SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
+ ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
}
SANITIZER_INTERFACE_ATTRIBUTE
int __tsan_atomic64_compare_exchange_weak(volatile a64 *a, a64 *c, a64 v,
morder mo, morder fmo) {
- SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
+ ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
}
#if __TSAN_HAS_INT128
SANITIZER_INTERFACE_ATTRIBUTE
int __tsan_atomic128_compare_exchange_weak(volatile a128 *a, a128 *c, a128 v,
morder mo, morder fmo) {
- SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
+ ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
}
#endif
SANITIZER_INTERFACE_ATTRIBUTE
a8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v,
morder mo, morder fmo) {
- SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
+ ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
}
SANITIZER_INTERFACE_ATTRIBUTE
a16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v,
morder mo, morder fmo) {
- SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
+ ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
}
SANITIZER_INTERFACE_ATTRIBUTE
a32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v,
morder mo, morder fmo) {
- SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
+ ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
}
SANITIZER_INTERFACE_ATTRIBUTE
a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v,
morder mo, morder fmo) {
- SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
+ ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
}
#if __TSAN_HAS_INT128
SANITIZER_INTERFACE_ATTRIBUTE
a128 __tsan_atomic128_compare_exchange_val(volatile a128 *a, a128 c, a128 v,
morder mo, morder fmo) {
- SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
+ ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
}
#endif
SANITIZER_INTERFACE_ATTRIBUTE
-void __tsan_atomic_thread_fence(morder mo) {
- char* a = 0;
- SCOPED_ATOMIC(Fence, mo);
-}
+void __tsan_atomic_thread_fence(morder mo) { ATOMIC_IMPL(Fence, mo); }
SANITIZER_INTERFACE_ATTRIBUTE
void __tsan_atomic_signal_fence(morder mo) {
// Go
-#define ATOMIC(func, ...) \
- if (thr->ignore_sync) { \
- NoTsanAtomic##func(__VA_ARGS__); \
- } else { \
- FuncEntry(thr, cpc); \
+# define ATOMIC(func, ...) \
+ if (thr->ignore_sync) { \
+ NoTsanAtomic##func(__VA_ARGS__); \
+ } else { \
+ FuncEntry(thr, cpc); \
Atomic##func(thr, pc, __VA_ARGS__); \
- FuncExit(thr); \
- } \
-/**/
-
-#define ATOMIC_RET(func, ret, ...) \
- if (thr->ignore_sync) { \
- (ret) = NoTsanAtomic##func(__VA_ARGS__); \
- } else { \
- FuncEntry(thr, cpc); \
+ FuncExit(thr); \
+ }
+
+# define ATOMIC_RET(func, ret, ...) \
+ if (thr->ignore_sync) { \
+ (ret) = NoTsanAtomic##func(__VA_ARGS__); \
+ } else { \
+ FuncEntry(thr, cpc); \
(ret) = Atomic##func(thr, pc, __VA_ARGS__); \
- FuncExit(thr); \
- } \
-/**/
+ FuncExit(thr); \
+ }
extern "C" {
SANITIZER_INTERFACE_ATTRIBUTE
}
};
-class ScopedJavaFunc {
- public:
- ScopedJavaFunc(ThreadState *thr, uptr pc)
- : thr_(thr) {
- Initialize(thr_);
- FuncEntry(thr, pc);
- }
-
- ~ScopedJavaFunc() {
- FuncExit(thr_);
- // FIXME(dvyukov): process pending signals.
- }
-
- private:
- ThreadState *thr_;
-};
-
static u64 jctx_buf[sizeof(JavaContext) / sizeof(u64) + 1];
static JavaContext *jctx;
+MBlock *JavaHeapBlock(uptr addr, uptr *start) {
+ if (!jctx || addr < jctx->heap_begin ||
+ addr >= jctx->heap_begin + jctx->heap_size)
+ return nullptr;
+ for (uptr p = RoundDown(addr, kMetaShadowCell); p >= jctx->heap_begin;
+ p -= kMetaShadowCell) {
+ MBlock *b = ctx->metamap.GetBlock(p);
+ if (!b)
+ continue;
+ if (p + b->siz <= addr)
+ return nullptr;
+ *start = p;
+ return b;
+ }
+ return nullptr;
+}
+
} // namespace __tsan
-#define SCOPED_JAVA_FUNC(func) \
+#define JAVA_FUNC_ENTER(func) \
ThreadState *thr = cur_thread(); \
- const uptr caller_pc = GET_CALLER_PC(); \
- const uptr pc = StackTrace::GetCurrentPc(); \
- (void)pc; \
- ScopedJavaFunc scoped(thr, caller_pc); \
-/**/
+ (void)thr;
void __tsan_java_init(jptr heap_begin, jptr heap_size) {
- SCOPED_JAVA_FUNC(__tsan_java_init);
- DPrintf("#%d: java_init(%p, %p)\n", thr->tid, heap_begin, heap_size);
- CHECK_EQ(jctx, 0);
- CHECK_GT(heap_begin, 0);
- CHECK_GT(heap_size, 0);
- CHECK_EQ(heap_begin % kHeapAlignment, 0);
- CHECK_EQ(heap_size % kHeapAlignment, 0);
- CHECK_LT(heap_begin, heap_begin + heap_size);
+ JAVA_FUNC_ENTER(__tsan_java_init);
+ Initialize(thr);
+ DPrintf("#%d: java_init(0x%zx, 0x%zx)\n", thr->tid, heap_begin, heap_size);
+ DCHECK_EQ(jctx, 0);
+ DCHECK_GT(heap_begin, 0);
+ DCHECK_GT(heap_size, 0);
+ DCHECK_EQ(heap_begin % kHeapAlignment, 0);
+ DCHECK_EQ(heap_size % kHeapAlignment, 0);
+ DCHECK_LT(heap_begin, heap_begin + heap_size);
jctx = new(jctx_buf) JavaContext(heap_begin, heap_size);
}
int __tsan_java_fini() {
- SCOPED_JAVA_FUNC(__tsan_java_fini);
+ JAVA_FUNC_ENTER(__tsan_java_fini);
DPrintf("#%d: java_fini()\n", thr->tid);
- CHECK_NE(jctx, 0);
+ DCHECK_NE(jctx, 0);
// FIXME(dvyukov): this does not call atexit() callbacks.
int status = Finalize(thr);
DPrintf("#%d: java_fini() = %d\n", thr->tid, status);
}
void __tsan_java_alloc(jptr ptr, jptr size) {
- SCOPED_JAVA_FUNC(__tsan_java_alloc);
- DPrintf("#%d: java_alloc(%p, %p)\n", thr->tid, ptr, size);
- CHECK_NE(jctx, 0);
- CHECK_NE(size, 0);
- CHECK_EQ(ptr % kHeapAlignment, 0);
- CHECK_EQ(size % kHeapAlignment, 0);
- CHECK_GE(ptr, jctx->heap_begin);
- CHECK_LE(ptr + size, jctx->heap_begin + jctx->heap_size);
-
- OnUserAlloc(thr, pc, ptr, size, false);
+ JAVA_FUNC_ENTER(__tsan_java_alloc);
+ DPrintf("#%d: java_alloc(0x%zx, 0x%zx)\n", thr->tid, ptr, size);
+ DCHECK_NE(jctx, 0);
+ DCHECK_NE(size, 0);
+ DCHECK_EQ(ptr % kHeapAlignment, 0);
+ DCHECK_EQ(size % kHeapAlignment, 0);
+ DCHECK_GE(ptr, jctx->heap_begin);
+ DCHECK_LE(ptr + size, jctx->heap_begin + jctx->heap_size);
+
+ OnUserAlloc(thr, 0, ptr, size, false);
}
void __tsan_java_free(jptr ptr, jptr size) {
- SCOPED_JAVA_FUNC(__tsan_java_free);
- DPrintf("#%d: java_free(%p, %p)\n", thr->tid, ptr, size);
- CHECK_NE(jctx, 0);
- CHECK_NE(size, 0);
- CHECK_EQ(ptr % kHeapAlignment, 0);
- CHECK_EQ(size % kHeapAlignment, 0);
- CHECK_GE(ptr, jctx->heap_begin);
- CHECK_LE(ptr + size, jctx->heap_begin + jctx->heap_size);
-
- ctx->metamap.FreeRange(thr->proc(), ptr, size);
+ JAVA_FUNC_ENTER(__tsan_java_free);
+ DPrintf("#%d: java_free(0x%zx, 0x%zx)\n", thr->tid, ptr, size);
+ DCHECK_NE(jctx, 0);
+ DCHECK_NE(size, 0);
+ DCHECK_EQ(ptr % kHeapAlignment, 0);
+ DCHECK_EQ(size % kHeapAlignment, 0);
+ DCHECK_GE(ptr, jctx->heap_begin);
+ DCHECK_LE(ptr + size, jctx->heap_begin + jctx->heap_size);
+
+ ctx->metamap.FreeRange(thr->proc(), ptr, size, false);
}
void __tsan_java_move(jptr src, jptr dst, jptr size) {
- SCOPED_JAVA_FUNC(__tsan_java_move);
- DPrintf("#%d: java_move(%p, %p, %p)\n", thr->tid, src, dst, size);
- CHECK_NE(jctx, 0);
- CHECK_NE(size, 0);
- CHECK_EQ(src % kHeapAlignment, 0);
- CHECK_EQ(dst % kHeapAlignment, 0);
- CHECK_EQ(size % kHeapAlignment, 0);
- CHECK_GE(src, jctx->heap_begin);
- CHECK_LE(src + size, jctx->heap_begin + jctx->heap_size);
- CHECK_GE(dst, jctx->heap_begin);
- CHECK_LE(dst + size, jctx->heap_begin + jctx->heap_size);
- CHECK_NE(dst, src);
- CHECK_NE(size, 0);
+ JAVA_FUNC_ENTER(__tsan_java_move);
+ DPrintf("#%d: java_move(0x%zx, 0x%zx, 0x%zx)\n", thr->tid, src, dst, size);
+ DCHECK_NE(jctx, 0);
+ DCHECK_NE(size, 0);
+ DCHECK_EQ(src % kHeapAlignment, 0);
+ DCHECK_EQ(dst % kHeapAlignment, 0);
+ DCHECK_EQ(size % kHeapAlignment, 0);
+ DCHECK_GE(src, jctx->heap_begin);
+ DCHECK_LE(src + size, jctx->heap_begin + jctx->heap_size);
+ DCHECK_GE(dst, jctx->heap_begin);
+ DCHECK_LE(dst + size, jctx->heap_begin + jctx->heap_size);
+ DCHECK_NE(dst, src);
+ DCHECK_NE(size, 0);
// Assuming it's not running concurrently with threads that do
// memory accesses and mutex operations (stop-the-world phase).
ctx->metamap.MoveMemory(src, dst, size);
- // Move shadow.
- u64 *s = (u64*)MemToShadow(src);
- u64 *d = (u64*)MemToShadow(dst);
- u64 *send = (u64*)MemToShadow(src + size);
- uptr inc = 1;
- if (dst > src) {
- s = (u64*)MemToShadow(src + size) - 1;
- d = (u64*)MemToShadow(dst + size) - 1;
- send = (u64*)MemToShadow(src) - 1;
- inc = -1;
- }
- for (; s != send; s += inc, d += inc) {
- *d = *s;
- *s = 0;
- }
+ // Clear the destination shadow range.
+ // We used to move shadow from src to dst, but the trace format does not
+ // support that anymore as it contains addresses of accesses.
+ RawShadow *d = MemToShadow(dst);
+ RawShadow *dend = MemToShadow(dst + size);
+ ShadowSet(d, dend, Shadow::kEmpty);
}
jptr __tsan_java_find(jptr *from_ptr, jptr to) {
- SCOPED_JAVA_FUNC(__tsan_java_find);
- DPrintf("#%d: java_find(&%p, %p)\n", *from_ptr, to);
- CHECK_EQ((*from_ptr) % kHeapAlignment, 0);
- CHECK_EQ(to % kHeapAlignment, 0);
- CHECK_GE(*from_ptr, jctx->heap_begin);
- CHECK_LE(to, jctx->heap_begin + jctx->heap_size);
+ JAVA_FUNC_ENTER(__tsan_java_find);
+ DPrintf("#%d: java_find(&0x%zx, 0x%zx)\n", thr->tid, *from_ptr, to);
+ DCHECK_EQ((*from_ptr) % kHeapAlignment, 0);
+ DCHECK_EQ(to % kHeapAlignment, 0);
+ DCHECK_GE(*from_ptr, jctx->heap_begin);
+ DCHECK_LE(to, jctx->heap_begin + jctx->heap_size);
for (uptr from = *from_ptr; from < to; from += kHeapAlignment) {
MBlock *b = ctx->metamap.GetBlock(from);
if (b) {
}
void __tsan_java_finalize() {
- SCOPED_JAVA_FUNC(__tsan_java_finalize);
- DPrintf("#%d: java_mutex_finalize()\n", thr->tid);
- AcquireGlobal(thr, 0);
+ JAVA_FUNC_ENTER(__tsan_java_finalize);
+ DPrintf("#%d: java_finalize()\n", thr->tid);
+ AcquireGlobal(thr);
}
void __tsan_java_mutex_lock(jptr addr) {
- SCOPED_JAVA_FUNC(__tsan_java_mutex_lock);
- DPrintf("#%d: java_mutex_lock(%p)\n", thr->tid, addr);
- CHECK_NE(jctx, 0);
- CHECK_GE(addr, jctx->heap_begin);
- CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
-
- MutexPostLock(thr, pc, addr, MutexFlagLinkerInit | MutexFlagWriteReentrant |
- MutexFlagDoPreLockOnPostLock);
+ JAVA_FUNC_ENTER(__tsan_java_mutex_lock);
+ DPrintf("#%d: java_mutex_lock(0x%zx)\n", thr->tid, addr);
+ DCHECK_NE(jctx, 0);
+ DCHECK_GE(addr, jctx->heap_begin);
+ DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
+
+ MutexPostLock(thr, 0, addr,
+ MutexFlagLinkerInit | MutexFlagWriteReentrant |
+ MutexFlagDoPreLockOnPostLock);
}
void __tsan_java_mutex_unlock(jptr addr) {
- SCOPED_JAVA_FUNC(__tsan_java_mutex_unlock);
- DPrintf("#%d: java_mutex_unlock(%p)\n", thr->tid, addr);
- CHECK_NE(jctx, 0);
- CHECK_GE(addr, jctx->heap_begin);
- CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
+ JAVA_FUNC_ENTER(__tsan_java_mutex_unlock);
+ DPrintf("#%d: java_mutex_unlock(0x%zx)\n", thr->tid, addr);
+ DCHECK_NE(jctx, 0);
+ DCHECK_GE(addr, jctx->heap_begin);
+ DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
- MutexUnlock(thr, pc, addr);
+ MutexUnlock(thr, 0, addr);
}
void __tsan_java_mutex_read_lock(jptr addr) {
- SCOPED_JAVA_FUNC(__tsan_java_mutex_read_lock);
- DPrintf("#%d: java_mutex_read_lock(%p)\n", thr->tid, addr);
- CHECK_NE(jctx, 0);
- CHECK_GE(addr, jctx->heap_begin);
- CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
-
- MutexPostReadLock(thr, pc, addr, MutexFlagLinkerInit |
- MutexFlagWriteReentrant | MutexFlagDoPreLockOnPostLock);
+ JAVA_FUNC_ENTER(__tsan_java_mutex_read_lock);
+ DPrintf("#%d: java_mutex_read_lock(0x%zx)\n", thr->tid, addr);
+ DCHECK_NE(jctx, 0);
+ DCHECK_GE(addr, jctx->heap_begin);
+ DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
+
+ MutexPostReadLock(thr, 0, addr,
+ MutexFlagLinkerInit | MutexFlagWriteReentrant |
+ MutexFlagDoPreLockOnPostLock);
}
void __tsan_java_mutex_read_unlock(jptr addr) {
- SCOPED_JAVA_FUNC(__tsan_java_mutex_read_unlock);
- DPrintf("#%d: java_mutex_read_unlock(%p)\n", thr->tid, addr);
- CHECK_NE(jctx, 0);
- CHECK_GE(addr, jctx->heap_begin);
- CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
+ JAVA_FUNC_ENTER(__tsan_java_mutex_read_unlock);
+ DPrintf("#%d: java_mutex_read_unlock(0x%zx)\n", thr->tid, addr);
+ DCHECK_NE(jctx, 0);
+ DCHECK_GE(addr, jctx->heap_begin);
+ DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
- MutexReadUnlock(thr, pc, addr);
+ MutexReadUnlock(thr, 0, addr);
}
void __tsan_java_mutex_lock_rec(jptr addr, int rec) {
- SCOPED_JAVA_FUNC(__tsan_java_mutex_lock_rec);
- DPrintf("#%d: java_mutex_lock_rec(%p, %d)\n", thr->tid, addr, rec);
- CHECK_NE(jctx, 0);
- CHECK_GE(addr, jctx->heap_begin);
- CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
- CHECK_GT(rec, 0);
-
- MutexPostLock(thr, pc, addr, MutexFlagLinkerInit | MutexFlagWriteReentrant |
- MutexFlagDoPreLockOnPostLock | MutexFlagRecursiveLock, rec);
+ JAVA_FUNC_ENTER(__tsan_java_mutex_lock_rec);
+ DPrintf("#%d: java_mutex_lock_rec(0x%zx, %d)\n", thr->tid, addr, rec);
+ DCHECK_NE(jctx, 0);
+ DCHECK_GE(addr, jctx->heap_begin);
+ DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
+ DCHECK_GT(rec, 0);
+
+ MutexPostLock(thr, 0, addr,
+ MutexFlagLinkerInit | MutexFlagWriteReentrant |
+ MutexFlagDoPreLockOnPostLock | MutexFlagRecursiveLock,
+ rec);
}
int __tsan_java_mutex_unlock_rec(jptr addr) {
- SCOPED_JAVA_FUNC(__tsan_java_mutex_unlock_rec);
- DPrintf("#%d: java_mutex_unlock_rec(%p)\n", thr->tid, addr);
- CHECK_NE(jctx, 0);
- CHECK_GE(addr, jctx->heap_begin);
- CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
+ JAVA_FUNC_ENTER(__tsan_java_mutex_unlock_rec);
+ DPrintf("#%d: java_mutex_unlock_rec(0x%zx)\n", thr->tid, addr);
+ DCHECK_NE(jctx, 0);
+ DCHECK_GE(addr, jctx->heap_begin);
+ DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
- return MutexUnlock(thr, pc, addr, MutexFlagRecursiveUnlock);
+ return MutexUnlock(thr, 0, addr, MutexFlagRecursiveUnlock);
}
void __tsan_java_acquire(jptr addr) {
- SCOPED_JAVA_FUNC(__tsan_java_acquire);
- DPrintf("#%d: java_acquire(%p)\n", thr->tid, addr);
- CHECK_NE(jctx, 0);
- CHECK_GE(addr, jctx->heap_begin);
- CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
+ JAVA_FUNC_ENTER(__tsan_java_acquire);
+ DPrintf("#%d: java_acquire(0x%zx)\n", thr->tid, addr);
+ DCHECK_NE(jctx, 0);
+ DCHECK_GE(addr, jctx->heap_begin);
+ DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
- Acquire(thr, caller_pc, addr);
+ Acquire(thr, 0, addr);
}
void __tsan_java_release(jptr addr) {
- SCOPED_JAVA_FUNC(__tsan_java_release);
- DPrintf("#%d: java_release(%p)\n", thr->tid, addr);
- CHECK_NE(jctx, 0);
- CHECK_GE(addr, jctx->heap_begin);
- CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
+ JAVA_FUNC_ENTER(__tsan_java_release);
+ DPrintf("#%d: java_release(0x%zx)\n", thr->tid, addr);
+ DCHECK_NE(jctx, 0);
+ DCHECK_GE(addr, jctx->heap_begin);
+ DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
- Release(thr, caller_pc, addr);
+ Release(thr, 0, addr);
}
void __tsan_java_release_store(jptr addr) {
- SCOPED_JAVA_FUNC(__tsan_java_release);
- DPrintf("#%d: java_release_store(%p)\n", thr->tid, addr);
- CHECK_NE(jctx, 0);
- CHECK_GE(addr, jctx->heap_begin);
- CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
+ JAVA_FUNC_ENTER(__tsan_java_release);
+ DPrintf("#%d: java_release_store(0x%zx)\n", thr->tid, addr);
+ DCHECK_NE(jctx, 0);
+ DCHECK_GE(addr, jctx->heap_begin);
+ DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
- ReleaseStore(thr, caller_pc, addr);
+ ReleaseStore(thr, 0, addr);
}
//===----------------------------------------------------------------------===//
#include "sanitizer_common/sanitizer_platform.h"
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
#include "sanitizer_common/sanitizer_errno.h"
#include "tsan_interceptors.h"
#include "tsan_report.h"
#include "tsan_flags.h"
-// May be overriden by front-end.
-SANITIZER_WEAK_DEFAULT_IMPL
-void __sanitizer_malloc_hook(void *ptr, uptr size) {
- (void)ptr;
- (void)size;
-}
-
-SANITIZER_WEAK_DEFAULT_IMPL
-void __sanitizer_free_hook(void *ptr) {
- (void)ptr;
-}
-
namespace __tsan {
struct MapUnmapCallback {
struct GlobalProc {
Mutex mtx;
Processor *proc;
-
- GlobalProc() : mtx(MutexTypeGlobalProc), proc(ProcCreate()) {}
+ // This mutex represents the internal allocator combined for
+ // the purposes of deadlock detection. The internal allocator
+ // uses multiple mutexes, moreover they are locked only occasionally
+ // and they are spin mutexes which don't support deadlock detection.
+ // So we use this fake mutex to serve as a substitute for these mutexes.
+ CheckedMutex internal_alloc_mtx;
+
+ GlobalProc()
+ : mtx(MutexTypeGlobalProc),
+ proc(ProcCreate()),
+ internal_alloc_mtx(MutexTypeInternalAlloc) {}
};
static char global_proc_placeholder[sizeof(GlobalProc)] ALIGNED(64);
return reinterpret_cast<GlobalProc*>(&global_proc_placeholder);
}
+static void InternalAllocAccess() {
+ global_proc()->internal_alloc_mtx.Lock();
+ global_proc()->internal_alloc_mtx.Unlock();
+}
+
ScopedGlobalProcessor::ScopedGlobalProcessor() {
GlobalProc *gp = global_proc();
ThreadState *thr = cur_thread();
gp->mtx.Unlock();
}
+void AllocatorLock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
+ global_proc()->internal_alloc_mtx.Lock();
+ InternalAllocatorLock();
+}
+
+void AllocatorUnlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
+ InternalAllocatorUnlock();
+ global_proc()->internal_alloc_mtx.Unlock();
+}
+
+void GlobalProcessorLock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
+ global_proc()->mtx.Lock();
+}
+
+void GlobalProcessorUnlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
+ global_proc()->mtx.Unlock();
+}
+
static constexpr uptr kMaxAllowedMallocSize = 1ull << 40;
static uptr max_user_defined_malloc_size;
ObtainCurrentStack(thr, pc, &stack);
if (IsFiredSuppression(ctx, ReportTypeSignalUnsafe, stack))
return;
- ThreadRegistryLock l(ctx->thread_registry);
+ ThreadRegistryLock l(&ctx->thread_registry);
ScopedReport rep(ReportTypeSignalUnsafe);
rep.AddStack(stack, true);
OutputReport(thr, rep);
GET_STACK_TRACE_FATAL(thr, pc);
ReportAllocationSizeTooBig(sz, malloc_limit, &stack);
}
+ if (UNLIKELY(IsRssLimitExceeded())) {
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ GET_STACK_TRACE_FATAL(thr, pc);
+ ReportRssLimitExceeded(&stack);
+ }
void *p = allocator()->Allocate(&thr->proc()->alloc_cache, sz, align);
if (UNLIKELY(!p)) {
SetAllocatorOutOfMemory();
}
void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write) {
- DPrintf("#%d: alloc(%zu) = %p\n", thr->tid, sz, p);
+ DPrintf("#%d: alloc(%zu) = 0x%zx\n", thr->tid, sz, p);
+ // Note: this can run before thread initialization/after finalization.
+ // As a result this is not necessarily synchronized with DoReset,
+ // which iterates over and resets all sync objects,
+ // but it is fine to create new MBlocks in this context.
ctx->metamap.AllocBlock(thr, pc, p, sz);
- if (write && thr->ignore_reads_and_writes == 0)
+ // If this runs before thread initialization/after finalization
+ // and we don't have trace initialized, we can't imitate writes.
+ // In such case just reset the shadow range, it is fine since
+ // it affects only a small fraction of special objects.
+ if (write && thr->ignore_reads_and_writes == 0 &&
+ atomic_load_relaxed(&thr->trace_pos))
MemoryRangeImitateWrite(thr, pc, (uptr)p, sz);
else
MemoryResetRange(thr, pc, (uptr)p, sz);
void OnUserFree(ThreadState *thr, uptr pc, uptr p, bool write) {
CHECK_NE(p, (void*)0);
- uptr sz = ctx->metamap.FreeBlock(thr->proc(), p);
- DPrintf("#%d: free(%p, %zu)\n", thr->tid, p, sz);
+ if (!thr->slot) {
+ // Very early/late in thread lifetime, or during fork.
+ UNUSED uptr sz = ctx->metamap.FreeBlock(thr->proc(), p, false);
+ DPrintf("#%d: free(0x%zx, %zu) (no slot)\n", thr->tid, p, sz);
+ return;
+ }
+ SlotLocker locker(thr);
+ uptr sz = ctx->metamap.FreeBlock(thr->proc(), p, true);
+ DPrintf("#%d: free(0x%zx, %zu)\n", thr->tid, p, sz);
if (write && thr->ignore_reads_and_writes == 0)
MemoryRangeFreed(thr, pc, (uptr)p, sz);
}
}
uptr user_alloc_usable_size(const void *p) {
- if (p == 0)
+ if (p == 0 || !IsAppMem((uptr)p))
return 0;
MBlock *b = ctx->metamap.GetBlock((uptr)p);
if (!b)
ThreadState *thr = cur_thread();
if (ctx == 0 || !ctx->initialized || thr->ignore_interceptors)
return;
- __sanitizer_malloc_hook(ptr, size);
RunMallocHooks(ptr, size);
}
ThreadState *thr = cur_thread();
if (ctx == 0 || !ctx->initialized || thr->ignore_interceptors)
return;
- __sanitizer_free_hook(ptr);
RunFreeHooks(ptr);
}
-void *internal_alloc(MBlockType typ, uptr sz) {
+void *Alloc(uptr sz) {
ThreadState *thr = cur_thread();
if (thr->nomalloc) {
thr->nomalloc = 0; // CHECK calls internal_malloc().
CHECK(0);
}
+ InternalAllocAccess();
return InternalAlloc(sz, &thr->proc()->internal_alloc_cache);
}
-void internal_free(void *p) {
+void FreeImpl(void *p) {
ThreadState *thr = cur_thread();
if (thr->nomalloc) {
thr->nomalloc = 0; // CHECK calls internal_malloc().
CHECK(0);
}
+ InternalAllocAccess();
InternalFree(p, &thr->proc()->internal_alloc_cache);
}
void __tsan_on_thread_idle() {
ThreadState *thr = cur_thread();
- thr->clock.ResetCached(&thr->proc()->clock_cache);
- thr->last_sleep_clock.ResetCached(&thr->proc()->clock_cache);
allocator()->SwallowCache(&thr->proc()->alloc_cache);
internal_allocator()->SwallowCache(&thr->proc()->internal_alloc_cache);
ctx->metamap.OnProcIdle(thr->proc());
void AllocatorProcStart(Processor *proc);
void AllocatorProcFinish(Processor *proc);
void AllocatorPrintStats();
+void AllocatorLock();
+void AllocatorUnlock();
+void GlobalProcessorLock();
+void GlobalProcessorUnlock();
// For user allocations.
void *user_alloc_internal(ThreadState *thr, uptr pc, uptr sz,
void invoke_malloc_hook(void *ptr, uptr size);
void invoke_free_hook(void *ptr);
-enum MBlockType {
- MBlockScopedBuf,
- MBlockString,
- MBlockStackTrace,
- MBlockShadowStack,
- MBlockSync,
- MBlockClock,
- MBlockThreadContex,
- MBlockDeadInfo,
- MBlockRacyStacks,
- MBlockRacyAddresses,
- MBlockAtExit,
- MBlockFlag,
- MBlockReport,
- MBlockReportMop,
- MBlockReportThread,
- MBlockReportMutex,
- MBlockReportLoc,
- MBlockReportStack,
- MBlockSuppression,
- MBlockExpectRace,
- MBlockSignal,
- MBlockJmpBuf,
+// For internal data structures.
+void *Alloc(uptr sz);
+void FreeImpl(void *p);
- // This must be the last.
- MBlockTypeCount
-};
+template <typename T, typename... Args>
+T *New(Args &&...args) {
+ return new (Alloc(sizeof(T))) T(static_cast<Args &&>(args)...);
+}
-// For internal data structures.
-void *internal_alloc(MBlockType typ, uptr sz);
-void internal_free(void *p);
+template <typename T>
+void Free(T *&p) {
+ if (p == nullptr)
+ return;
+ FreeImpl(p);
+ p = nullptr;
+}
template <typename T>
-void DestroyAndFree(T *p) {
+void DestroyAndFree(T *&p) {
+ if (p == nullptr)
+ return;
p->~T();
- internal_free(p);
+ Free(p);
}
} // namespace __tsan
//
//===----------------------------------------------------------------------===//
#include "tsan_mutexset.h"
+
+#include "sanitizer_common/sanitizer_placement_new.h"
#include "tsan_rtl.h"
namespace __tsan {
-const uptr MutexSet::kMaxSize;
-
MutexSet::MutexSet() {
- size_ = 0;
- internal_memset(&descs_, 0, sizeof(descs_));
}
-void MutexSet::Add(u64 id, bool write, u64 epoch) {
+void MutexSet::Reset() { internal_memset(this, 0, sizeof(*this)); }
+
+void MutexSet::AddAddr(uptr addr, StackID stack_id, bool write) {
// Look up existing mutex with the same id.
for (uptr i = 0; i < size_; i++) {
- if (descs_[i].id == id) {
+ if (descs_[i].addr == addr) {
descs_[i].count++;
- descs_[i].epoch = epoch;
+ descs_[i].seq = seq_++;
return;
}
}
// On overflow, find the oldest mutex and drop it.
if (size_ == kMaxSize) {
- u64 minepoch = (u64)-1;
- u64 mini = (u64)-1;
+ uptr min = 0;
for (uptr i = 0; i < size_; i++) {
- if (descs_[i].epoch < minepoch) {
- minepoch = descs_[i].epoch;
- mini = i;
- }
+ if (descs_[i].seq < descs_[min].seq)
+ min = i;
}
- RemovePos(mini);
+ RemovePos(min);
CHECK_EQ(size_, kMaxSize - 1);
}
// Add new mutex descriptor.
- descs_[size_].id = id;
+ descs_[size_].addr = addr;
+ descs_[size_].stack_id = stack_id;
descs_[size_].write = write;
- descs_[size_].epoch = epoch;
+ descs_[size_].seq = seq_++;
descs_[size_].count = 1;
size_++;
}
-void MutexSet::Del(u64 id, bool write) {
+void MutexSet::DelAddr(uptr addr, bool destroy) {
for (uptr i = 0; i < size_; i++) {
- if (descs_[i].id == id) {
- if (--descs_[i].count == 0)
+ if (descs_[i].addr == addr) {
+ if (destroy || --descs_[i].count == 0)
RemovePos(i);
return;
}
}
}
-void MutexSet::Remove(u64 id) {
- for (uptr i = 0; i < size_; i++) {
- if (descs_[i].id == id) {
- RemovePos(i);
- return;
- }
- }
-}
-
void MutexSet::RemovePos(uptr i) {
CHECK_LT(i, size_);
descs_[i] = descs_[size_ - 1];
return descs_[i];
}
+DynamicMutexSet::DynamicMutexSet() : ptr_(New<MutexSet>()) {}
+DynamicMutexSet::~DynamicMutexSet() { DestroyAndFree(ptr_); }
+
} // namespace __tsan
public:
// Holds limited number of mutexes.
// The oldest mutexes are discarded on overflow.
- static const uptr kMaxSize = 16;
+ static constexpr uptr kMaxSize = 16;
struct Desc {
- u64 id;
- u64 epoch;
- int count;
+ uptr addr;
+ StackID stack_id;
+ u32 seq;
+ u32 count;
bool write;
+
+ Desc() { internal_memset(this, 0, sizeof(*this)); }
+ Desc(const Desc& other) { *this = other; }
+ Desc& operator=(const MutexSet::Desc& other) {
+ internal_memcpy(this, &other, sizeof(*this));
+ return *this;
+ }
};
MutexSet();
- // The 'id' is obtained from SyncVar::GetId().
- void Add(u64 id, bool write, u64 epoch);
- void Del(u64 id, bool write);
- void Remove(u64 id); // Removes the mutex completely (if it's destroyed).
+ void Reset();
+ void AddAddr(uptr addr, StackID stack_id, bool write);
+ void DelAddr(uptr addr, bool destroy = false);
uptr Size() const;
Desc Get(uptr i) const;
- void operator=(const MutexSet &other) {
- internal_memcpy(this, &other, sizeof(*this));
- }
-
private:
#if !SANITIZER_GO
- uptr size_;
+ u32 seq_ = 0;
+ uptr size_ = 0;
Desc descs_[kMaxSize];
-#endif
void RemovePos(uptr i);
- MutexSet(const MutexSet&);
+#endif
+};
+
+// MutexSet is too large to live on stack.
+// DynamicMutexSet can be use used to create local MutexSet's.
+class DynamicMutexSet {
+ public:
+ DynamicMutexSet();
+ ~DynamicMutexSet();
+ MutexSet* operator->() { return ptr_; }
+ operator MutexSet*() { return ptr_; }
+ DynamicMutexSet(const DynamicMutexSet&) = delete;
+ DynamicMutexSet& operator=(const DynamicMutexSet&) = delete;
+
+ private:
+ MutexSet* ptr_;
+#if SANITIZER_GO
+ MutexSet set_;
+#endif
};
// Go does not have mutexes, so do not spend memory and time.
// in different goroutine).
#if SANITIZER_GO
MutexSet::MutexSet() {}
-void MutexSet::Add(u64 id, bool write, u64 epoch) {}
-void MutexSet::Del(u64 id, bool write) {}
-void MutexSet::Remove(u64 id) {}
-void MutexSet::RemovePos(uptr i) {}
+void MutexSet::Reset() {}
+void MutexSet::AddAddr(uptr addr, StackID stack_id, bool write) {}
+void MutexSet::DelAddr(uptr addr, bool destroy) {}
uptr MutexSet::Size() const { return 0; }
MutexSet::Desc MutexSet::Get(uptr i) const { return Desc(); }
+DynamicMutexSet::DynamicMutexSet() : ptr_(&set_) {}
+DynamicMutexSet::~DynamicMutexSet() {}
#endif
} // namespace __tsan
# error "Only 64-bit is supported"
#endif
+#include "sanitizer_common/sanitizer_common.h"
#include "tsan_defs.h"
-#include "tsan_trace.h"
namespace __tsan {
-#if defined(__x86_64__)
-#define HAS_48_BIT_ADDRESS_SPACE 1
-#elif SANITIZER_IOSSIM // arm64 iOS simulators (order of #if matters)
-#define HAS_48_BIT_ADDRESS_SPACE 1
-#elif SANITIZER_IOS // arm64 iOS devices (order of #if matters)
-#define HAS_48_BIT_ADDRESS_SPACE 0
-#elif SANITIZER_MAC // arm64 macOS (order of #if matters)
-#define HAS_48_BIT_ADDRESS_SPACE 1
-#else
-#define HAS_48_BIT_ADDRESS_SPACE 0
-#endif
-
-#if !SANITIZER_GO
+enum {
+ // App memory is not mapped onto shadow memory range.
+ kBrokenMapping = 1 << 0,
+ // Mapping app memory and back does not produce the same address,
+ // this can lead to wrong addresses in reports and potentially
+ // other bad consequences.
+ kBrokenReverseMapping = 1 << 1,
+ // Mapping is non-linear for linear user range.
+ // This is bad and can lead to unpredictable memory corruptions, etc
+ // because range access functions assume linearity.
+ kBrokenLinearity = 1 << 2,
+ // Meta for an app region overlaps with the meta of another app region.
+ // This is determined by recomputing the individual meta regions for
+ // each app region.
+ //
+ // N.B. There is no "kBrokenReverseMetaMapping" constant because there
+ // is no MetaToMem function. However, note that (!kBrokenLinearity
+ // && !kBrokenAliasedMetas) implies that MemToMeta is invertible.
+ kBrokenAliasedMetas = 1 << 3,
+};
-#if HAS_48_BIT_ADDRESS_SPACE
/*
C/C++ on linux/x86_64 and freebsd/x86_64
0000 0000 1000 - 0080 0000 0000: main binary and/or MAP_32BIT mappings (512GB)
0040 0000 0000 - 0100 0000 0000: -
-0100 0000 0000 - 2000 0000 0000: shadow
-2000 0000 0000 - 3000 0000 0000: -
+0100 0000 0000 - 1000 0000 0000: shadow
+1000 0000 0000 - 3000 0000 0000: -
3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
4000 0000 0000 - 5500 0000 0000: -
5500 0000 0000 - 5680 0000 0000: pie binaries without ASLR or on 4.1+ kernels
-5680 0000 0000 - 6000 0000 0000: -
-6000 0000 0000 - 6200 0000 0000: traces
-6200 0000 0000 - 7d00 0000 0000: -
+5680 0000 0000 - 7d00 0000 0000: -
7b00 0000 0000 - 7c00 0000 0000: heap
7c00 0000 0000 - 7e80 0000 0000: -
7e80 0000 0000 - 8000 0000 0000: modules and main thread stack
* Stack on NetBSD/amd64 has prereserved 128MB.
* Heap grows downwards (top-down).
* ASLR must be disabled per-process or globally.
-
*/
-struct Mapping {
+struct Mapping48AddressSpace {
static const uptr kMetaShadowBeg = 0x300000000000ull;
static const uptr kMetaShadowEnd = 0x340000000000ull;
- static const uptr kTraceMemBeg = 0x600000000000ull;
- static const uptr kTraceMemEnd = 0x620000000000ull;
static const uptr kShadowBeg = 0x010000000000ull;
- static const uptr kShadowEnd = 0x200000000000ull;
+ static const uptr kShadowEnd = 0x100000000000ull;
static const uptr kHeapMemBeg = 0x7b0000000000ull;
static const uptr kHeapMemEnd = 0x7c0000000000ull;
static const uptr kLoAppMemBeg = 0x000000001000ull;
static const uptr kMidAppMemEnd = 0x568000000000ull;
static const uptr kHiAppMemBeg = 0x7e8000000000ull;
static const uptr kHiAppMemEnd = 0x800000000000ull;
- static const uptr kAppMemMsk = 0x780000000000ull;
- static const uptr kAppMemXor = 0x040000000000ull;
+ static const uptr kShadowMsk = 0x780000000000ull;
+ static const uptr kShadowXor = 0x040000000000ull;
+ static const uptr kShadowAdd = 0x000000000000ull;
static const uptr kVdsoBeg = 0xf000000000000000ull;
};
-#define TSAN_MID_APP_RANGE 1
-#elif defined(__mips64)
/*
C/C++ on linux/mips64 (40-bit VMA)
0000 0000 00 - 0100 0000 00: - (4 GB)
0100 0000 00 - 0200 0000 00: main binary (4 GB)
-0200 0000 00 - 2000 0000 00: - (120 GB)
-2000 0000 00 - 4000 0000 00: shadow (128 GB)
+0200 0000 00 - 1200 0000 00: - (64 GB)
+1200 0000 00 - 2200 0000 00: shadow (64 GB)
+2200 0000 00 - 4000 0000 00: - (120 GB)
4000 0000 00 - 5000 0000 00: metainfo (memory blocks and sync objects) (64 GB)
5000 0000 00 - aa00 0000 00: - (360 GB)
aa00 0000 00 - ab00 0000 00: main binary (PIE) (4 GB)
-ab00 0000 00 - b000 0000 00: - (20 GB)
-b000 0000 00 - b200 0000 00: traces (8 GB)
-b200 0000 00 - fe00 0000 00: - (304 GB)
+ab00 0000 00 - fe00 0000 00: - (332 GB)
fe00 0000 00 - ff00 0000 00: heap (4 GB)
ff00 0000 00 - ff80 0000 00: - (2 GB)
ff80 0000 00 - ffff ffff ff: modules and main thread stack (<2 GB)
*/
-struct Mapping40 {
+struct MappingMips64_40 {
static const uptr kMetaShadowBeg = 0x4000000000ull;
static const uptr kMetaShadowEnd = 0x5000000000ull;
- static const uptr kTraceMemBeg = 0xb000000000ull;
- static const uptr kTraceMemEnd = 0xb200000000ull;
- static const uptr kShadowBeg = 0x2000000000ull;
- static const uptr kShadowEnd = 0x4000000000ull;
+ static const uptr kShadowBeg = 0x1200000000ull;
+ static const uptr kShadowEnd = 0x2200000000ull;
static const uptr kHeapMemBeg = 0xfe00000000ull;
static const uptr kHeapMemEnd = 0xff00000000ull;
static const uptr kLoAppMemBeg = 0x0100000000ull;
static const uptr kMidAppMemEnd = 0xab00000000ull;
static const uptr kHiAppMemBeg = 0xff80000000ull;
static const uptr kHiAppMemEnd = 0xffffffffffull;
- static const uptr kAppMemMsk = 0xf800000000ull;
- static const uptr kAppMemXor = 0x0800000000ull;
+ static const uptr kShadowMsk = 0xf800000000ull;
+ static const uptr kShadowXor = 0x0800000000ull;
+ static const uptr kShadowAdd = 0x0000000000ull;
static const uptr kVdsoBeg = 0xfffff00000ull;
};
-#define TSAN_MID_APP_RANGE 1
-#define TSAN_RUNTIME_VMA 1
-#elif defined(__aarch64__) && defined(__APPLE__)
/*
C/C++ on Darwin/iOS/ARM64 (36-bit VMA, 64 GB VM)
0000 0000 00 - 0100 0000 00: - (4 GB)
0100 0000 00 - 0200 0000 00: main binary, modules, thread stacks (4 GB)
0200 0000 00 - 0300 0000 00: heap (4 GB)
0300 0000 00 - 0400 0000 00: - (4 GB)
-0400 0000 00 - 0c00 0000 00: shadow memory (32 GB)
-0c00 0000 00 - 0d00 0000 00: - (4 GB)
+0400 0000 00 - 0800 0000 00: shadow memory (16 GB)
+0800 0000 00 - 0d00 0000 00: - (20 GB)
0d00 0000 00 - 0e00 0000 00: metainfo (4 GB)
-0e00 0000 00 - 0f00 0000 00: - (4 GB)
-0f00 0000 00 - 0fc0 0000 00: traces (3 GB)
-0fc0 0000 00 - 1000 0000 00: -
+0e00 0000 00 - 1000 0000 00: -
*/
-struct Mapping {
+struct MappingAppleAarch64 {
static const uptr kLoAppMemBeg = 0x0100000000ull;
static const uptr kLoAppMemEnd = 0x0200000000ull;
static const uptr kHeapMemBeg = 0x0200000000ull;
static const uptr kHeapMemEnd = 0x0300000000ull;
static const uptr kShadowBeg = 0x0400000000ull;
- static const uptr kShadowEnd = 0x0c00000000ull;
+ static const uptr kShadowEnd = 0x0800000000ull;
static const uptr kMetaShadowBeg = 0x0d00000000ull;
static const uptr kMetaShadowEnd = 0x0e00000000ull;
- static const uptr kTraceMemBeg = 0x0f00000000ull;
- static const uptr kTraceMemEnd = 0x0fc0000000ull;
static const uptr kHiAppMemBeg = 0x0fc0000000ull;
static const uptr kHiAppMemEnd = 0x0fc0000000ull;
- static const uptr kAppMemMsk = 0x0ull;
- static const uptr kAppMemXor = 0x0ull;
+ static const uptr kShadowMsk = 0x0ull;
+ static const uptr kShadowXor = 0x0ull;
+ static const uptr kShadowAdd = 0x0200000000ull;
static const uptr kVdsoBeg = 0x7000000000000000ull;
+ static const uptr kMidAppMemBeg = 0;
+ static const uptr kMidAppMemEnd = 0;
};
-#elif defined(__aarch64__) && !defined(__APPLE__)
-// AArch64 supports multiple VMA which leads to multiple address transformation
-// functions. To support these multiple VMAS transformations and mappings TSAN
-// runtime for AArch64 uses an external memory read (vmaSize) to select which
-// mapping to use. Although slower, it make a same instrumented binary run on
-// multiple kernels.
-
/*
C/C++ on linux/aarch64 (39-bit VMA)
-0000 0010 00 - 0100 0000 00: main binary
-0100 0000 00 - 0800 0000 00: -
-0800 0000 00 - 2000 0000 00: shadow memory
-2000 0000 00 - 3100 0000 00: -
-3100 0000 00 - 3400 0000 00: metainfo
-3400 0000 00 - 5500 0000 00: -
-5500 0000 00 - 5600 0000 00: main binary (PIE)
-5600 0000 00 - 6000 0000 00: -
-6000 0000 00 - 6200 0000 00: traces
-6200 0000 00 - 7d00 0000 00: -
-7c00 0000 00 - 7d00 0000 00: heap
-7d00 0000 00 - 7fff ffff ff: modules and main thread stack
+0000 0010 00 - 0500 0000 00: main binary (20 GB)
+0100 0000 00 - 2000 0000 00: -
+2000 0000 00 - 4000 0000 00: shadow memory (128 GB)
+4000 0000 00 - 4800 0000 00: metainfo (32 GB)
+4800 0000 00 - 5500 0000 00: -
+5500 0000 00 - 5a00 0000 00: main binary (PIE) (20 GB)
+5600 0000 00 - 7c00 0000 00: -
+7a00 0000 00 - 7d00 0000 00: heap (12 GB)
+7d00 0000 00 - 7fff ffff ff: modules and main thread stack (12 GB)
*/
-struct Mapping39 {
+struct MappingAarch64_39 {
static const uptr kLoAppMemBeg = 0x0000001000ull;
- static const uptr kLoAppMemEnd = 0x0100000000ull;
- static const uptr kShadowBeg = 0x0800000000ull;
- static const uptr kShadowEnd = 0x2000000000ull;
- static const uptr kMetaShadowBeg = 0x3100000000ull;
- static const uptr kMetaShadowEnd = 0x3400000000ull;
+ static const uptr kLoAppMemEnd = 0x0500000000ull;
+ static const uptr kShadowBeg = 0x2000000000ull;
+ static const uptr kShadowEnd = 0x4000000000ull;
+ static const uptr kMetaShadowBeg = 0x4000000000ull;
+ static const uptr kMetaShadowEnd = 0x4800000000ull;
static const uptr kMidAppMemBeg = 0x5500000000ull;
- static const uptr kMidAppMemEnd = 0x5600000000ull;
- static const uptr kTraceMemBeg = 0x6000000000ull;
- static const uptr kTraceMemEnd = 0x6200000000ull;
- static const uptr kHeapMemBeg = 0x7c00000000ull;
+ static const uptr kMidAppMemEnd = 0x5a00000000ull;
+ static const uptr kHeapMemBeg = 0x7a00000000ull;
static const uptr kHeapMemEnd = 0x7d00000000ull;
- static const uptr kHiAppMemBeg = 0x7e00000000ull;
+ static const uptr kHiAppMemBeg = 0x7d00000000ull;
static const uptr kHiAppMemEnd = 0x7fffffffffull;
- static const uptr kAppMemMsk = 0x7800000000ull;
- static const uptr kAppMemXor = 0x0200000000ull;
+ static const uptr kShadowMsk = 0x7000000000ull;
+ static const uptr kShadowXor = 0x1000000000ull;
+ static const uptr kShadowAdd = 0x0000000000ull;
static const uptr kVdsoBeg = 0x7f00000000ull;
};
/*
C/C++ on linux/aarch64 (42-bit VMA)
-00000 0010 00 - 01000 0000 00: main binary
-01000 0000 00 - 10000 0000 00: -
-10000 0000 00 - 20000 0000 00: shadow memory
-20000 0000 00 - 26000 0000 00: -
-26000 0000 00 - 28000 0000 00: metainfo
-28000 0000 00 - 2aa00 0000 00: -
-2aa00 0000 00 - 2ab00 0000 00: main binary (PIE)
-2ab00 0000 00 - 36200 0000 00: -
-36200 0000 00 - 36240 0000 00: traces
-36240 0000 00 - 3e000 0000 00: -
-3e000 0000 00 - 3f000 0000 00: heap
-3f000 0000 00 - 3ffff ffff ff: modules and main thread stack
+00000 0010 00 - 02000 0000 00: main binary (128 GB)
+02000 0000 00 - 08000 0000 00: -
+10000 0000 00 - 20000 0000 00: shadow memory (1024 GB)
+20000 0000 00 - 24000 0000 00: metainfo (256 GB)
+24000 0000 00 - 2aa00 0000 00: -
+2aa00 0000 00 - 2c000 0000 00: main binary (PIE) (88 GB)
+2c000 0000 00 - 3c000 0000 00: -
+3c000 0000 00 - 3f000 0000 00: heap (192 GB)
+3f000 0000 00 - 3ffff ffff ff: modules and main thread stack (64 GB)
*/
-struct Mapping42 {
+struct MappingAarch64_42 {
static const uptr kLoAppMemBeg = 0x00000001000ull;
- static const uptr kLoAppMemEnd = 0x01000000000ull;
+ static const uptr kLoAppMemEnd = 0x02000000000ull;
static const uptr kShadowBeg = 0x10000000000ull;
static const uptr kShadowEnd = 0x20000000000ull;
- static const uptr kMetaShadowBeg = 0x26000000000ull;
- static const uptr kMetaShadowEnd = 0x28000000000ull;
+ static const uptr kMetaShadowBeg = 0x20000000000ull;
+ static const uptr kMetaShadowEnd = 0x24000000000ull;
static const uptr kMidAppMemBeg = 0x2aa00000000ull;
- static const uptr kMidAppMemEnd = 0x2ab00000000ull;
- static const uptr kTraceMemBeg = 0x36200000000ull;
- static const uptr kTraceMemEnd = 0x36400000000ull;
- static const uptr kHeapMemBeg = 0x3e000000000ull;
+ static const uptr kMidAppMemEnd = 0x2c000000000ull;
+ static const uptr kHeapMemBeg = 0x3c000000000ull;
static const uptr kHeapMemEnd = 0x3f000000000ull;
static const uptr kHiAppMemBeg = 0x3f000000000ull;
static const uptr kHiAppMemEnd = 0x3ffffffffffull;
- static const uptr kAppMemMsk = 0x3c000000000ull;
- static const uptr kAppMemXor = 0x04000000000ull;
+ static const uptr kShadowMsk = 0x38000000000ull;
+ static const uptr kShadowXor = 0x08000000000ull;
+ static const uptr kShadowAdd = 0x00000000000ull;
static const uptr kVdsoBeg = 0x37f00000000ull;
};
-struct Mapping48 {
+/*
+C/C++ on linux/aarch64 (48-bit VMA)
+0000 0000 1000 - 0a00 0000 0000: main binary (10240 GB)
+0a00 0000 1000 - 1554 0000 0000: -
+1554 0000 1000 - 5400 0000 0000: shadow memory (64176 GB)
+5400 0000 1000 - 8000 0000 0000: -
+8000 0000 1000 - 0a00 0000 0000: metainfo (32768 GB)
+a000 0000 1000 - aaaa 0000 0000: -
+aaaa 0000 1000 - ac00 0000 0000: main binary (PIE) (1368 GB)
+ac00 0000 1000 - fc00 0000 0000: -
+fc00 0000 1000 - ffff ffff ffff: modules and main thread stack (4096 GB)
+
+N.B. the shadow memory region has a strange start address, because it
+contains the shadows for the mid, high and low app regions (in this
+unusual order).
+*/
+struct MappingAarch64_48 {
static const uptr kLoAppMemBeg = 0x0000000001000ull;
- static const uptr kLoAppMemEnd = 0x0000200000000ull;
- static const uptr kShadowBeg = 0x0002000000000ull;
- static const uptr kShadowEnd = 0x0004000000000ull;
- static const uptr kMetaShadowBeg = 0x0005000000000ull;
- static const uptr kMetaShadowEnd = 0x0006000000000ull;
+ static const uptr kLoAppMemEnd = 0x00a0000000000ull;
+ static const uptr kShadowBeg = 0x0155400000000ull;
+ static const uptr kShadowEnd = 0x0540000000000ull;
+ static const uptr kMetaShadowBeg = 0x0800000000000ull;
+ static const uptr kMetaShadowEnd = 0x0a00000000000ull;
static const uptr kMidAppMemBeg = 0x0aaaa00000000ull;
- static const uptr kMidAppMemEnd = 0x0aaaf00000000ull;
- static const uptr kTraceMemBeg = 0x0f06000000000ull;
- static const uptr kTraceMemEnd = 0x0f06200000000ull;
- static const uptr kHeapMemBeg = 0x0ffff00000000ull;
- static const uptr kHeapMemEnd = 0x0ffff00000000ull;
- static const uptr kHiAppMemBeg = 0x0ffff00000000ull;
+ static const uptr kMidAppMemEnd = 0x0ac0000000000ull;
+ static const uptr kHiAppMemBeg = 0x0fc0000000000ull;
static const uptr kHiAppMemEnd = 0x1000000000000ull;
- static const uptr kAppMemMsk = 0x0fff800000000ull;
- static const uptr kAppMemXor = 0x0000800000000ull;
+ static const uptr kHeapMemBeg = 0x0fc0000000000ull;
+ static const uptr kHeapMemEnd = 0x0fc0000000000ull;
+ static const uptr kShadowMsk = 0x0c00000000000ull;
+ static const uptr kShadowXor = 0x0200000000000ull;
+ static const uptr kShadowAdd = 0x0000000000000ull;
static const uptr kVdsoBeg = 0xffff000000000ull;
};
-// Indicates the runtime will define the memory regions at runtime.
-#define TSAN_RUNTIME_VMA 1
-// Indicates that mapping defines a mid range memory segment.
-#define TSAN_MID_APP_RANGE 1
-#elif defined(__powerpc64__)
-// PPC64 supports multiple VMA which leads to multiple address transformation
-// functions. To support these multiple VMAS transformations and mappings TSAN
-// runtime for PPC64 uses an external memory read (vmaSize) to select which
-// mapping to use. Although slower, it make a same instrumented binary run on
-// multiple kernels.
+/* C/C++ on linux/loongarch64 (47-bit VMA)
+0000 0000 4000 - 0080 0000 0000: main binary
+0080 0000 0000 - 0100 0000 0000: -
+0100 0000 0000 - 1000 0000 0000: shadow memory
+1000 0000 0000 - 3000 0000 0000: -
+3000 0000 0000 - 3400 0000 0000: metainfo
+3400 0000 0000 - 5555 0000 0000: -
+5555 0000 0000 - 5556 0000 0000: main binary (PIE)
+5556 0000 0000 - 7ffe 0000 0000: -
+7ffe 0000 0000 - 7fff 0000 0000: heap
+7fff 0000 0000 - 7fff 8000 0000: -
+7fff 8000 0000 - 8000 0000 0000: modules and main thread stack
+*/
+struct MappingLoongArch64_47 {
+ static const uptr kMetaShadowBeg = 0x300000000000ull;
+ static const uptr kMetaShadowEnd = 0x340000000000ull;
+ static const uptr kShadowBeg = 0x010000000000ull;
+ static const uptr kShadowEnd = 0x100000000000ull;
+ static const uptr kHeapMemBeg = 0x7ffe00000000ull;
+ static const uptr kHeapMemEnd = 0x7fff00000000ull;
+ static const uptr kLoAppMemBeg = 0x000000004000ull;
+ static const uptr kLoAppMemEnd = 0x008000000000ull;
+ static const uptr kMidAppMemBeg = 0x555500000000ull;
+ static const uptr kMidAppMemEnd = 0x555600000000ull;
+ static const uptr kHiAppMemBeg = 0x7fff80000000ull;
+ static const uptr kHiAppMemEnd = 0x800000000000ull;
+ static const uptr kShadowMsk = 0x780000000000ull;
+ static const uptr kShadowXor = 0x040000000000ull;
+ static const uptr kShadowAdd = 0x000000000000ull;
+ static const uptr kVdsoBeg = 0x7fffffffc000ull;
+};
/*
C/C++ on linux/powerpc64 (44-bit VMA)
0001 0000 0000 - 0b00 0000 0000: shadow
0b00 0000 0000 - 0b00 0000 0000: -
0b00 0000 0000 - 0d00 0000 0000: metainfo (memory blocks and sync objects)
-0d00 0000 0000 - 0d00 0000 0000: -
-0d00 0000 0000 - 0f00 0000 0000: traces
-0f00 0000 0000 - 0f00 0000 0000: -
+0d00 0000 0000 - 0f00 0000 0000: -
0f00 0000 0000 - 0f50 0000 0000: heap
0f50 0000 0000 - 0f60 0000 0000: -
0f60 0000 0000 - 1000 0000 0000: modules and main thread stack
*/
-struct Mapping44 {
+struct MappingPPC64_44 {
+ static const uptr kBroken = kBrokenMapping | kBrokenReverseMapping |
+ kBrokenLinearity | kBrokenAliasedMetas;
static const uptr kMetaShadowBeg = 0x0b0000000000ull;
static const uptr kMetaShadowEnd = 0x0d0000000000ull;
- static const uptr kTraceMemBeg = 0x0d0000000000ull;
- static const uptr kTraceMemEnd = 0x0f0000000000ull;
static const uptr kShadowBeg = 0x000100000000ull;
static const uptr kShadowEnd = 0x0b0000000000ull;
static const uptr kLoAppMemBeg = 0x000000000100ull;
static const uptr kHeapMemEnd = 0x0f5000000000ull;
static const uptr kHiAppMemBeg = 0x0f6000000000ull;
static const uptr kHiAppMemEnd = 0x100000000000ull; // 44 bits
- static const uptr kAppMemMsk = 0x0f0000000000ull;
- static const uptr kAppMemXor = 0x002100000000ull;
+ static const uptr kShadowMsk = 0x0f0000000000ull;
+ static const uptr kShadowXor = 0x002100000000ull;
+ static const uptr kShadowAdd = 0x000000000000ull;
static const uptr kVdsoBeg = 0x3c0000000000000ull;
+ static const uptr kMidAppMemBeg = 0;
+ static const uptr kMidAppMemEnd = 0;
};
/*
C/C++ on linux/powerpc64 (46-bit VMA)
0000 0000 1000 - 0100 0000 0000: main binary
0100 0000 0000 - 0200 0000 0000: -
-0100 0000 0000 - 1000 0000 0000: shadow
-1000 0000 0000 - 1000 0000 0000: -
-1000 0000 0000 - 2000 0000 0000: metainfo (memory blocks and sync objects)
-2000 0000 0000 - 2000 0000 0000: -
-2000 0000 0000 - 2200 0000 0000: traces
-2200 0000 0000 - 3d00 0000 0000: -
+0100 0000 0000 - 0800 0000 0000: shadow
+0800 0000 0000 - 1000 0000 0000: -
+1000 0000 0000 - 1200 0000 0000: metainfo (memory blocks and sync objects)
+1200 0000 0000 - 3d00 0000 0000: -
3d00 0000 0000 - 3e00 0000 0000: heap
3e00 0000 0000 - 3e80 0000 0000: -
3e80 0000 0000 - 4000 0000 0000: modules and main thread stack
*/
-struct Mapping46 {
+struct MappingPPC64_46 {
static const uptr kMetaShadowBeg = 0x100000000000ull;
- static const uptr kMetaShadowEnd = 0x200000000000ull;
- static const uptr kTraceMemBeg = 0x200000000000ull;
- static const uptr kTraceMemEnd = 0x220000000000ull;
+ static const uptr kMetaShadowEnd = 0x120000000000ull;
static const uptr kShadowBeg = 0x010000000000ull;
- static const uptr kShadowEnd = 0x100000000000ull;
+ static const uptr kShadowEnd = 0x080000000000ull;
static const uptr kHeapMemBeg = 0x3d0000000000ull;
static const uptr kHeapMemEnd = 0x3e0000000000ull;
static const uptr kLoAppMemBeg = 0x000000001000ull;
static const uptr kLoAppMemEnd = 0x010000000000ull;
static const uptr kHiAppMemBeg = 0x3e8000000000ull;
static const uptr kHiAppMemEnd = 0x400000000000ull; // 46 bits
- static const uptr kAppMemMsk = 0x3c0000000000ull;
- static const uptr kAppMemXor = 0x020000000000ull;
+ static const uptr kShadowMsk = 0x3c0000000000ull;
+ static const uptr kShadowXor = 0x020000000000ull;
+ static const uptr kShadowAdd = 0x000000000000ull;
static const uptr kVdsoBeg = 0x7800000000000000ull;
+ static const uptr kMidAppMemBeg = 0;
+ static const uptr kMidAppMemEnd = 0;
};
/*
C/C++ on linux/powerpc64 (47-bit VMA)
0000 0000 1000 - 0100 0000 0000: main binary
0100 0000 0000 - 0200 0000 0000: -
-0100 0000 0000 - 1000 0000 0000: shadow
-1000 0000 0000 - 1000 0000 0000: -
-1000 0000 0000 - 2000 0000 0000: metainfo (memory blocks and sync objects)
-2000 0000 0000 - 2000 0000 0000: -
-2000 0000 0000 - 2200 0000 0000: traces
-2200 0000 0000 - 7d00 0000 0000: -
+0100 0000 0000 - 0800 0000 0000: shadow
+0800 0000 0000 - 1000 0000 0000: -
+1000 0000 0000 - 1200 0000 0000: metainfo (memory blocks and sync objects)
+1200 0000 0000 - 7d00 0000 0000: -
7d00 0000 0000 - 7e00 0000 0000: heap
7e00 0000 0000 - 7e80 0000 0000: -
7e80 0000 0000 - 8000 0000 0000: modules and main thread stack
*/
-struct Mapping47 {
+struct MappingPPC64_47 {
static const uptr kMetaShadowBeg = 0x100000000000ull;
- static const uptr kMetaShadowEnd = 0x200000000000ull;
- static const uptr kTraceMemBeg = 0x200000000000ull;
- static const uptr kTraceMemEnd = 0x220000000000ull;
+ static const uptr kMetaShadowEnd = 0x120000000000ull;
static const uptr kShadowBeg = 0x010000000000ull;
- static const uptr kShadowEnd = 0x100000000000ull;
+ static const uptr kShadowEnd = 0x080000000000ull;
static const uptr kHeapMemBeg = 0x7d0000000000ull;
static const uptr kHeapMemEnd = 0x7e0000000000ull;
static const uptr kLoAppMemBeg = 0x000000001000ull;
static const uptr kLoAppMemEnd = 0x010000000000ull;
static const uptr kHiAppMemBeg = 0x7e8000000000ull;
static const uptr kHiAppMemEnd = 0x800000000000ull; // 47 bits
- static const uptr kAppMemMsk = 0x7c0000000000ull;
- static const uptr kAppMemXor = 0x020000000000ull;
+ static const uptr kShadowMsk = 0x7c0000000000ull;
+ static const uptr kShadowXor = 0x020000000000ull;
+ static const uptr kShadowAdd = 0x000000000000ull;
static const uptr kVdsoBeg = 0x7800000000000000ull;
+ static const uptr kMidAppMemBeg = 0;
+ static const uptr kMidAppMemEnd = 0;
};
-// Indicates the runtime will define the memory regions at runtime.
-#define TSAN_RUNTIME_VMA 1
-#elif defined(__s390x__)
/*
C/C++ on linux/s390x
While the kernel provides a 64-bit address space, we have to restrict ourselves
to 48 bits due to how e.g. SyncVar::GetId() works.
0000 0000 1000 - 0e00 0000 0000: binary, modules, stacks - 14 TiB
-0e00 0000 0000 - 4000 0000 0000: -
-4000 0000 0000 - 8000 0000 0000: shadow - 64TiB (4 * app)
-8000 0000 0000 - 9000 0000 0000: -
+0e00 0000 0000 - 2000 0000 0000: -
+2000 0000 0000 - 4000 0000 0000: shadow - 32TiB (2 * app)
+4000 0000 0000 - 9000 0000 0000: -
9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app)
-9800 0000 0000 - a000 0000 0000: -
-a000 0000 0000 - b000 0000 0000: traces - 16TiB (max history * 128k threads)
-b000 0000 0000 - be00 0000 0000: -
+9800 0000 0000 - be00 0000 0000: -
be00 0000 0000 - c000 0000 0000: heap - 2TiB (max supported by the allocator)
*/
-struct Mapping {
+struct MappingS390x {
static const uptr kMetaShadowBeg = 0x900000000000ull;
static const uptr kMetaShadowEnd = 0x980000000000ull;
- static const uptr kTraceMemBeg = 0xa00000000000ull;
- static const uptr kTraceMemEnd = 0xb00000000000ull;
- static const uptr kShadowBeg = 0x400000000000ull;
- static const uptr kShadowEnd = 0x800000000000ull;
+ static const uptr kShadowBeg = 0x200000000000ull;
+ static const uptr kShadowEnd = 0x400000000000ull;
static const uptr kHeapMemBeg = 0xbe0000000000ull;
static const uptr kHeapMemEnd = 0xc00000000000ull;
static const uptr kLoAppMemBeg = 0x000000001000ull;
static const uptr kLoAppMemEnd = 0x0e0000000000ull;
static const uptr kHiAppMemBeg = 0xc00000004000ull;
static const uptr kHiAppMemEnd = 0xc00000004000ull;
- static const uptr kAppMemMsk = 0xb00000000000ull;
- static const uptr kAppMemXor = 0x100000000000ull;
+ static const uptr kShadowMsk = 0xb00000000000ull;
+ static const uptr kShadowXor = 0x100000000000ull;
+ static const uptr kShadowAdd = 0x000000000000ull;
static const uptr kVdsoBeg = 0xfffffffff000ull;
+ static const uptr kMidAppMemBeg = 0;
+ static const uptr kMidAppMemEnd = 0;
};
-#endif
-
-#elif SANITIZER_GO && !SANITIZER_WINDOWS && HAS_48_BIT_ADDRESS_SPACE
/* Go on linux, darwin and freebsd on x86_64
0000 0000 1000 - 0000 1000 0000: executable
0000 1000 0000 - 00c0 0000 0000: -
00c0 0000 0000 - 00e0 0000 0000: heap
00e0 0000 0000 - 2000 0000 0000: -
-2000 0000 0000 - 2380 0000 0000: shadow
-2380 0000 0000 - 3000 0000 0000: -
+2000 0000 0000 - 21c0 0000 0000: shadow
+21c0 0000 0000 - 3000 0000 0000: -
3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
-4000 0000 0000 - 6000 0000 0000: -
-6000 0000 0000 - 6200 0000 0000: traces
-6200 0000 0000 - 8000 0000 0000: -
+4000 0000 0000 - 8000 0000 0000: -
*/
-struct Mapping {
+struct MappingGo48 {
static const uptr kMetaShadowBeg = 0x300000000000ull;
static const uptr kMetaShadowEnd = 0x400000000000ull;
- static const uptr kTraceMemBeg = 0x600000000000ull;
- static const uptr kTraceMemEnd = 0x620000000000ull;
static const uptr kShadowBeg = 0x200000000000ull;
- static const uptr kShadowEnd = 0x238000000000ull;
- static const uptr kAppMemBeg = 0x000000001000ull;
- static const uptr kAppMemEnd = 0x00e000000000ull;
+ static const uptr kShadowEnd = 0x21c000000000ull;
+ static const uptr kLoAppMemBeg = 0x000000001000ull;
+ static const uptr kLoAppMemEnd = 0x00e000000000ull;
+ static const uptr kMidAppMemBeg = 0;
+ static const uptr kMidAppMemEnd = 0;
+ static const uptr kHiAppMemBeg = 0;
+ static const uptr kHiAppMemEnd = 0;
+ static const uptr kHeapMemBeg = 0;
+ static const uptr kHeapMemEnd = 0;
+ static const uptr kVdsoBeg = 0;
+ static const uptr kShadowMsk = 0;
+ static const uptr kShadowXor = 0;
+ static const uptr kShadowAdd = 0x200000000000ull;
};
-#elif SANITIZER_GO && SANITIZER_WINDOWS
-
/* Go on windows
0000 0000 1000 - 0000 1000 0000: executable
0000 1000 0000 - 00f8 0000 0000: -
00c0 0000 0000 - 00e0 0000 0000: heap
00e0 0000 0000 - 0100 0000 0000: -
-0100 0000 0000 - 0500 0000 0000: shadow
-0500 0000 0000 - 0560 0000 0000: -
-0560 0000 0000 - 0760 0000 0000: traces
-0760 0000 0000 - 07d0 0000 0000: metainfo (memory blocks and sync objects)
+0100 0000 0000 - 0300 0000 0000: shadow
+0300 0000 0000 - 0700 0000 0000: -
+0700 0000 0000 - 0770 0000 0000: metainfo (memory blocks and sync objects)
07d0 0000 0000 - 8000 0000 0000: -
+PIE binaries currently not supported, but it should be theoretically possible.
*/
-struct Mapping {
- static const uptr kMetaShadowBeg = 0x076000000000ull;
- static const uptr kMetaShadowEnd = 0x07d000000000ull;
- static const uptr kTraceMemBeg = 0x056000000000ull;
- static const uptr kTraceMemEnd = 0x076000000000ull;
+struct MappingGoWindows {
+ static const uptr kMetaShadowBeg = 0x070000000000ull;
+ static const uptr kMetaShadowEnd = 0x077000000000ull;
static const uptr kShadowBeg = 0x010000000000ull;
- static const uptr kShadowEnd = 0x050000000000ull;
- static const uptr kAppMemBeg = 0x000000001000ull;
- static const uptr kAppMemEnd = 0x00e000000000ull;
+ static const uptr kShadowEnd = 0x030000000000ull;
+ static const uptr kLoAppMemBeg = 0x000000001000ull;
+ static const uptr kLoAppMemEnd = 0x00e000000000ull;
+ static const uptr kMidAppMemBeg = 0;
+ static const uptr kMidAppMemEnd = 0;
+ static const uptr kHiAppMemBeg = 0;
+ static const uptr kHiAppMemEnd = 0;
+ static const uptr kHeapMemBeg = 0;
+ static const uptr kHeapMemEnd = 0;
+ static const uptr kVdsoBeg = 0;
+ static const uptr kShadowMsk = 0;
+ static const uptr kShadowXor = 0;
+ static const uptr kShadowAdd = 0x010000000000ull;
};
-#elif SANITIZER_GO && defined(__powerpc64__)
-
-/* Only Mapping46 and Mapping47 are currently supported for powercp64 on Go. */
-
/* Go on linux/powerpc64 (46-bit VMA)
0000 0000 1000 - 0000 1000 0000: executable
0000 1000 0000 - 00c0 0000 0000: -
00c0 0000 0000 - 00e0 0000 0000: heap
00e0 0000 0000 - 2000 0000 0000: -
-2000 0000 0000 - 2380 0000 0000: shadow
-2380 0000 0000 - 2400 0000 0000: -
-2400 0000 0000 - 3400 0000 0000: metainfo (memory blocks and sync objects)
-3400 0000 0000 - 3600 0000 0000: -
-3600 0000 0000 - 3800 0000 0000: traces
-3800 0000 0000 - 4000 0000 0000: -
+2000 0000 0000 - 21c0 0000 0000: shadow
+21c0 0000 0000 - 2400 0000 0000: -
+2400 0000 0000 - 2470 0000 0000: metainfo (memory blocks and sync objects)
+2470 0000 0000 - 4000 0000 0000: -
*/
-struct Mapping46 {
+struct MappingGoPPC64_46 {
static const uptr kMetaShadowBeg = 0x240000000000ull;
- static const uptr kMetaShadowEnd = 0x340000000000ull;
- static const uptr kTraceMemBeg = 0x360000000000ull;
- static const uptr kTraceMemEnd = 0x380000000000ull;
+ static const uptr kMetaShadowEnd = 0x247000000000ull;
static const uptr kShadowBeg = 0x200000000000ull;
- static const uptr kShadowEnd = 0x238000000000ull;
- static const uptr kAppMemBeg = 0x000000001000ull;
- static const uptr kAppMemEnd = 0x00e000000000ull;
+ static const uptr kShadowEnd = 0x21c000000000ull;
+ static const uptr kLoAppMemBeg = 0x000000001000ull;
+ static const uptr kLoAppMemEnd = 0x00e000000000ull;
+ static const uptr kMidAppMemBeg = 0;
+ static const uptr kMidAppMemEnd = 0;
+ static const uptr kHiAppMemBeg = 0;
+ static const uptr kHiAppMemEnd = 0;
+ static const uptr kHeapMemBeg = 0;
+ static const uptr kHeapMemEnd = 0;
+ static const uptr kVdsoBeg = 0;
+ static const uptr kShadowMsk = 0;
+ static const uptr kShadowXor = 0;
+ static const uptr kShadowAdd = 0x200000000000ull;
};
/* Go on linux/powerpc64 (47-bit VMA)
0000 1000 0000 - 00c0 0000 0000: -
00c0 0000 0000 - 00e0 0000 0000: heap
00e0 0000 0000 - 2000 0000 0000: -
-2000 0000 0000 - 3000 0000 0000: shadow
-3000 0000 0000 - 3000 0000 0000: -
-3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
-4000 0000 0000 - 6000 0000 0000: -
-6000 0000 0000 - 6200 0000 0000: traces
-6200 0000 0000 - 8000 0000 0000: -
+2000 0000 0000 - 2800 0000 0000: shadow
+2800 0000 0000 - 3000 0000 0000: -
+3000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects)
+3200 0000 0000 - 8000 0000 0000: -
*/
-struct Mapping47 {
+struct MappingGoPPC64_47 {
static const uptr kMetaShadowBeg = 0x300000000000ull;
- static const uptr kMetaShadowEnd = 0x400000000000ull;
- static const uptr kTraceMemBeg = 0x600000000000ull;
- static const uptr kTraceMemEnd = 0x620000000000ull;
+ static const uptr kMetaShadowEnd = 0x320000000000ull;
static const uptr kShadowBeg = 0x200000000000ull;
- static const uptr kShadowEnd = 0x300000000000ull;
- static const uptr kAppMemBeg = 0x000000001000ull;
- static const uptr kAppMemEnd = 0x00e000000000ull;
+ static const uptr kShadowEnd = 0x280000000000ull;
+ static const uptr kLoAppMemBeg = 0x000000001000ull;
+ static const uptr kLoAppMemEnd = 0x00e000000000ull;
+ static const uptr kMidAppMemBeg = 0;
+ static const uptr kMidAppMemEnd = 0;
+ static const uptr kHiAppMemBeg = 0;
+ static const uptr kHiAppMemEnd = 0;
+ static const uptr kHeapMemBeg = 0;
+ static const uptr kHeapMemEnd = 0;
+ static const uptr kVdsoBeg = 0;
+ static const uptr kShadowMsk = 0;
+ static const uptr kShadowXor = 0;
+ static const uptr kShadowAdd = 0x200000000000ull;
};
-#define TSAN_RUNTIME_VMA 1
-
-#elif SANITIZER_GO && defined(__aarch64__)
-
/* Go on linux/aarch64 (48-bit VMA) and darwin/aarch64 (47-bit VMA)
0000 0000 1000 - 0000 1000 0000: executable
0000 1000 0000 - 00c0 0000 0000: -
00c0 0000 0000 - 00e0 0000 0000: heap
00e0 0000 0000 - 2000 0000 0000: -
-2000 0000 0000 - 3000 0000 0000: shadow
-3000 0000 0000 - 3000 0000 0000: -
-3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
-4000 0000 0000 - 6000 0000 0000: -
-6000 0000 0000 - 6200 0000 0000: traces
-6200 0000 0000 - 8000 0000 0000: -
+2000 0000 0000 - 2800 0000 0000: shadow
+2800 0000 0000 - 3000 0000 0000: -
+3000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects)
+3200 0000 0000 - 8000 0000 0000: -
*/
-
-struct Mapping {
+struct MappingGoAarch64 {
static const uptr kMetaShadowBeg = 0x300000000000ull;
- static const uptr kMetaShadowEnd = 0x400000000000ull;
- static const uptr kTraceMemBeg = 0x600000000000ull;
- static const uptr kTraceMemEnd = 0x620000000000ull;
+ static const uptr kMetaShadowEnd = 0x320000000000ull;
static const uptr kShadowBeg = 0x200000000000ull;
- static const uptr kShadowEnd = 0x300000000000ull;
- static const uptr kAppMemBeg = 0x000000001000ull;
- static const uptr kAppMemEnd = 0x00e000000000ull;
+ static const uptr kShadowEnd = 0x280000000000ull;
+ static const uptr kLoAppMemBeg = 0x000000001000ull;
+ static const uptr kLoAppMemEnd = 0x00e000000000ull;
+ static const uptr kMidAppMemBeg = 0;
+ static const uptr kMidAppMemEnd = 0;
+ static const uptr kHiAppMemBeg = 0;
+ static const uptr kHiAppMemEnd = 0;
+ static const uptr kHeapMemBeg = 0;
+ static const uptr kHeapMemEnd = 0;
+ static const uptr kVdsoBeg = 0;
+ static const uptr kShadowMsk = 0;
+ static const uptr kShadowXor = 0;
+ static const uptr kShadowAdd = 0x200000000000ull;
};
-// Indicates the runtime will define the memory regions at runtime.
-#define TSAN_RUNTIME_VMA 1
-
-#elif SANITIZER_GO && defined(__mips64)
/*
Go on linux/mips64 (47-bit VMA)
0000 0000 1000 - 0000 1000 0000: executable
0000 1000 0000 - 00c0 0000 0000: -
00c0 0000 0000 - 00e0 0000 0000: heap
00e0 0000 0000 - 2000 0000 0000: -
-2000 0000 0000 - 3000 0000 0000: shadow
-3000 0000 0000 - 3000 0000 0000: -
-3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
-4000 0000 0000 - 6000 0000 0000: -
-6000 0000 0000 - 6200 0000 0000: traces
-6200 0000 0000 - 8000 0000 0000: -
+2000 0000 0000 - 2800 0000 0000: shadow
+2800 0000 0000 - 3000 0000 0000: -
+3000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects)
+3200 0000 0000 - 8000 0000 0000: -
*/
-struct Mapping47 {
+struct MappingGoMips64_47 {
static const uptr kMetaShadowBeg = 0x300000000000ull;
- static const uptr kMetaShadowEnd = 0x400000000000ull;
- static const uptr kTraceMemBeg = 0x600000000000ull;
- static const uptr kTraceMemEnd = 0x620000000000ull;
+ static const uptr kMetaShadowEnd = 0x320000000000ull;
static const uptr kShadowBeg = 0x200000000000ull;
- static const uptr kShadowEnd = 0x300000000000ull;
- static const uptr kAppMemBeg = 0x000000001000ull;
- static const uptr kAppMemEnd = 0x00e000000000ull;
+ static const uptr kShadowEnd = 0x280000000000ull;
+ static const uptr kLoAppMemBeg = 0x000000001000ull;
+ static const uptr kLoAppMemEnd = 0x00e000000000ull;
+ static const uptr kMidAppMemBeg = 0;
+ static const uptr kMidAppMemEnd = 0;
+ static const uptr kHiAppMemBeg = 0;
+ static const uptr kHiAppMemEnd = 0;
+ static const uptr kHeapMemBeg = 0;
+ static const uptr kHeapMemEnd = 0;
+ static const uptr kVdsoBeg = 0;
+ static const uptr kShadowMsk = 0;
+ static const uptr kShadowXor = 0;
+ static const uptr kShadowAdd = 0x200000000000ull;
};
-#define TSAN_RUNTIME_VMA 1
-
-#elif SANITIZER_GO && defined(__s390x__)
/*
Go on linux/s390x
0000 0000 1000 - 1000 0000 0000: executable and heap - 16 TiB
1000 0000 0000 - 4000 0000 0000: -
-4000 0000 0000 - 8000 0000 0000: shadow - 64TiB (4 * app)
-8000 0000 0000 - 9000 0000 0000: -
+4000 0000 0000 - 6000 0000 0000: shadow - 64TiB (4 * app)
+6000 0000 0000 - 9000 0000 0000: -
9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app)
-9800 0000 0000 - a000 0000 0000: -
-a000 0000 0000 - b000 0000 0000: traces - 16TiB (max history * 128k threads)
*/
-struct Mapping {
+struct MappingGoS390x {
static const uptr kMetaShadowBeg = 0x900000000000ull;
static const uptr kMetaShadowEnd = 0x980000000000ull;
- static const uptr kTraceMemBeg = 0xa00000000000ull;
- static const uptr kTraceMemEnd = 0xb00000000000ull;
static const uptr kShadowBeg = 0x400000000000ull;
- static const uptr kShadowEnd = 0x800000000000ull;
- static const uptr kAppMemBeg = 0x000000001000ull;
- static const uptr kAppMemEnd = 0x100000000000ull;
+ static const uptr kShadowEnd = 0x600000000000ull;
+ static const uptr kLoAppMemBeg = 0x000000001000ull;
+ static const uptr kLoAppMemEnd = 0x100000000000ull;
+ static const uptr kMidAppMemBeg = 0;
+ static const uptr kMidAppMemEnd = 0;
+ static const uptr kHiAppMemBeg = 0;
+ static const uptr kHiAppMemEnd = 0;
+ static const uptr kHeapMemBeg = 0;
+ static const uptr kHeapMemEnd = 0;
+ static const uptr kVdsoBeg = 0;
+ static const uptr kShadowMsk = 0;
+ static const uptr kShadowXor = 0;
+ static const uptr kShadowAdd = 0x400000000000ull;
};
-#else
-# error "Unknown platform"
-#endif
-
-
-#ifdef TSAN_RUNTIME_VMA
extern uptr vmaSize;
-#endif
-
-
-enum MappingType {
- MAPPING_LO_APP_BEG,
- MAPPING_LO_APP_END,
- MAPPING_HI_APP_BEG,
- MAPPING_HI_APP_END,
-#ifdef TSAN_MID_APP_RANGE
- MAPPING_MID_APP_BEG,
- MAPPING_MID_APP_END,
-#endif
- MAPPING_HEAP_BEG,
- MAPPING_HEAP_END,
- MAPPING_APP_BEG,
- MAPPING_APP_END,
- MAPPING_SHADOW_BEG,
- MAPPING_SHADOW_END,
- MAPPING_META_SHADOW_BEG,
- MAPPING_META_SHADOW_END,
- MAPPING_TRACE_BEG,
- MAPPING_TRACE_END,
- MAPPING_VDSO_BEG,
-};
-template<typename Mapping, int Type>
-uptr MappingImpl(void) {
- switch (Type) {
-#if !SANITIZER_GO
- case MAPPING_LO_APP_BEG: return Mapping::kLoAppMemBeg;
- case MAPPING_LO_APP_END: return Mapping::kLoAppMemEnd;
-# ifdef TSAN_MID_APP_RANGE
- case MAPPING_MID_APP_BEG: return Mapping::kMidAppMemBeg;
- case MAPPING_MID_APP_END: return Mapping::kMidAppMemEnd;
-# endif
- case MAPPING_HI_APP_BEG: return Mapping::kHiAppMemBeg;
- case MAPPING_HI_APP_END: return Mapping::kHiAppMemEnd;
- case MAPPING_HEAP_BEG: return Mapping::kHeapMemBeg;
- case MAPPING_HEAP_END: return Mapping::kHeapMemEnd;
- case MAPPING_VDSO_BEG: return Mapping::kVdsoBeg;
-#else
- case MAPPING_APP_BEG: return Mapping::kAppMemBeg;
- case MAPPING_APP_END: return Mapping::kAppMemEnd;
-#endif
- case MAPPING_SHADOW_BEG: return Mapping::kShadowBeg;
- case MAPPING_SHADOW_END: return Mapping::kShadowEnd;
- case MAPPING_META_SHADOW_BEG: return Mapping::kMetaShadowBeg;
- case MAPPING_META_SHADOW_END: return Mapping::kMetaShadowEnd;
- case MAPPING_TRACE_BEG: return Mapping::kTraceMemBeg;
- case MAPPING_TRACE_END: return Mapping::kTraceMemEnd;
- }
-}
-
-template<int Type>
-uptr MappingArchImpl(void) {
-#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO
+template <typename Func, typename Arg>
+ALWAYS_INLINE auto SelectMapping(Arg arg) {
+#if SANITIZER_GO
+# if defined(__powerpc64__)
switch (vmaSize) {
- case 39: return MappingImpl<Mapping39, Type>();
- case 42: return MappingImpl<Mapping42, Type>();
- case 48: return MappingImpl<Mapping48, Type>();
+ case 46:
+ return Func::template Apply<MappingGoPPC64_46>(arg);
+ case 47:
+ return Func::template Apply<MappingGoPPC64_47>(arg);
}
- DCHECK(0);
- return 0;
-#elif defined(__powerpc64__)
+# elif defined(__mips64)
+ return Func::template Apply<MappingGoMips64_47>(arg);
+# elif defined(__s390x__)
+ return Func::template Apply<MappingGoS390x>(arg);
+# elif defined(__aarch64__)
+ return Func::template Apply<MappingGoAarch64>(arg);
+# elif SANITIZER_WINDOWS
+ return Func::template Apply<MappingGoWindows>(arg);
+# else
+ return Func::template Apply<MappingGo48>(arg);
+# endif
+#else // SANITIZER_GO
+# if SANITIZER_IOS && !SANITIZER_IOSSIM
+ return Func::template Apply<MappingAppleAarch64>(arg);
+# elif defined(__x86_64__) || SANITIZER_APPLE
+ return Func::template Apply<Mapping48AddressSpace>(arg);
+# elif defined(__aarch64__)
switch (vmaSize) {
-#if !SANITIZER_GO
- case 44: return MappingImpl<Mapping44, Type>();
-#endif
- case 46: return MappingImpl<Mapping46, Type>();
- case 47: return MappingImpl<Mapping47, Type>();
+ case 39:
+ return Func::template Apply<MappingAarch64_39>(arg);
+ case 42:
+ return Func::template Apply<MappingAarch64_42>(arg);
+ case 48:
+ return Func::template Apply<MappingAarch64_48>(arg);
}
- DCHECK(0);
- return 0;
-#elif defined(__mips64)
+# elif SANITIZER_LOONGARCH64
+ return Func::template Apply<MappingLoongArch64_47>(arg);
+# elif defined(__powerpc64__)
switch (vmaSize) {
-#if !SANITIZER_GO
- case 40: return MappingImpl<Mapping40, Type>();
-#else
- case 47: return MappingImpl<Mapping47, Type>();
-#endif
+ case 44:
+ return Func::template Apply<MappingPPC64_44>(arg);
+ case 46:
+ return Func::template Apply<MappingPPC64_46>(arg);
+ case 47:
+ return Func::template Apply<MappingPPC64_47>(arg);
}
- DCHECK(0);
- return 0;
-#else
- return MappingImpl<Mapping, Type>();
-#endif
+# elif defined(__mips64)
+ return Func::template Apply<MappingMips64_40>(arg);
+# elif defined(__s390x__)
+ return Func::template Apply<MappingS390x>(arg);
+# else
+# error "unsupported platform"
+# endif
+#endif
+ Die();
+}
+
+template <typename Func>
+void ForEachMapping() {
+ Func::template Apply<Mapping48AddressSpace>();
+ Func::template Apply<MappingMips64_40>();
+ Func::template Apply<MappingAppleAarch64>();
+ Func::template Apply<MappingAarch64_39>();
+ Func::template Apply<MappingAarch64_42>();
+ Func::template Apply<MappingAarch64_48>();
+ Func::template Apply<MappingLoongArch64_47>();
+ Func::template Apply<MappingPPC64_44>();
+ Func::template Apply<MappingPPC64_46>();
+ Func::template Apply<MappingPPC64_47>();
+ Func::template Apply<MappingS390x>();
+ Func::template Apply<MappingGo48>();
+ Func::template Apply<MappingGoWindows>();
+ Func::template Apply<MappingGoPPC64_46>();
+ Func::template Apply<MappingGoPPC64_47>();
+ Func::template Apply<MappingGoAarch64>();
+ Func::template Apply<MappingGoMips64_47>();
+ Func::template Apply<MappingGoS390x>();
}
-#if !SANITIZER_GO
-ALWAYS_INLINE
-uptr LoAppMemBeg(void) {
- return MappingArchImpl<MAPPING_LO_APP_BEG>();
-}
-ALWAYS_INLINE
-uptr LoAppMemEnd(void) {
- return MappingArchImpl<MAPPING_LO_APP_END>();
-}
+enum MappingType {
+ kLoAppMemBeg,
+ kLoAppMemEnd,
+ kHiAppMemBeg,
+ kHiAppMemEnd,
+ kMidAppMemBeg,
+ kMidAppMemEnd,
+ kHeapMemBeg,
+ kHeapMemEnd,
+ kShadowBeg,
+ kShadowEnd,
+ kMetaShadowBeg,
+ kMetaShadowEnd,
+ kVdsoBeg,
+};
-#ifdef TSAN_MID_APP_RANGE
-ALWAYS_INLINE
-uptr MidAppMemBeg(void) {
- return MappingArchImpl<MAPPING_MID_APP_BEG>();
-}
-ALWAYS_INLINE
-uptr MidAppMemEnd(void) {
- return MappingArchImpl<MAPPING_MID_APP_END>();
-}
-#endif
+struct MappingField {
+ template <typename Mapping>
+ static uptr Apply(MappingType type) {
+ switch (type) {
+ case kLoAppMemBeg:
+ return Mapping::kLoAppMemBeg;
+ case kLoAppMemEnd:
+ return Mapping::kLoAppMemEnd;
+ case kMidAppMemBeg:
+ return Mapping::kMidAppMemBeg;
+ case kMidAppMemEnd:
+ return Mapping::kMidAppMemEnd;
+ case kHiAppMemBeg:
+ return Mapping::kHiAppMemBeg;
+ case kHiAppMemEnd:
+ return Mapping::kHiAppMemEnd;
+ case kHeapMemBeg:
+ return Mapping::kHeapMemBeg;
+ case kHeapMemEnd:
+ return Mapping::kHeapMemEnd;
+ case kVdsoBeg:
+ return Mapping::kVdsoBeg;
+ case kShadowBeg:
+ return Mapping::kShadowBeg;
+ case kShadowEnd:
+ return Mapping::kShadowEnd;
+ case kMetaShadowBeg:
+ return Mapping::kMetaShadowBeg;
+ case kMetaShadowEnd:
+ return Mapping::kMetaShadowEnd;
+ }
+ Die();
+ }
+};
ALWAYS_INLINE
-uptr HeapMemBeg(void) {
- return MappingArchImpl<MAPPING_HEAP_BEG>();
-}
+uptr LoAppMemBeg(void) { return SelectMapping<MappingField>(kLoAppMemBeg); }
ALWAYS_INLINE
-uptr HeapMemEnd(void) {
- return MappingArchImpl<MAPPING_HEAP_END>();
-}
+uptr LoAppMemEnd(void) { return SelectMapping<MappingField>(kLoAppMemEnd); }
ALWAYS_INLINE
-uptr HiAppMemBeg(void) {
- return MappingArchImpl<MAPPING_HI_APP_BEG>();
-}
+uptr MidAppMemBeg(void) { return SelectMapping<MappingField>(kMidAppMemBeg); }
ALWAYS_INLINE
-uptr HiAppMemEnd(void) {
- return MappingArchImpl<MAPPING_HI_APP_END>();
-}
+uptr MidAppMemEnd(void) { return SelectMapping<MappingField>(kMidAppMemEnd); }
ALWAYS_INLINE
-uptr VdsoBeg(void) {
- return MappingArchImpl<MAPPING_VDSO_BEG>();
-}
-
-#else
+uptr HeapMemBeg(void) { return SelectMapping<MappingField>(kHeapMemBeg); }
+ALWAYS_INLINE
+uptr HeapMemEnd(void) { return SelectMapping<MappingField>(kHeapMemEnd); }
ALWAYS_INLINE
-uptr AppMemBeg(void) {
- return MappingArchImpl<MAPPING_APP_BEG>();
-}
+uptr HiAppMemBeg(void) { return SelectMapping<MappingField>(kHiAppMemBeg); }
ALWAYS_INLINE
-uptr AppMemEnd(void) {
- return MappingArchImpl<MAPPING_APP_END>();
-}
-
-#endif
-
-static inline
-bool GetUserRegion(int i, uptr *start, uptr *end) {
- switch (i) {
- default:
- return false;
-#if !SANITIZER_GO
- case 0:
- *start = LoAppMemBeg();
- *end = LoAppMemEnd();
- return true;
- case 1:
- *start = HiAppMemBeg();
- *end = HiAppMemEnd();
- return true;
- case 2:
- *start = HeapMemBeg();
- *end = HeapMemEnd();
- return true;
-# ifdef TSAN_MID_APP_RANGE
- case 3:
- *start = MidAppMemBeg();
- *end = MidAppMemEnd();
- return true;
-# endif
-#else
- case 0:
- *start = AppMemBeg();
- *end = AppMemEnd();
- return true;
-#endif
- }
-}
+uptr HiAppMemEnd(void) { return SelectMapping<MappingField>(kHiAppMemEnd); }
ALWAYS_INLINE
-uptr ShadowBeg(void) {
- return MappingArchImpl<MAPPING_SHADOW_BEG>();
-}
-ALWAYS_INLINE
-uptr ShadowEnd(void) {
- return MappingArchImpl<MAPPING_SHADOW_END>();
-}
+uptr VdsoBeg(void) { return SelectMapping<MappingField>(kVdsoBeg); }
ALWAYS_INLINE
-uptr MetaShadowBeg(void) {
- return MappingArchImpl<MAPPING_META_SHADOW_BEG>();
-}
+uptr ShadowBeg(void) { return SelectMapping<MappingField>(kShadowBeg); }
ALWAYS_INLINE
-uptr MetaShadowEnd(void) {
- return MappingArchImpl<MAPPING_META_SHADOW_END>();
-}
+uptr ShadowEnd(void) { return SelectMapping<MappingField>(kShadowEnd); }
ALWAYS_INLINE
-uptr TraceMemBeg(void) {
- return MappingArchImpl<MAPPING_TRACE_BEG>();
-}
+uptr MetaShadowBeg(void) { return SelectMapping<MappingField>(kMetaShadowBeg); }
ALWAYS_INLINE
-uptr TraceMemEnd(void) {
- return MappingArchImpl<MAPPING_TRACE_END>();
-}
+uptr MetaShadowEnd(void) { return SelectMapping<MappingField>(kMetaShadowEnd); }
-
-template<typename Mapping>
-bool IsAppMemImpl(uptr mem) {
-#if !SANITIZER_GO
+struct IsAppMemImpl {
+ template <typename Mapping>
+ static bool Apply(uptr mem) {
return (mem >= Mapping::kHeapMemBeg && mem < Mapping::kHeapMemEnd) ||
-# ifdef TSAN_MID_APP_RANGE
(mem >= Mapping::kMidAppMemBeg && mem < Mapping::kMidAppMemEnd) ||
-# endif
(mem >= Mapping::kLoAppMemBeg && mem < Mapping::kLoAppMemEnd) ||
(mem >= Mapping::kHiAppMemBeg && mem < Mapping::kHiAppMemEnd);
-#else
- return mem >= Mapping::kAppMemBeg && mem < Mapping::kAppMemEnd;
-#endif
-}
-
-ALWAYS_INLINE
-bool IsAppMem(uptr mem) {
-#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO
- switch (vmaSize) {
- case 39: return IsAppMemImpl<Mapping39>(mem);
- case 42: return IsAppMemImpl<Mapping42>(mem);
- case 48: return IsAppMemImpl<Mapping48>(mem);
- }
- DCHECK(0);
- return false;
-#elif defined(__powerpc64__)
- switch (vmaSize) {
-#if !SANITIZER_GO
- case 44: return IsAppMemImpl<Mapping44>(mem);
-#endif
- case 46: return IsAppMemImpl<Mapping46>(mem);
- case 47: return IsAppMemImpl<Mapping47>(mem);
}
- DCHECK(0);
- return false;
-#elif defined(__mips64)
- switch (vmaSize) {
-#if !SANITIZER_GO
- case 40: return IsAppMemImpl<Mapping40>(mem);
-#else
- case 47: return IsAppMemImpl<Mapping47>(mem);
-#endif
- }
- DCHECK(0);
- return false;
-#else
- return IsAppMemImpl<Mapping>(mem);
-#endif
-}
-
-
-template<typename Mapping>
-bool IsShadowMemImpl(uptr mem) {
- return mem >= Mapping::kShadowBeg && mem <= Mapping::kShadowEnd;
-}
+};
ALWAYS_INLINE
-bool IsShadowMem(uptr mem) {
-#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO
- switch (vmaSize) {
- case 39: return IsShadowMemImpl<Mapping39>(mem);
- case 42: return IsShadowMemImpl<Mapping42>(mem);
- case 48: return IsShadowMemImpl<Mapping48>(mem);
- }
- DCHECK(0);
- return false;
-#elif defined(__powerpc64__)
- switch (vmaSize) {
-#if !SANITIZER_GO
- case 44: return IsShadowMemImpl<Mapping44>(mem);
-#endif
- case 46: return IsShadowMemImpl<Mapping46>(mem);
- case 47: return IsShadowMemImpl<Mapping47>(mem);
- }
- DCHECK(0);
- return false;
-#elif defined(__mips64)
- switch (vmaSize) {
-#if !SANITIZER_GO
- case 40: return IsShadowMemImpl<Mapping40>(mem);
-#else
- case 47: return IsShadowMemImpl<Mapping47>(mem);
-#endif
- }
- DCHECK(0);
- return false;
-#else
- return IsShadowMemImpl<Mapping>(mem);
-#endif
-}
-
+bool IsAppMem(uptr mem) { return SelectMapping<IsAppMemImpl>(mem); }
-template<typename Mapping>
-bool IsMetaMemImpl(uptr mem) {
- return mem >= Mapping::kMetaShadowBeg && mem <= Mapping::kMetaShadowEnd;
-}
-
-ALWAYS_INLINE
-bool IsMetaMem(uptr mem) {
-#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO
- switch (vmaSize) {
- case 39: return IsMetaMemImpl<Mapping39>(mem);
- case 42: return IsMetaMemImpl<Mapping42>(mem);
- case 48: return IsMetaMemImpl<Mapping48>(mem);
- }
- DCHECK(0);
- return false;
-#elif defined(__powerpc64__)
- switch (vmaSize) {
-#if !SANITIZER_GO
- case 44: return IsMetaMemImpl<Mapping44>(mem);
-#endif
- case 46: return IsMetaMemImpl<Mapping46>(mem);
- case 47: return IsMetaMemImpl<Mapping47>(mem);
- }
- DCHECK(0);
- return false;
-#elif defined(__mips64)
- switch (vmaSize) {
-#if !SANITIZER_GO
- case 40: return IsMetaMemImpl<Mapping40>(mem);
-#else
- case 47: return IsMetaMemImpl<Mapping47>(mem);
-#endif
+struct IsShadowMemImpl {
+ template <typename Mapping>
+ static bool Apply(uptr mem) {
+ return mem >= Mapping::kShadowBeg && mem <= Mapping::kShadowEnd;
}
- DCHECK(0);
- return false;
-#else
- return IsMetaMemImpl<Mapping>(mem);
-#endif
-}
-
-
-template<typename Mapping>
-uptr MemToShadowImpl(uptr x) {
- DCHECK(IsAppMem(x));
-#if !SANITIZER_GO
- return (((x) & ~(Mapping::kAppMemMsk | (kShadowCell - 1)))
- ^ Mapping::kAppMemXor) * kShadowCnt;
-#else
-# ifndef SANITIZER_WINDOWS
- return ((x & ~(kShadowCell - 1)) * kShadowCnt) | Mapping::kShadowBeg;
-# else
- return ((x & ~(kShadowCell - 1)) * kShadowCnt) + Mapping::kShadowBeg;
-# endif
-#endif
-}
+};
ALWAYS_INLINE
-uptr MemToShadow(uptr x) {
-#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO
- switch (vmaSize) {
- case 39: return MemToShadowImpl<Mapping39>(x);
- case 42: return MemToShadowImpl<Mapping42>(x);
- case 48: return MemToShadowImpl<Mapping48>(x);
- }
- DCHECK(0);
- return 0;
-#elif defined(__powerpc64__)
- switch (vmaSize) {
-#if !SANITIZER_GO
- case 44: return MemToShadowImpl<Mapping44>(x);
-#endif
- case 46: return MemToShadowImpl<Mapping46>(x);
- case 47: return MemToShadowImpl<Mapping47>(x);
- }
- DCHECK(0);
- return 0;
-#elif defined(__mips64)
- switch (vmaSize) {
-#if !SANITIZER_GO
- case 40: return MemToShadowImpl<Mapping40>(x);
-#else
- case 47: return MemToShadowImpl<Mapping47>(x);
-#endif
- }
- DCHECK(0);
- return 0;
-#else
- return MemToShadowImpl<Mapping>(x);
-#endif
+bool IsShadowMem(RawShadow *p) {
+ return SelectMapping<IsShadowMemImpl>(reinterpret_cast<uptr>(p));
}
-
-template<typename Mapping>
-u32 *MemToMetaImpl(uptr x) {
- DCHECK(IsAppMem(x));
-#if !SANITIZER_GO
- return (u32*)(((((x) & ~(Mapping::kAppMemMsk | (kMetaShadowCell - 1)))) /
- kMetaShadowCell * kMetaShadowSize) | Mapping::kMetaShadowBeg);
-#else
-# ifndef SANITIZER_WINDOWS
- return (u32*)(((x & ~(kMetaShadowCell - 1)) / \
- kMetaShadowCell * kMetaShadowSize) | Mapping::kMetaShadowBeg);
-# else
- return (u32*)(((x & ~(kMetaShadowCell - 1)) / \
- kMetaShadowCell * kMetaShadowSize) + Mapping::kMetaShadowBeg);
-# endif
-#endif
-}
+struct IsMetaMemImpl {
+ template <typename Mapping>
+ static bool Apply(uptr mem) {
+ return mem >= Mapping::kMetaShadowBeg && mem <= Mapping::kMetaShadowEnd;
+ }
+};
ALWAYS_INLINE
-u32 *MemToMeta(uptr x) {
-#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO
- switch (vmaSize) {
- case 39: return MemToMetaImpl<Mapping39>(x);
- case 42: return MemToMetaImpl<Mapping42>(x);
- case 48: return MemToMetaImpl<Mapping48>(x);
+bool IsMetaMem(const u32 *p) {
+ return SelectMapping<IsMetaMemImpl>(reinterpret_cast<uptr>(p));
+}
+
+struct MemToShadowImpl {
+ template <typename Mapping>
+ static uptr Apply(uptr x) {
+ DCHECK(IsAppMemImpl::Apply<Mapping>(x));
+ return (((x) & ~(Mapping::kShadowMsk | (kShadowCell - 1))) ^
+ Mapping::kShadowXor) *
+ kShadowMultiplier +
+ Mapping::kShadowAdd;
}
- DCHECK(0);
- return 0;
-#elif defined(__powerpc64__)
- switch (vmaSize) {
-#if !SANITIZER_GO
- case 44: return MemToMetaImpl<Mapping44>(x);
-#endif
- case 46: return MemToMetaImpl<Mapping46>(x);
- case 47: return MemToMetaImpl<Mapping47>(x);
- }
- DCHECK(0);
- return 0;
-#elif defined(__mips64)
- switch (vmaSize) {
-#if !SANITIZER_GO
- case 40: return MemToMetaImpl<Mapping40>(x);
-#else
- case 47: return MemToMetaImpl<Mapping47>(x);
-#endif
- }
- DCHECK(0);
- return 0;
-#else
- return MemToMetaImpl<Mapping>(x);
-#endif
-}
-
-
-template<typename Mapping>
-uptr ShadowToMemImpl(uptr s) {
- DCHECK(IsShadowMem(s));
-#if !SANITIZER_GO
- // The shadow mapping is non-linear and we've lost some bits, so we don't have
- // an easy way to restore the original app address. But the mapping is a
- // bijection, so we try to restore the address as belonging to low/mid/high
- // range consecutively and see if shadow->app->shadow mapping gives us the
- // same address.
- uptr p = (s / kShadowCnt) ^ Mapping::kAppMemXor;
- if (p >= Mapping::kLoAppMemBeg && p < Mapping::kLoAppMemEnd &&
- MemToShadow(p) == s)
- return p;
-# ifdef TSAN_MID_APP_RANGE
- p = ((s / kShadowCnt) ^ Mapping::kAppMemXor) +
- (Mapping::kMidAppMemBeg & Mapping::kAppMemMsk);
- if (p >= Mapping::kMidAppMemBeg && p < Mapping::kMidAppMemEnd &&
- MemToShadow(p) == s)
- return p;
-# endif
- return ((s / kShadowCnt) ^ Mapping::kAppMemXor) | Mapping::kAppMemMsk;
-#else // #if !SANITIZER_GO
-# ifndef SANITIZER_WINDOWS
- return (s & ~Mapping::kShadowBeg) / kShadowCnt;
-# else
- return (s - Mapping::kShadowBeg) / kShadowCnt;
-# endif // SANITIZER_WINDOWS
-#endif
-}
+};
ALWAYS_INLINE
-uptr ShadowToMem(uptr s) {
-#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO
- switch (vmaSize) {
- case 39: return ShadowToMemImpl<Mapping39>(s);
- case 42: return ShadowToMemImpl<Mapping42>(s);
- case 48: return ShadowToMemImpl<Mapping48>(s);
- }
- DCHECK(0);
- return 0;
-#elif defined(__powerpc64__)
- switch (vmaSize) {
-#if !SANITIZER_GO
- case 44: return ShadowToMemImpl<Mapping44>(s);
-#endif
- case 46: return ShadowToMemImpl<Mapping46>(s);
- case 47: return ShadowToMemImpl<Mapping47>(s);
- }
- DCHECK(0);
- return 0;
-#elif defined(__mips64)
- switch (vmaSize) {
-#if !SANITIZER_GO
- case 40: return ShadowToMemImpl<Mapping40>(s);
-#else
- case 47: return ShadowToMemImpl<Mapping47>(s);
-#endif
- }
- DCHECK(0);
- return 0;
-#else
- return ShadowToMemImpl<Mapping>(s);
-#endif
+RawShadow *MemToShadow(uptr x) {
+ return reinterpret_cast<RawShadow *>(SelectMapping<MemToShadowImpl>(x));
}
-
-
-// The additional page is to catch shadow stack overflow as paging fault.
-// Windows wants 64K alignment for mmaps.
-const uptr kTotalTraceSize = (kTraceSize * sizeof(Event) + sizeof(Trace)
- + (64 << 10) + (64 << 10) - 1) & ~((64 << 10) - 1);
-
-template<typename Mapping>
-uptr GetThreadTraceImpl(int tid) {
- uptr p = Mapping::kTraceMemBeg + (uptr)tid * kTotalTraceSize;
- DCHECK_LT(p, Mapping::kTraceMemEnd);
- return p;
-}
+struct MemToMetaImpl {
+ template <typename Mapping>
+ static u32 *Apply(uptr x) {
+ DCHECK(IsAppMemImpl::Apply<Mapping>(x));
+ return (u32 *)(((((x) & ~(Mapping::kShadowMsk | (kMetaShadowCell - 1)))) /
+ kMetaShadowCell * kMetaShadowSize) |
+ Mapping::kMetaShadowBeg);
+ }
+};
ALWAYS_INLINE
-uptr GetThreadTrace(int tid) {
-#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO
- switch (vmaSize) {
- case 39: return GetThreadTraceImpl<Mapping39>(tid);
- case 42: return GetThreadTraceImpl<Mapping42>(tid);
- case 48: return GetThreadTraceImpl<Mapping48>(tid);
+u32 *MemToMeta(uptr x) { return SelectMapping<MemToMetaImpl>(x); }
+
+struct ShadowToMemImpl {
+ template <typename Mapping>
+ static uptr Apply(uptr sp) {
+ if (!IsShadowMemImpl::Apply<Mapping>(sp))
+ return 0;
+ // The shadow mapping is non-linear and we've lost some bits, so we don't
+ // have an easy way to restore the original app address. But the mapping is
+ // a bijection, so we try to restore the address as belonging to
+ // low/mid/high range consecutively and see if shadow->app->shadow mapping
+ // gives us the same address.
+ uptr p =
+ ((sp - Mapping::kShadowAdd) / kShadowMultiplier) ^ Mapping::kShadowXor;
+ if (p >= Mapping::kLoAppMemBeg && p < Mapping::kLoAppMemEnd &&
+ MemToShadowImpl::Apply<Mapping>(p) == sp)
+ return p;
+ if (Mapping::kMidAppMemBeg) {
+ uptr p_mid = p + (Mapping::kMidAppMemBeg & Mapping::kShadowMsk);
+ if (p_mid >= Mapping::kMidAppMemBeg && p_mid < Mapping::kMidAppMemEnd &&
+ MemToShadowImpl::Apply<Mapping>(p_mid) == sp)
+ return p_mid;
+ }
+ return p | Mapping::kShadowMsk;
}
- DCHECK(0);
- return 0;
-#elif defined(__powerpc64__)
- switch (vmaSize) {
-#if !SANITIZER_GO
- case 44: return GetThreadTraceImpl<Mapping44>(tid);
-#endif
- case 46: return GetThreadTraceImpl<Mapping46>(tid);
- case 47: return GetThreadTraceImpl<Mapping47>(tid);
- }
- DCHECK(0);
- return 0;
-#elif defined(__mips64)
- switch (vmaSize) {
-#if !SANITIZER_GO
- case 40: return GetThreadTraceImpl<Mapping40>(tid);
-#else
- case 47: return GetThreadTraceImpl<Mapping47>(tid);
-#endif
- }
- DCHECK(0);
- return 0;
-#else
- return GetThreadTraceImpl<Mapping>(tid);
-#endif
-}
-
-
-template<typename Mapping>
-uptr GetThreadTraceHeaderImpl(int tid) {
- uptr p = Mapping::kTraceMemBeg + (uptr)tid * kTotalTraceSize
- + kTraceSize * sizeof(Event);
- DCHECK_LT(p, Mapping::kTraceMemEnd);
- return p;
-}
+};
ALWAYS_INLINE
-uptr GetThreadTraceHeader(int tid) {
-#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO
- switch (vmaSize) {
- case 39: return GetThreadTraceHeaderImpl<Mapping39>(tid);
- case 42: return GetThreadTraceHeaderImpl<Mapping42>(tid);
- case 48: return GetThreadTraceHeaderImpl<Mapping48>(tid);
- }
- DCHECK(0);
- return 0;
-#elif defined(__powerpc64__)
- switch (vmaSize) {
-#if !SANITIZER_GO
- case 44: return GetThreadTraceHeaderImpl<Mapping44>(tid);
-#endif
- case 46: return GetThreadTraceHeaderImpl<Mapping46>(tid);
- case 47: return GetThreadTraceHeaderImpl<Mapping47>(tid);
+uptr ShadowToMem(RawShadow *s) {
+ return SelectMapping<ShadowToMemImpl>(reinterpret_cast<uptr>(s));
+}
+
+// Compresses addr to kCompressedAddrBits stored in least significant bits.
+ALWAYS_INLINE uptr CompressAddr(uptr addr) {
+ return addr & ((1ull << kCompressedAddrBits) - 1);
+}
+
+struct RestoreAddrImpl {
+ typedef uptr Result;
+ template <typename Mapping>
+ static Result Apply(uptr addr) {
+ // To restore the address we go over all app memory ranges and check if top
+ // 3 bits of the compressed addr match that of the app range. If yes, we
+ // assume that the compressed address come from that range and restore the
+ // missing top bits to match the app range address.
+ const uptr ranges[] = {
+ Mapping::kLoAppMemBeg, Mapping::kLoAppMemEnd, Mapping::kMidAppMemBeg,
+ Mapping::kMidAppMemEnd, Mapping::kHiAppMemBeg, Mapping::kHiAppMemEnd,
+ Mapping::kHeapMemBeg, Mapping::kHeapMemEnd,
+ };
+ const uptr indicator = 0x0e0000000000ull;
+ const uptr ind_lsb = 1ull << LeastSignificantSetBitIndex(indicator);
+ for (uptr i = 0; i < ARRAY_SIZE(ranges); i += 2) {
+ uptr beg = ranges[i];
+ uptr end = ranges[i + 1];
+ if (beg == end)
+ continue;
+ for (uptr p = beg; p < end; p = RoundDown(p + ind_lsb, ind_lsb)) {
+ if ((addr & indicator) == (p & indicator))
+ return addr | (p & ~(ind_lsb - 1));
+ }
+ }
+ Printf("ThreadSanitizer: failed to restore address 0x%zx\n", addr);
+ Die();
}
- DCHECK(0);
- return 0;
-#elif defined(__mips64)
- switch (vmaSize) {
-#if !SANITIZER_GO
- case 40: return GetThreadTraceHeaderImpl<Mapping40>(tid);
-#else
- case 47: return GetThreadTraceHeaderImpl<Mapping47>(tid);
-#endif
- }
- DCHECK(0);
- return 0;
-#else
- return GetThreadTraceHeaderImpl<Mapping>(tid);
-#endif
+};
+
+// Restores compressed addr from kCompressedAddrBits to full representation.
+// This is called only during reporting and is not performance-critical.
+inline uptr RestoreAddr(uptr addr) {
+ return SelectMapping<RestoreAddrImpl>(addr);
}
void InitializePlatform();
void InitializePlatformEarly();
void CheckAndProtect();
void InitializeShadowMemoryPlatform();
-void FlushShadowMemory();
-void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive);
+void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns);
int ExtractResolvFDs(void *state, int *fds, int nfd);
int ExtractRecvmsgFDs(void *msg, int *fds, int nfd);
uptr ExtractLongJmpSp(uptr *env);
static uptr longjmp_xor_key;
#endif
-#ifdef TSAN_RUNTIME_VMA
// Runtime detected VMA size.
uptr vmaSize;
-#endif
enum {
- MemTotal = 0,
- MemShadow = 1,
- MemMeta = 2,
- MemFile = 3,
- MemMmap = 4,
- MemTrace = 5,
- MemHeap = 6,
- MemOther = 7,
- MemCount = 8,
+ MemTotal,
+ MemShadow,
+ MemMeta,
+ MemFile,
+ MemMmap,
+ MemHeap,
+ MemOther,
+ MemCount,
};
-void FillProfileCallback(uptr p, uptr rss, bool file,
- uptr *mem, uptr stats_size) {
+void FillProfileCallback(uptr p, uptr rss, bool file, uptr *mem) {
mem[MemTotal] += rss;
if (p >= ShadowBeg() && p < ShadowEnd())
mem[MemShadow] += rss;
else if (p >= MetaShadowBeg() && p < MetaShadowEnd())
mem[MemMeta] += rss;
-#if !SANITIZER_GO
+ else if ((p >= LoAppMemBeg() && p < LoAppMemEnd()) ||
+ (p >= MidAppMemBeg() && p < MidAppMemEnd()) ||
+ (p >= HiAppMemBeg() && p < HiAppMemEnd()))
+ mem[file ? MemFile : MemMmap] += rss;
else if (p >= HeapMemBeg() && p < HeapMemEnd())
mem[MemHeap] += rss;
- else if (p >= LoAppMemBeg() && p < LoAppMemEnd())
- mem[file ? MemFile : MemMmap] += rss;
- else if (p >= HiAppMemBeg() && p < HiAppMemEnd())
- mem[file ? MemFile : MemMmap] += rss;
-#else
- else if (p >= AppMemBeg() && p < AppMemEnd())
- mem[file ? MemFile : MemMmap] += rss;
-#endif
- else if (p >= TraceMemBeg() && p < TraceMemEnd())
- mem[MemTrace] += rss;
else
mem[MemOther] += rss;
}
-void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive) {
+void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns) {
uptr mem[MemCount];
- internal_memset(mem, 0, sizeof(mem[0]) * MemCount);
- __sanitizer::GetMemoryProfile(FillProfileCallback, mem, 7);
- StackDepotStats *stacks = StackDepotGetStats();
- internal_snprintf(buf, buf_size,
- "RSS %zd MB: shadow:%zd meta:%zd file:%zd mmap:%zd"
- " trace:%zd heap:%zd other:%zd stacks=%zd[%zd] nthr=%zd/%zd\n",
+ internal_memset(mem, 0, sizeof(mem));
+ GetMemoryProfile(FillProfileCallback, mem);
+ auto meta = ctx->metamap.GetMemoryStats();
+ StackDepotStats stacks = StackDepotGetStats();
+ uptr nthread, nlive;
+ ctx->thread_registry.GetNumberOfThreads(&nthread, &nlive);
+ uptr trace_mem;
+ {
+ Lock l(&ctx->slot_mtx);
+ trace_mem = ctx->trace_part_total_allocated * sizeof(TracePart);
+ }
+ uptr internal_stats[AllocatorStatCount];
+ internal_allocator()->GetStats(internal_stats);
+ // All these are allocated from the common mmap region.
+ mem[MemMmap] -= meta.mem_block + meta.sync_obj + trace_mem +
+ stacks.allocated + internal_stats[AllocatorStatMapped];
+ if (s64(mem[MemMmap]) < 0)
+ mem[MemMmap] = 0;
+ internal_snprintf(
+ buf, buf_size,
+ "==%zu== %llus [%zu]: RSS %zd MB: shadow:%zd meta:%zd file:%zd"
+ " mmap:%zd heap:%zd other:%zd intalloc:%zd memblocks:%zd syncobj:%zu"
+ " trace:%zu stacks=%zd threads=%zu/%zu\n",
+ internal_getpid(), uptime_ns / (1000 * 1000 * 1000), ctx->global_epoch,
mem[MemTotal] >> 20, mem[MemShadow] >> 20, mem[MemMeta] >> 20,
- mem[MemFile] >> 20, mem[MemMmap] >> 20, mem[MemTrace] >> 20,
- mem[MemHeap] >> 20, mem[MemOther] >> 20,
- stacks->allocated >> 20, stacks->n_uniq_ids,
- nlive, nthread);
-}
-
-#if SANITIZER_LINUX
-void FlushShadowMemoryCallback(
- const SuspendedThreadsList &suspended_threads_list,
- void *argument) {
- ReleaseMemoryPagesToOS(ShadowBeg(), ShadowEnd());
-}
-#endif
-
-void FlushShadowMemory() {
-#if SANITIZER_LINUX
- StopTheWorld(FlushShadowMemoryCallback, 0);
-#endif
+ mem[MemFile] >> 20, mem[MemMmap] >> 20, mem[MemHeap] >> 20,
+ mem[MemOther] >> 20, internal_stats[AllocatorStatMapped] >> 20,
+ meta.mem_block >> 20, meta.sync_obj >> 20, trace_mem >> 20,
+ stacks.allocated >> 20, nlive, nthread);
}
#if !SANITIZER_GO
-// Mark shadow for .rodata sections with the special kShadowRodata marker.
+// Mark shadow for .rodata sections with the special Shadow::kRodata marker.
// Accesses to .rodata can't race, so this saves time, memory and trace space.
static void MapRodata() {
// First create temp file.
return;
internal_unlink(name); // Unlink it now, so that we can reuse the buffer.
fd_t fd = openrv;
- // Fill the file with kShadowRodata.
- const uptr kMarkerSize = 512 * 1024 / sizeof(u64);
- InternalMmapVector<u64> marker(kMarkerSize);
+ // Fill the file with Shadow::kRodata.
+ const uptr kMarkerSize = 512 * 1024 / sizeof(RawShadow);
+ InternalMmapVector<RawShadow> marker(kMarkerSize);
// volatile to prevent insertion of memset
- for (volatile u64 *p = marker.data(); p < marker.data() + kMarkerSize; p++)
- *p = kShadowRodata;
- internal_write(fd, marker.data(), marker.size() * sizeof(u64));
+ for (volatile RawShadow *p = marker.data(); p < marker.data() + kMarkerSize;
+ p++)
+ *p = Shadow::kRodata;
+ internal_write(fd, marker.data(), marker.size() * sizeof(RawShadow));
// Map the file into memory.
uptr page = internal_mmap(0, GetPageSizeCached(), PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, fd, 0);
char *shadow_start = (char *)MemToShadow(segment.start);
char *shadow_end = (char *)MemToShadow(segment.end);
for (char *p = shadow_start; p < shadow_end;
- p += marker.size() * sizeof(u64)) {
- internal_mmap(p, Min<uptr>(marker.size() * sizeof(u64), shadow_end - p),
- PROT_READ, MAP_PRIVATE | MAP_FIXED, fd, 0);
+ p += marker.size() * sizeof(RawShadow)) {
+ internal_mmap(
+ p, Min<uptr>(marker.size() * sizeof(RawShadow), shadow_end - p),
+ PROT_READ, MAP_PRIVATE | MAP_FIXED, fd, 0);
}
}
}
#endif // #if !SANITIZER_GO
void InitializePlatformEarly() {
-#ifdef TSAN_RUNTIME_VMA
vmaSize =
(MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1);
#if defined(__aarch64__)
Die();
}
#endif
+#elif SANITIZER_LOONGARCH64
+# if !SANITIZER_GO
+ if (vmaSize != 47) {
+ Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
+ Printf("FATAL: Found %zd - Supported 47\n", vmaSize);
+ Die();
+ }
+# endif
#elif defined(__powerpc64__)
# if !SANITIZER_GO
if (vmaSize != 44 && vmaSize != 46 && vmaSize != 47) {
}
# endif
#endif
-#endif
}
void InitializePlatform() {
SetAddressSpaceUnlimited();
reexec = true;
}
-#if SANITIZER_LINUX && defined(__aarch64__)
+#if SANITIZER_ANDROID && (defined(__aarch64__) || defined(__x86_64__))
// After patch "arm64: mm: support ARCH_MMAP_RND_BITS." is introduced in
// linux kernel, the random gap between stack and mapped area is increased
// from 128M to 36G on 39-bit aarch64. As it is almost impossible to cover
// this big range, we should disable randomized virtual space on aarch64.
+ // ASLR personality check.
int old_personality = personality(0xffffffff);
if (old_personality != -1 && (old_personality & ADDR_NO_RANDOMIZE) == 0) {
VReport(1, "WARNING: Program is run with randomized virtual address "
CHECK_NE(personality(old_personality | ADDR_NO_RANDOMIZE), -1);
reexec = true;
}
+
+#endif
+#if SANITIZER_LINUX && defined(__aarch64__)
// Initialize the xor key used in {sig}{set,long}jump.
InitializeLongjmpXorKey();
#endif
}
// Extract file descriptors passed via UNIX domain sockets.
-// This is requried to properly handle "open" of these fds.
+// This is required to properly handle "open" of these fds.
// see 'man recvmsg' and 'man 3 cmsg'.
int ExtractRecvmsgFDs(void *msgp, int *fds, int nfd) {
int res = 0;
# else
return mangled_sp;
# endif
+#elif defined(__loongarch__)
+ return mangled_sp;
#elif defined(__powerpc64__)
// Reverse of:
// ld r4, -28696(r13)
#elif defined(__powerpc__)
# define LONG_JMP_SP_ENV_SLOT 0
#elif SANITIZER_FREEBSD
-# define LONG_JMP_SP_ENV_SLOT 2
+# ifdef __aarch64__
+# define LONG_JMP_SP_ENV_SLOT 1
+# else
+# define LONG_JMP_SP_ENV_SLOT 2
+# endif
#elif SANITIZER_LINUX
# ifdef __aarch64__
# define LONG_JMP_SP_ENV_SLOT 13
+# elif defined(__loongarch__)
+# define LONG_JMP_SP_ENV_SLOT 1
# elif defined(__mips64)
# define LONG_JMP_SP_ENV_SLOT 1
# elif defined(__s390x__)
}
#endif
+extern "C" void __tsan_tls_initialization() {}
+
void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size) {
// Check that the thr object is in tls;
const uptr thr_beg = (uptr)thr;
CHECK_GE(thr_end, tls_addr);
CHECK_LE(thr_end, tls_addr + tls_size);
// Since the thr object is huge, skip it.
- MemoryRangeImitateWrite(thr, /*pc=*/2, tls_addr, thr_beg - tls_addr);
- MemoryRangeImitateWrite(thr, /*pc=*/2, thr_end,
- tls_addr + tls_size - thr_end);
+ const uptr pc = StackTrace::GetNextInstructionPc(
+ reinterpret_cast<uptr>(__tsan_tls_initialization));
+ MemoryRangeImitateWrite(thr, pc, tls_addr, thr_beg - tls_addr);
+ MemoryRangeImitateWrite(thr, pc, thr_end, tls_addr + tls_size - thr_end);
}
// Note: this function runs with async signals enabled,
//===----------------------------------------------------------------------===//
#include "sanitizer_common/sanitizer_platform.h"
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
#include "sanitizer_common/sanitizer_atomic.h"
#include "sanitizer_common/sanitizer_common.h"
#include "tsan_rtl.h"
#include "tsan_flags.h"
+#include <limits.h>
#include <mach/mach.h>
#include <pthread.h>
#include <signal.h>
namespace __tsan {
#if !SANITIZER_GO
-static void *SignalSafeGetOrAllocate(uptr *dst, uptr size) {
- atomic_uintptr_t *a = (atomic_uintptr_t *)dst;
- void *val = (void *)atomic_load_relaxed(a);
- atomic_signal_fence(memory_order_acquire); // Turns the previous load into
- // acquire wrt signals.
- if (UNLIKELY(val == nullptr)) {
- val = (void *)internal_mmap(nullptr, size, PROT_READ | PROT_WRITE,
- MAP_PRIVATE | MAP_ANON, -1, 0);
- CHECK(val);
- void *cmp = nullptr;
- if (!atomic_compare_exchange_strong(a, (uintptr_t *)&cmp, (uintptr_t)val,
- memory_order_acq_rel)) {
- internal_munmap(val, size);
- val = cmp;
- }
- }
- return val;
+static char main_thread_state[sizeof(ThreadState)] ALIGNED(
+ SANITIZER_CACHE_LINE_SIZE);
+static ThreadState *dead_thread_state;
+static pthread_key_t thread_state_key;
+
+// We rely on the following documented, but Darwin-specific behavior to keep the
+// reference to the ThreadState object alive in TLS:
+// pthread_key_create man page:
+// If, after all the destructors have been called for all non-NULL values with
+// associated destructors, there are still some non-NULL values with
+// associated destructors, then the process is repeated. If, after at least
+// [PTHREAD_DESTRUCTOR_ITERATIONS] iterations of destructor calls for
+// outstanding non-NULL values, there are still some non-NULL values with
+// associated destructors, the implementation stops calling destructors.
+static_assert(PTHREAD_DESTRUCTOR_ITERATIONS == 4, "Small number of iterations");
+static void ThreadStateDestructor(void *thr) {
+ int res = pthread_setspecific(thread_state_key, thr);
+ CHECK_EQ(res, 0);
}
-// On OS X, accessing TLVs via __thread or manually by using pthread_key_* is
-// problematic, because there are several places where interceptors are called
-// when TLVs are not accessible (early process startup, thread cleanup, ...).
-// The following provides a "poor man's TLV" implementation, where we use the
-// shadow memory of the pointer returned by pthread_self() to store a pointer to
-// the ThreadState object. The main thread's ThreadState is stored separately
-// in a static variable, because we need to access it even before the
-// shadow memory is set up.
-static uptr main_thread_identity = 0;
-ALIGNED(64) static char main_thread_state[sizeof(ThreadState)];
-static ThreadState *main_thread_state_loc = (ThreadState *)main_thread_state;
-
-// We cannot use pthread_self() before libpthread has been initialized. Our
-// current heuristic for guarding this is checking `main_thread_identity` which
-// is only assigned in `__tsan::InitializePlatform`.
-static ThreadState **cur_thread_location() {
- if (main_thread_identity == 0)
- return &main_thread_state_loc;
- uptr thread_identity = (uptr)pthread_self();
- if (thread_identity == main_thread_identity)
- return &main_thread_state_loc;
- return (ThreadState **)MemToShadow(thread_identity);
+static void InitializeThreadStateStorage() {
+ int res;
+ CHECK_EQ(thread_state_key, 0);
+ res = pthread_key_create(&thread_state_key, ThreadStateDestructor);
+ CHECK_EQ(res, 0);
+ res = pthread_setspecific(thread_state_key, main_thread_state);
+ CHECK_EQ(res, 0);
+
+ auto dts = (ThreadState *)MmapOrDie(sizeof(ThreadState), "ThreadState");
+ dts->fast_state.SetIgnoreBit();
+ dts->ignore_interceptors = 1;
+ dts->is_dead = true;
+ const_cast<Tid &>(dts->tid) = kInvalidTid;
+ res = internal_mprotect(dts, sizeof(ThreadState), PROT_READ); // immutable
+ CHECK_EQ(res, 0);
+ dead_thread_state = dts;
}
ThreadState *cur_thread() {
- return (ThreadState *)SignalSafeGetOrAllocate(
- (uptr *)cur_thread_location(), sizeof(ThreadState));
+ // Some interceptors get called before libpthread has been initialized and in
+ // these cases we must avoid calling any pthread APIs.
+ if (UNLIKELY(!thread_state_key)) {
+ return (ThreadState *)main_thread_state;
+ }
+
+ // We only reach this line after InitializeThreadStateStorage() ran, i.e,
+ // after TSan (and therefore libpthread) have been initialized.
+ ThreadState *thr = (ThreadState *)pthread_getspecific(thread_state_key);
+ if (UNLIKELY(!thr)) {
+ thr = (ThreadState *)MmapOrDie(sizeof(ThreadState), "ThreadState");
+ int res = pthread_setspecific(thread_state_key, thr);
+ CHECK_EQ(res, 0);
+ }
+ return thr;
}
void set_cur_thread(ThreadState *thr) {
- *cur_thread_location() = thr;
+ int res = pthread_setspecific(thread_state_key, thr);
+ CHECK_EQ(res, 0);
}
-// TODO(kuba.brecka): This is not async-signal-safe. In particular, we call
-// munmap first and then clear `fake_tls`; if we receive a signal in between,
-// handler will try to access the unmapped ThreadState.
void cur_thread_finalize() {
- ThreadState **thr_state_loc = cur_thread_location();
- if (thr_state_loc == &main_thread_state_loc) {
+ ThreadState *thr = (ThreadState *)pthread_getspecific(thread_state_key);
+ CHECK(thr);
+ if (thr == (ThreadState *)main_thread_state) {
// Calling dispatch_main() or xpc_main() actually invokes pthread_exit to
// exit the main thread. Let's keep the main thread's ThreadState.
return;
}
- internal_munmap(*thr_state_loc, sizeof(ThreadState));
- *thr_state_loc = nullptr;
+ // Intercepted functions can still get called after cur_thread_finalize()
+ // (called from DestroyThreadState()), so put a fake thread state for "dead"
+ // threads. An alternative solution would be to release the ThreadState
+ // object from THREAD_DESTROY (which is delivered later and on the parent
+ // thread) instead of THREAD_TERMINATE.
+ int res = pthread_setspecific(thread_state_key, dead_thread_state);
+ CHECK_EQ(res, 0);
+ UnmapOrDie(thr, sizeof(ThreadState));
}
#endif
-void FlushShadowMemory() {
-}
-
static void RegionMemUsage(uptr start, uptr end, uptr *res, uptr *dirty) {
vm_address_t address = start;
vm_address_t end_address = end;
*dirty = dirty_pages * GetPageSizeCached();
}
-void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive) {
+void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns) {
uptr shadow_res, shadow_dirty;
uptr meta_res, meta_dirty;
- uptr trace_res, trace_dirty;
RegionMemUsage(ShadowBeg(), ShadowEnd(), &shadow_res, &shadow_dirty);
RegionMemUsage(MetaShadowBeg(), MetaShadowEnd(), &meta_res, &meta_dirty);
- RegionMemUsage(TraceMemBeg(), TraceMemEnd(), &trace_res, &trace_dirty);
-#if !SANITIZER_GO
+# if !SANITIZER_GO
uptr low_res, low_dirty;
uptr high_res, high_dirty;
uptr heap_res, heap_dirty;
RegionMemUsage(HeapMemBeg(), HeapMemEnd(), &heap_res, &heap_dirty);
#else // !SANITIZER_GO
uptr app_res, app_dirty;
- RegionMemUsage(AppMemBeg(), AppMemEnd(), &app_res, &app_dirty);
+ RegionMemUsage(LoAppMemBeg(), LoAppMemEnd(), &app_res, &app_dirty);
#endif
- StackDepotStats *stacks = StackDepotGetStats();
- internal_snprintf(buf, buf_size,
- "shadow (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
- "meta (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
- "traces (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
-#if !SANITIZER_GO
- "low app (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
- "high app (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
- "heap (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
-#else // !SANITIZER_GO
- "app (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
-#endif
- "stacks: %zd unique IDs, %zd kB allocated\n"
- "threads: %zd total, %zd live\n"
- "------------------------------\n",
- ShadowBeg(), ShadowEnd(), shadow_res / 1024, shadow_dirty / 1024,
- MetaShadowBeg(), MetaShadowEnd(), meta_res / 1024, meta_dirty / 1024,
- TraceMemBeg(), TraceMemEnd(), trace_res / 1024, trace_dirty / 1024,
-#if !SANITIZER_GO
- LoAppMemBeg(), LoAppMemEnd(), low_res / 1024, low_dirty / 1024,
- HiAppMemBeg(), HiAppMemEnd(), high_res / 1024, high_dirty / 1024,
- HeapMemBeg(), HeapMemEnd(), heap_res / 1024, heap_dirty / 1024,
-#else // !SANITIZER_GO
- AppMemBeg(), AppMemEnd(), app_res / 1024, app_dirty / 1024,
-#endif
- stacks->n_uniq_ids, stacks->allocated / 1024,
- nthread, nlive);
+ StackDepotStats stacks = StackDepotGetStats();
+ uptr nthread, nlive;
+ ctx->thread_registry.GetNumberOfThreads(&nthread, &nlive);
+ internal_snprintf(
+ buf, buf_size,
+ "shadow (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
+ "meta (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
+# if !SANITIZER_GO
+ "low app (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
+ "high app (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
+ "heap (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
+# else // !SANITIZER_GO
+ "app (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
+# endif
+ "stacks: %zd unique IDs, %zd kB allocated\n"
+ "threads: %zd total, %zd live\n"
+ "------------------------------\n",
+ ShadowBeg(), ShadowEnd(), shadow_res / 1024, shadow_dirty / 1024,
+ MetaShadowBeg(), MetaShadowEnd(), meta_res / 1024, meta_dirty / 1024,
+# if !SANITIZER_GO
+ LoAppMemBeg(), LoAppMemEnd(), low_res / 1024, low_dirty / 1024,
+ HiAppMemBeg(), HiAppMemEnd(), high_res / 1024, high_dirty / 1024,
+ HeapMemBeg(), HeapMemEnd(), heap_res / 1024, heap_dirty / 1024,
+# else // !SANITIZER_GO
+ LoAppMemBeg(), LoAppMemEnd(), app_res / 1024, app_dirty / 1024,
+# endif
+ stacks.n_uniq_ids, stacks.allocated / 1024, nthread, nlive);
}
-#if !SANITIZER_GO
+# if !SANITIZER_GO
void InitializeShadowMemoryPlatform() { }
-// On OS X, GCD worker threads are created without a call to pthread_create. We
-// need to properly register these threads with ThreadCreate and ThreadStart.
-// These threads don't have a parent thread, as they are created "spuriously".
-// We're using a libpthread API that notifies us about a newly created thread.
-// The `thread == pthread_self()` check indicates this is actually a worker
-// thread. If it's just a regular thread, this hook is called on the parent
-// thread.
-typedef void (*pthread_introspection_hook_t)(unsigned int event,
- pthread_t thread, void *addr,
- size_t size);
-extern "C" pthread_introspection_hook_t pthread_introspection_hook_install(
- pthread_introspection_hook_t hook);
-static const uptr PTHREAD_INTROSPECTION_THREAD_CREATE = 1;
-static const uptr PTHREAD_INTROSPECTION_THREAD_TERMINATE = 3;
-static pthread_introspection_hook_t prev_pthread_introspection_hook;
-static void my_pthread_introspection_hook(unsigned int event, pthread_t thread,
- void *addr, size_t size) {
- if (event == PTHREAD_INTROSPECTION_THREAD_CREATE) {
- if (thread == pthread_self()) {
- // The current thread is a newly created GCD worker thread.
- ThreadState *thr = cur_thread();
- Processor *proc = ProcCreate();
- ProcWire(proc, thr);
- ThreadState *parent_thread_state = nullptr; // No parent.
- int tid = ThreadCreate(parent_thread_state, 0, (uptr)thread, true);
- CHECK_NE(tid, 0);
- ThreadStart(thr, tid, GetTid(), ThreadType::Worker);
- }
- } else if (event == PTHREAD_INTROSPECTION_THREAD_TERMINATE) {
- if (thread == pthread_self()) {
- ThreadState *thr = cur_thread();
- if (thr->tctx) {
- DestroyThreadState();
- }
- }
+// Register GCD worker threads, which are created without an observable call to
+// pthread_create().
+static void ThreadCreateCallback(uptr thread, bool gcd_worker) {
+ if (gcd_worker) {
+ ThreadState *thr = cur_thread();
+ Processor *proc = ProcCreate();
+ ProcWire(proc, thr);
+ ThreadState *parent_thread_state = nullptr; // No parent.
+ Tid tid = ThreadCreate(parent_thread_state, 0, (uptr)thread, true);
+ CHECK_NE(tid, kMainTid);
+ ThreadStart(thr, tid, GetTid(), ThreadType::Worker);
}
+}
- if (prev_pthread_introspection_hook != nullptr)
- prev_pthread_introspection_hook(event, thread, addr, size);
+// Destroy thread state for *all* threads.
+static void ThreadTerminateCallback(uptr thread) {
+ ThreadState *thr = cur_thread();
+ if (thr->tctx) {
+ DestroyThreadState();
+ }
}
#endif
void InitializePlatformEarly() {
-#if !SANITIZER_GO && !HAS_48_BIT_ADDRESS_SPACE
+# if !SANITIZER_GO && SANITIZER_IOS
uptr max_vm = GetMaxUserVirtualAddress() + 1;
- if (max_vm != Mapping::kHiAppMemEnd) {
+ if (max_vm != HiAppMemEnd()) {
Printf("ThreadSanitizer: unsupported vm address limit %p, expected %p.\n",
- max_vm, Mapping::kHiAppMemEnd);
+ (void *)max_vm, (void *)HiAppMemEnd());
Die();
}
#endif
#if !SANITIZER_GO
CheckAndProtect();
- CHECK_EQ(main_thread_identity, 0);
- main_thread_identity = (uptr)pthread_self();
+ InitializeThreadStateStorage();
- prev_pthread_introspection_hook =
- pthread_introspection_hook_install(&my_pthread_introspection_hook);
+ ThreadEventCallbacks callbacks = {
+ .create = ThreadCreateCallback,
+ .terminate = ThreadTerminateCallback,
+ };
+ InstallPthreadIntrospectionHook(callbacks);
#endif
if (GetMacosAlignedVersion() >= MacosVersion(10, 14)) {
}
#if !SANITIZER_GO
+extern "C" void __tsan_tls_initialization() {}
+
void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size) {
- // The pointer to the ThreadState object is stored in the shadow memory
- // of the tls.
- uptr tls_end = tls_addr + tls_size;
- uptr thread_identity = (uptr)pthread_self();
- if (thread_identity == main_thread_identity) {
- MemoryRangeImitateWrite(thr, /*pc=*/2, tls_addr, tls_size);
- } else {
- uptr thr_state_start = thread_identity;
- uptr thr_state_end = thr_state_start + sizeof(uptr);
- CHECK_GE(thr_state_start, tls_addr);
- CHECK_LE(thr_state_start, tls_addr + tls_size);
- CHECK_GE(thr_state_end, tls_addr);
- CHECK_LE(thr_state_end, tls_addr + tls_size);
- MemoryRangeImitateWrite(thr, /*pc=*/2, tls_addr,
- thr_state_start - tls_addr);
- MemoryRangeImitateWrite(thr, /*pc=*/2, thr_state_end,
- tls_end - thr_state_end);
- }
+ const uptr pc = StackTrace::GetNextInstructionPc(
+ reinterpret_cast<uptr>(__tsan_tls_initialization));
+ // Unlike Linux, we only store a pointer to the ThreadState object in TLS;
+ // just mark the entire range as written to.
+ MemoryRangeImitateWrite(thr, pc, tls_addr, tls_size);
}
#endif
} // namespace __tsan
-#endif // SANITIZER_MAC
+#endif // SANITIZER_APPLE
#include "sanitizer_common/sanitizer_platform.h"
#if SANITIZER_POSIX
-#include "sanitizer_common/sanitizer_common.h"
-#include "sanitizer_common/sanitizer_errno.h"
-#include "sanitizer_common/sanitizer_libc.h"
-#include "sanitizer_common/sanitizer_procmaps.h"
-#include "tsan_platform.h"
-#include "tsan_rtl.h"
+# include <dlfcn.h>
+
+# include "sanitizer_common/sanitizer_common.h"
+# include "sanitizer_common/sanitizer_errno.h"
+# include "sanitizer_common/sanitizer_libc.h"
+# include "sanitizer_common/sanitizer_procmaps.h"
+# include "tsan_platform.h"
+# include "tsan_rtl.h"
namespace __tsan {
"HINT: if %s is not supported in your environment, you may set "
"TSAN_OPTIONS=%s=0\n";
-static void DontDumpShadow(uptr addr, uptr size) {
+# if !SANITIZER_GO
+void DontDumpShadow(uptr addr, uptr size) {
if (common_flags()->use_madv_dontdump)
if (!DontDumpShadowMemory(addr, size)) {
Printf(kShadowMemoryMappingWarning, SanitizerToolName, addr, addr + size,
}
}
-#if !SANITIZER_GO
void InitializeShadowMemory() {
// Map memory shadow.
if (!MmapFixedSuperNoReserve(ShadowBeg(), ShadowEnd() - ShadowBeg(),
meta, meta + meta_size, meta_size >> 30);
InitializeShadowMemoryPlatform();
+
+ on_initialize = reinterpret_cast<void (*)(void)>(
+ dlsym(RTLD_DEFAULT, "__tsan_on_initialize"));
+ on_finalize =
+ reinterpret_cast<int (*)(int)>(dlsym(RTLD_DEFAULT, "__tsan_on_finalize"));
}
static bool TryProtectRange(uptr beg, uptr end) {
continue;
if (segment.start >= VdsoBeg()) // vdso
break;
- Printf("FATAL: ThreadSanitizer: unexpected memory mapping %p-%p\n",
+ Printf("FATAL: ThreadSanitizer: unexpected memory mapping 0x%zx-0x%zx\n",
segment.start, segment.end);
Die();
}
-#if defined(__aarch64__) && defined(__APPLE__) && !HAS_48_BIT_ADDRESS_SPACE
+# if SANITIZER_IOS && !SANITIZER_IOSSIM
ProtectRange(HeapMemEnd(), ShadowBeg());
ProtectRange(ShadowEnd(), MetaShadowBeg());
- ProtectRange(MetaShadowEnd(), TraceMemBeg());
-#else
+ ProtectRange(MetaShadowEnd(), HiAppMemBeg());
+# else
ProtectRange(LoAppMemEnd(), ShadowBeg());
ProtectRange(ShadowEnd(), MetaShadowBeg());
-#ifdef TSAN_MID_APP_RANGE
- ProtectRange(MetaShadowEnd(), MidAppMemBeg());
- ProtectRange(MidAppMemEnd(), TraceMemBeg());
-#else
- ProtectRange(MetaShadowEnd(), TraceMemBeg());
-#endif
- // Memory for traces is mapped lazily in MapThreadTrace.
- // Protect the whole range for now, so that user does not map something here.
- ProtectRange(TraceMemBeg(), TraceMemEnd());
- ProtectRange(TraceMemEnd(), HeapMemBeg());
+ if (MidAppMemBeg()) {
+ ProtectRange(MetaShadowEnd(), MidAppMemBeg());
+ ProtectRange(MidAppMemEnd(), HeapMemBeg());
+ } else {
+ ProtectRange(MetaShadowEnd(), HeapMemBeg());
+ }
ProtectRange(HeapEnd(), HiAppMemBeg());
-#endif
+# endif
-#if defined(__s390x__)
+# if defined(__s390x__)
// Protect the rest of the address space.
const uptr user_addr_max_l4 = 0x0020000000000000ull;
const uptr user_addr_max_l5 = 0xfffffffffffff000ull;
namespace __tsan {
-void FlushShadowMemory() {
-}
-
-void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive) {
-}
+void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns) {}
void InitializePlatformEarly() {
}
namespace __tsan {
-ReportStack::ReportStack() : frames(nullptr), suppressable(false) {}
-
-ReportStack *ReportStack::New() {
- void *mem = internal_alloc(MBlockReportStack, sizeof(ReportStack));
- return new(mem) ReportStack();
-}
-
-ReportLocation::ReportLocation(ReportLocationType type)
- : type(type), global(), heap_chunk_start(0), heap_chunk_size(0), tid(0),
- fd(0), suppressable(false), stack(nullptr) {}
-
-ReportLocation *ReportLocation::New(ReportLocationType type) {
- void *mem = internal_alloc(MBlockReportStack, sizeof(ReportLocation));
- return new(mem) ReportLocation(type);
-}
-
class Decorator: public __sanitizer::SanitizerCommonDecorator {
public:
Decorator() : SanitizerCommonDecorator() { }
#if !SANITIZER_GO
const int kThreadBufSize = 32;
-const char *thread_name(char *buf, int tid) {
+const char *thread_name(char *buf, Tid tid) {
if (tid == kMainTid)
return "main thread";
internal_snprintf(buf, kThreadBufSize, "thread T%d", tid);
UNREACHABLE("missing case");
}
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
static const char *const kInterposedFunctionPrefix = "wrap_";
#else
static const char *const kInterposedFunctionPrefix = "__interceptor_";
if (i == 0)
Printf(" (mutexes:");
const ReportMopMutex m = mset[i];
- Printf(" %s M%llu", m.write ? "write" : "read", m.id);
+ Printf(" %s M%u", m.write ? "write" : "read", m.id);
Printf(i == mset.Size() - 1 ? ")" : ",");
}
}
if (loc->type == ReportLocationGlobal) {
const DataInfo &global = loc->global;
if (global.size != 0)
- Printf(" Location is global '%s' of size %zu at %p (%s+%p)\n\n",
- global.name, global.size, global.start,
+ Printf(" Location is global '%s' of size %zu at %p (%s+0x%zx)\n\n",
+ global.name, global.size, reinterpret_cast<void *>(global.start),
StripModuleName(global.module), global.module_offset);
else
- Printf(" Location is global '%s' at %p (%s+%p)\n\n", global.name,
- global.start, StripModuleName(global.module),
- global.module_offset);
+ Printf(" Location is global '%s' at %p (%s+0x%zx)\n\n", global.name,
+ reinterpret_cast<void *>(global.start),
+ StripModuleName(global.module), global.module_offset);
} else if (loc->type == ReportLocationHeap) {
char thrbuf[kThreadBufSize];
const char *object_type = GetObjectTypeFromTag(loc->external_tag);
if (!object_type) {
Printf(" Location is heap block of size %zu at %p allocated by %s:\n",
- loc->heap_chunk_size, loc->heap_chunk_start,
+ loc->heap_chunk_size,
+ reinterpret_cast<void *>(loc->heap_chunk_start),
thread_name(thrbuf, loc->tid));
} else {
Printf(" Location is %s of size %zu at %p allocated by %s:\n",
- object_type, loc->heap_chunk_size, loc->heap_chunk_start,
+ object_type, loc->heap_chunk_size,
+ reinterpret_cast<void *>(loc->heap_chunk_start),
thread_name(thrbuf, loc->tid));
}
print_stack = true;
} else if (loc->type == ReportLocationTLS) {
Printf(" Location is TLS of %s.\n\n", thread_name(thrbuf, loc->tid));
} else if (loc->type == ReportLocationFD) {
- Printf(" Location is file descriptor %d created by %s at:\n",
- loc->fd, thread_name(thrbuf, loc->tid));
+ Printf(" Location is file descriptor %d %s by %s at:\n", loc->fd,
+ loc->fd_closed ? "destroyed" : "created",
+ thread_name(thrbuf, loc->tid));
print_stack = true;
}
Printf("%s", d.Default());
static void PrintMutexShort(const ReportMutex *rm, const char *after) {
Decorator d;
- Printf("%sM%zd%s%s", d.Mutex(), rm->id, d.Default(), after);
+ Printf("%sM%d%s%s", d.Mutex(), rm->id, d.Default(), after);
}
static void PrintMutexShortWithAddress(const ReportMutex *rm,
const char *after) {
Decorator d;
- Printf("%sM%zd (%p)%s%s", d.Mutex(), rm->id, rm->addr, d.Default(), after);
+ Printf("%sM%d (%p)%s%s", d.Mutex(), rm->id,
+ reinterpret_cast<void *>(rm->addr), d.Default(), after);
}
static void PrintMutex(const ReportMutex *rm) {
Decorator d;
- if (rm->destroyed) {
- Printf("%s", d.Mutex());
- Printf(" Mutex M%llu is already destroyed.\n\n", rm->id);
- Printf("%s", d.Default());
- } else {
- Printf("%s", d.Mutex());
- Printf(" Mutex M%llu (%p) created at:\n", rm->id, rm->addr);
- Printf("%s", d.Default());
- PrintStack(rm->stack);
- }
+ Printf("%s", d.Mutex());
+ Printf(" Mutex M%u (%p) created at:\n", rm->id,
+ reinterpret_cast<void *>(rm->addr));
+ Printf("%s", d.Default());
+ PrintStack(rm->stack);
}
static void PrintThread(const ReportThread *rt) {
char thrbuf[kThreadBufSize];
const char *thread_status = rt->running ? "running" : "finished";
if (rt->thread_type == ThreadType::Worker) {
- Printf(" (tid=%zu, %s) is a GCD worker thread\n", rt->os_id, thread_status);
+ Printf(" (tid=%llu, %s) is a GCD worker thread\n", rt->os_id,
+ thread_status);
Printf("\n");
Printf("%s", d.Default());
return;
}
- Printf(" (tid=%zu, %s) created by %s", rt->os_id, thread_status,
+ Printf(" (tid=%llu, %s) created by %s", rt->os_id, thread_status,
thread_name(thrbuf, rt->parent_tid));
if (rt->stack)
Printf(" at:");
(int)internal_getpid());
Printf("%s", d.Default());
+ if (rep->typ == ReportTypeErrnoInSignal)
+ Printf(" Signal %u handler invoked at:\n", rep->signum);
+
if (rep->typ == ReportTypeDeadlock) {
char thrbuf[kThreadBufSize];
Printf(" Cycle in lock order graph: ");
#else // #if !SANITIZER_GO
-const u32 kMainGoroutineId = 1;
+const Tid kMainGoroutineId = 1;
void PrintStack(const ReportStack *ent) {
if (ent == 0 || ent->frames == 0) {
for (int i = 0; frame; frame = frame->next, i++) {
const AddressInfo &info = frame->info;
Printf(" %s()\n %s:%d +0x%zx\n", info.function,
- StripPathPrefix(info.file, common_flags()->strip_path_prefix),
- info.line, (void *)info.module_offset);
+ StripPathPrefix(info.file, common_flags()->strip_path_prefix),
+ info.line, info.module_offset);
}
}
static void PrintMop(const ReportMop *mop, bool first) {
Printf("\n");
Printf("%s at %p by ",
- (first ? (mop->write ? "Write" : "Read")
- : (mop->write ? "Previous write" : "Previous read")), mop->addr);
+ (first ? (mop->write ? "Write" : "Read")
+ : (mop->write ? "Previous write" : "Previous read")),
+ reinterpret_cast<void *>(mop->addr));
if (mop->tid == kMainGoroutineId)
Printf("main goroutine:\n");
else
switch (loc->type) {
case ReportLocationHeap: {
Printf("\n");
- Printf("Heap block of size %zu at %p allocated by ",
- loc->heap_chunk_size, loc->heap_chunk_start);
+ Printf("Heap block of size %zu at %p allocated by ", loc->heap_chunk_size,
+ reinterpret_cast<void *>(loc->heap_chunk_start));
if (loc->tid == kMainGoroutineId)
Printf("main goroutine:\n");
else
case ReportLocationGlobal: {
Printf("\n");
Printf("Global var %s of size %zu at %p declared at %s:%zu\n",
- loc->global.name, loc->global.size, loc->global.start,
- loc->global.file, loc->global.line);
+ loc->global.name, loc->global.size,
+ reinterpret_cast<void *>(loc->global.start), loc->global.file,
+ loc->global.line);
break;
}
default:
} else if (rep->typ == ReportTypeDeadlock) {
Printf("WARNING: DEADLOCK\n");
for (uptr i = 0; i < rep->mutexes.Size(); i++) {
- Printf("Goroutine %d lock mutex %d while holding mutex %d:\n",
- 999, rep->mutexes[i]->id,
- rep->mutexes[(i+1) % rep->mutexes.Size()]->id);
+ Printf("Goroutine %d lock mutex %u while holding mutex %u:\n", 999,
+ rep->mutexes[i]->id,
+ rep->mutexes[(i + 1) % rep->mutexes.Size()]->id);
PrintStack(rep->stacks[2*i]);
Printf("\n");
- Printf("Mutex %d was previously locked here:\n",
- rep->mutexes[(i+1) % rep->mutexes.Size()]->id);
+ Printf("Mutex %u was previously locked here:\n",
+ rep->mutexes[(i + 1) % rep->mutexes.Size()]->id);
PrintStack(rep->stacks[2*i + 1]);
Printf("\n");
}
};
struct ReportStack {
- SymbolizedStack *frames;
- bool suppressable;
- static ReportStack *New();
-
- private:
- ReportStack();
+ SymbolizedStack *frames = nullptr;
+ bool suppressable = false;
};
struct ReportMopMutex {
- u64 id;
+ int id;
bool write;
};
};
struct ReportLocation {
- ReportLocationType type;
- DataInfo global;
- uptr heap_chunk_start;
- uptr heap_chunk_size;
- uptr external_tag;
- int tid;
- int fd;
- bool suppressable;
- ReportStack *stack;
-
- static ReportLocation *New(ReportLocationType type);
- private:
- explicit ReportLocation(ReportLocationType type);
+ ReportLocationType type = ReportLocationGlobal;
+ DataInfo global = {};
+ uptr heap_chunk_start = 0;
+ uptr heap_chunk_size = 0;
+ uptr external_tag = 0;
+ Tid tid = kInvalidTid;
+ int fd = 0;
+ bool fd_closed = false;
+ bool suppressable = false;
+ ReportStack *stack = nullptr;
};
struct ReportThread {
- int id;
+ Tid id;
tid_t os_id;
bool running;
ThreadType thread_type;
char *name;
- u32 parent_tid;
+ Tid parent_tid;
ReportStack *stack;
};
struct ReportMutex {
- u64 id;
+ int id;
uptr addr;
- bool destroyed;
ReportStack *stack;
};
Vector<ReportLocation*> locs;
Vector<ReportMutex*> mutexes;
Vector<ReportThread*> threads;
- Vector<int> unique_tids;
+ Vector<Tid> unique_tids;
ReportStack *sleep;
int count;
+ int signum = 0;
ReportDesc();
~ReportDesc();
#include "sanitizer_common/sanitizer_atomic.h"
#include "sanitizer_common/sanitizer_common.h"
#include "sanitizer_common/sanitizer_file.h"
+#include "sanitizer_common/sanitizer_interface_internal.h"
#include "sanitizer_common/sanitizer_libc.h"
#include "sanitizer_common/sanitizer_placement_new.h"
#include "sanitizer_common/sanitizer_stackdepot.h"
#include "tsan_symbolize.h"
#include "ubsan/ubsan_init.h"
-#ifdef __SSE3__
-// <emmintrin.h> transitively includes <stdlib.h>,
-// and it's prohibited to include std headers into tsan runtime.
-// So we do this dirty trick.
-#define _MM_MALLOC_H_INCLUDED
-#define __MM_MALLOC_H
-#include <emmintrin.h>
-typedef __m128i m128;
-#endif
-
volatile int __tsan_resumed = 0;
extern "C" void __tsan_resume() {
__tsan_resumed = 1;
}
+SANITIZER_WEAK_DEFAULT_IMPL
+void __tsan_test_only_on_fork() {}
+
namespace __tsan {
-#if !SANITIZER_GO && !SANITIZER_MAC
+#if !SANITIZER_GO
+void (*on_initialize)(void);
+int (*on_finalize)(int);
+#endif
+
+#if !SANITIZER_GO && !SANITIZER_APPLE
__attribute__((tls_model("initial-exec")))
-THREADLOCAL char cur_thread_placeholder[sizeof(ThreadState)] ALIGNED(64);
+THREADLOCAL char cur_thread_placeholder[sizeof(ThreadState)] ALIGNED(
+ SANITIZER_CACHE_LINE_SIZE);
#endif
-static char ctx_placeholder[sizeof(Context)] ALIGNED(64);
+static char ctx_placeholder[sizeof(Context)] ALIGNED(SANITIZER_CACHE_LINE_SIZE);
Context *ctx;
// Can be overriden by a front-end.
bool OnFinalize(bool failed);
void OnInitialize();
#else
-#include <dlfcn.h>
SANITIZER_WEAK_CXX_DEFAULT_IMPL
bool OnFinalize(bool failed) {
-#if !SANITIZER_GO
- if (auto *ptr = dlsym(RTLD_DEFAULT, "__tsan_on_finalize"))
- return reinterpret_cast<decltype(&__tsan_on_finalize)>(ptr)(failed);
-#endif
+# if !SANITIZER_GO
+ if (on_finalize)
+ return on_finalize(failed);
+# endif
return failed;
}
+
SANITIZER_WEAK_CXX_DEFAULT_IMPL
void OnInitialize() {
-#if !SANITIZER_GO
- if (auto *ptr = dlsym(RTLD_DEFAULT, "__tsan_on_initialize")) {
- return reinterpret_cast<decltype(&__tsan_on_initialize)>(ptr)();
- }
+# if !SANITIZER_GO
+ if (on_initialize)
+ on_initialize();
+# endif
+}
#endif
+
+static TracePart* TracePartAlloc(ThreadState* thr) {
+ TracePart* part = nullptr;
+ {
+ Lock lock(&ctx->slot_mtx);
+ uptr max_parts = Trace::kMinParts + flags()->history_size;
+ Trace* trace = &thr->tctx->trace;
+ if (trace->parts_allocated == max_parts ||
+ ctx->trace_part_finished_excess) {
+ part = ctx->trace_part_recycle.PopFront();
+ DPrintf("#%d: TracePartAlloc: part=%p\n", thr->tid, part);
+ if (part && part->trace) {
+ Trace* trace1 = part->trace;
+ Lock trace_lock(&trace1->mtx);
+ part->trace = nullptr;
+ TracePart* part1 = trace1->parts.PopFront();
+ CHECK_EQ(part, part1);
+ if (trace1->parts_allocated > trace1->parts.Size()) {
+ ctx->trace_part_finished_excess +=
+ trace1->parts_allocated - trace1->parts.Size();
+ trace1->parts_allocated = trace1->parts.Size();
+ }
+ }
+ }
+ if (trace->parts_allocated < max_parts) {
+ trace->parts_allocated++;
+ if (ctx->trace_part_finished_excess)
+ ctx->trace_part_finished_excess--;
+ }
+ if (!part)
+ ctx->trace_part_total_allocated++;
+ else if (ctx->trace_part_recycle_finished)
+ ctx->trace_part_recycle_finished--;
+ }
+ if (!part)
+ part = new (MmapOrDie(sizeof(*part), "TracePart")) TracePart();
+ return part;
+}
+
+static void TracePartFree(TracePart* part) SANITIZER_REQUIRES(ctx->slot_mtx) {
+ DCHECK(part->trace);
+ part->trace = nullptr;
+ ctx->trace_part_recycle.PushFront(part);
+}
+
+void TraceResetForTesting() {
+ Lock lock(&ctx->slot_mtx);
+ while (auto* part = ctx->trace_part_recycle.PopFront()) {
+ if (auto trace = part->trace)
+ CHECK_EQ(trace->parts.PopFront(), part);
+ UnmapOrDie(part, sizeof(*part));
+ }
+ ctx->trace_part_total_allocated = 0;
+ ctx->trace_part_recycle_finished = 0;
+ ctx->trace_part_finished_excess = 0;
}
+
+static void DoResetImpl(uptr epoch) {
+ ThreadRegistryLock lock0(&ctx->thread_registry);
+ Lock lock1(&ctx->slot_mtx);
+ CHECK_EQ(ctx->global_epoch, epoch);
+ ctx->global_epoch++;
+ CHECK(!ctx->resetting);
+ ctx->resetting = true;
+ for (u32 i = ctx->thread_registry.NumThreadsLocked(); i--;) {
+ ThreadContext* tctx = (ThreadContext*)ctx->thread_registry.GetThreadLocked(
+ static_cast<Tid>(i));
+ // Potentially we could purge all ThreadStatusDead threads from the
+ // registry. Since we reset all shadow, they can't race with anything
+ // anymore. However, their tid's can still be stored in some aux places
+ // (e.g. tid of thread that created something).
+ auto trace = &tctx->trace;
+ Lock lock(&trace->mtx);
+ bool attached = tctx->thr && tctx->thr->slot;
+ auto parts = &trace->parts;
+ bool local = false;
+ while (!parts->Empty()) {
+ auto part = parts->Front();
+ local = local || part == trace->local_head;
+ if (local)
+ CHECK(!ctx->trace_part_recycle.Queued(part));
+ else
+ ctx->trace_part_recycle.Remove(part);
+ if (attached && parts->Size() == 1) {
+ // The thread is running and this is the last/current part.
+ // Set the trace position to the end of the current part
+ // to force the thread to call SwitchTracePart and re-attach
+ // to a new slot and allocate a new trace part.
+ // Note: the thread is concurrently modifying the position as well,
+ // so this is only best-effort. The thread can only modify position
+ // within this part, because switching parts is protected by
+ // slot/trace mutexes that we hold here.
+ atomic_store_relaxed(
+ &tctx->thr->trace_pos,
+ reinterpret_cast<uptr>(&part->events[TracePart::kSize]));
+ break;
+ }
+ parts->Remove(part);
+ TracePartFree(part);
+ }
+ CHECK_LE(parts->Size(), 1);
+ trace->local_head = parts->Front();
+ if (tctx->thr && !tctx->thr->slot) {
+ atomic_store_relaxed(&tctx->thr->trace_pos, 0);
+ tctx->thr->trace_prev_pc = 0;
+ }
+ if (trace->parts_allocated > trace->parts.Size()) {
+ ctx->trace_part_finished_excess +=
+ trace->parts_allocated - trace->parts.Size();
+ trace->parts_allocated = trace->parts.Size();
+ }
+ }
+ while (ctx->slot_queue.PopFront()) {
+ }
+ for (auto& slot : ctx->slots) {
+ slot.SetEpoch(kEpochZero);
+ slot.journal.Reset();
+ slot.thr = nullptr;
+ ctx->slot_queue.PushBack(&slot);
+ }
+
+ DPrintf("Resetting shadow...\n");
+ auto shadow_begin = ShadowBeg();
+ auto shadow_end = ShadowEnd();
+#if SANITIZER_GO
+ CHECK_NE(0, ctx->mapped_shadow_begin);
+ shadow_begin = ctx->mapped_shadow_begin;
+ shadow_end = ctx->mapped_shadow_end;
+ VPrintf(2, "shadow_begin-shadow_end: (0x%zx-0x%zx)\n",
+ shadow_begin, shadow_end);
#endif
-static ALIGNED(64) char thread_registry_placeholder[sizeof(ThreadRegistry)];
-
-static ThreadContextBase *CreateThreadContext(u32 tid) {
- // Map thread trace when context is created.
- char name[50];
- internal_snprintf(name, sizeof(name), "trace %u", tid);
- MapThreadTrace(GetThreadTrace(tid), TraceSize() * sizeof(Event), name);
- const uptr hdr = GetThreadTraceHeader(tid);
- internal_snprintf(name, sizeof(name), "trace header %u", tid);
- MapThreadTrace(hdr, sizeof(Trace), name);
- new((void*)hdr) Trace();
- // We are going to use only a small part of the trace with the default
- // value of history_size. However, the constructor writes to the whole trace.
- // Release the unused part.
- uptr hdr_end = hdr + sizeof(Trace);
- hdr_end -= sizeof(TraceHeader) * (kTraceParts - TraceParts());
- hdr_end = RoundUp(hdr_end, GetPageSizeCached());
- if (hdr_end < hdr + sizeof(Trace)) {
- ReleaseMemoryPagesToOS(hdr_end, hdr + sizeof(Trace));
- uptr unused = hdr + sizeof(Trace) - hdr_end;
- if (hdr_end != (uptr)MmapFixedNoAccess(hdr_end, unused)) {
- Report("ThreadSanitizer: failed to mprotect(%p, %p)\n",
- hdr_end, unused);
- CHECK("unable to mprotect" && 0);
+#if SANITIZER_WINDOWS
+ auto resetFailed =
+ !ZeroMmapFixedRegion(shadow_begin, shadow_end - shadow_begin);
+#else
+ auto resetFailed =
+ !MmapFixedSuperNoReserve(shadow_begin, shadow_end-shadow_begin, "shadow");
+# if !SANITIZER_GO
+ DontDumpShadow(shadow_begin, shadow_end - shadow_begin);
+# endif
+#endif
+ if (resetFailed) {
+ Printf("failed to reset shadow memory\n");
+ Die();
+ }
+ DPrintf("Resetting meta shadow...\n");
+ ctx->metamap.ResetClocks();
+ StoreShadow(&ctx->last_spurious_race, Shadow::kEmpty);
+ ctx->resetting = false;
+}
+
+// Clang does not understand locking all slots in the loop:
+// error: expecting mutex 'slot.mtx' to be held at start of each loop
+void DoReset(ThreadState* thr, uptr epoch) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
+ for (auto& slot : ctx->slots) {
+ slot.mtx.Lock();
+ if (UNLIKELY(epoch == 0))
+ epoch = ctx->global_epoch;
+ if (UNLIKELY(epoch != ctx->global_epoch)) {
+ // Epoch can't change once we've locked the first slot.
+ CHECK_EQ(slot.sid, 0);
+ slot.mtx.Unlock();
+ return;
+ }
+ }
+ DPrintf("#%d: DoReset epoch=%lu\n", thr ? thr->tid : -1, epoch);
+ DoResetImpl(epoch);
+ for (auto& slot : ctx->slots) slot.mtx.Unlock();
+}
+
+void FlushShadowMemory() { DoReset(nullptr, 0); }
+
+static TidSlot* FindSlotAndLock(ThreadState* thr)
+ SANITIZER_ACQUIRE(thr->slot->mtx) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
+ CHECK(!thr->slot);
+ TidSlot* slot = nullptr;
+ for (;;) {
+ uptr epoch;
+ {
+ Lock lock(&ctx->slot_mtx);
+ epoch = ctx->global_epoch;
+ if (slot) {
+ // This is an exhausted slot from the previous iteration.
+ if (ctx->slot_queue.Queued(slot))
+ ctx->slot_queue.Remove(slot);
+ thr->slot_locked = false;
+ slot->mtx.Unlock();
+ }
+ for (;;) {
+ slot = ctx->slot_queue.PopFront();
+ if (!slot)
+ break;
+ if (slot->epoch() != kEpochLast) {
+ ctx->slot_queue.PushBack(slot);
+ break;
+ }
+ }
+ }
+ if (!slot) {
+ DoReset(thr, epoch);
+ continue;
}
+ slot->mtx.Lock();
+ CHECK(!thr->slot_locked);
+ thr->slot_locked = true;
+ if (slot->thr) {
+ DPrintf("#%d: preempting sid=%d tid=%d\n", thr->tid, (u32)slot->sid,
+ slot->thr->tid);
+ slot->SetEpoch(slot->thr->fast_state.epoch());
+ slot->thr = nullptr;
+ }
+ if (slot->epoch() != kEpochLast)
+ return slot;
}
- void *mem = internal_alloc(MBlockThreadContex, sizeof(ThreadContext));
- return new(mem) ThreadContext(tid);
}
+void SlotAttachAndLock(ThreadState* thr) {
+ TidSlot* slot = FindSlotAndLock(thr);
+ DPrintf("#%d: SlotAttach: slot=%u\n", thr->tid, static_cast<int>(slot->sid));
+ CHECK(!slot->thr);
+ CHECK(!thr->slot);
+ slot->thr = thr;
+ thr->slot = slot;
+ Epoch epoch = EpochInc(slot->epoch());
+ CHECK(!EpochOverflow(epoch));
+ slot->SetEpoch(epoch);
+ thr->fast_state.SetSid(slot->sid);
+ thr->fast_state.SetEpoch(epoch);
+ if (thr->slot_epoch != ctx->global_epoch) {
+ thr->slot_epoch = ctx->global_epoch;
+ thr->clock.Reset();
#if !SANITIZER_GO
-static const u32 kThreadQuarantineSize = 16;
-#else
-static const u32 kThreadQuarantineSize = 64;
+ thr->last_sleep_stack_id = kInvalidStackID;
+ thr->last_sleep_clock.Reset();
+#endif
+ }
+ thr->clock.Set(slot->sid, epoch);
+ slot->journal.PushBack({thr->tid, epoch});
+}
+
+static void SlotDetachImpl(ThreadState* thr, bool exiting) {
+ TidSlot* slot = thr->slot;
+ thr->slot = nullptr;
+ if (thr != slot->thr) {
+ slot = nullptr; // we don't own the slot anymore
+ if (thr->slot_epoch != ctx->global_epoch) {
+ TracePart* part = nullptr;
+ auto* trace = &thr->tctx->trace;
+ {
+ Lock l(&trace->mtx);
+ auto* parts = &trace->parts;
+ // The trace can be completely empty in an unlikely event
+ // the thread is preempted right after it acquired the slot
+ // in ThreadStart and did not trace any events yet.
+ CHECK_LE(parts->Size(), 1);
+ part = parts->PopFront();
+ thr->tctx->trace.local_head = nullptr;
+ atomic_store_relaxed(&thr->trace_pos, 0);
+ thr->trace_prev_pc = 0;
+ }
+ if (part) {
+ Lock l(&ctx->slot_mtx);
+ TracePartFree(part);
+ }
+ }
+ return;
+ }
+ CHECK(exiting || thr->fast_state.epoch() == kEpochLast);
+ slot->SetEpoch(thr->fast_state.epoch());
+ slot->thr = nullptr;
+}
+
+void SlotDetach(ThreadState* thr) {
+ Lock lock(&thr->slot->mtx);
+ SlotDetachImpl(thr, true);
+}
+
+void SlotLock(ThreadState* thr) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
+ DCHECK(!thr->slot_locked);
+#if SANITIZER_DEBUG
+ // Check these mutexes are not locked.
+ // We can call DoReset from SlotAttachAndLock, which will lock
+ // these mutexes, but it happens only every once in a while.
+ { ThreadRegistryLock lock(&ctx->thread_registry); }
+ { Lock lock(&ctx->slot_mtx); }
#endif
+ TidSlot* slot = thr->slot;
+ slot->mtx.Lock();
+ thr->slot_locked = true;
+ if (LIKELY(thr == slot->thr && thr->fast_state.epoch() != kEpochLast))
+ return;
+ SlotDetachImpl(thr, false);
+ thr->slot_locked = false;
+ slot->mtx.Unlock();
+ SlotAttachAndLock(thr);
+}
+
+void SlotUnlock(ThreadState* thr) {
+ DCHECK(thr->slot_locked);
+ thr->slot_locked = false;
+ thr->slot->mtx.Unlock();
+}
Context::Context()
: initialized(),
report_mtx(MutexTypeReport),
nreported(),
- nmissed_expected(),
- thread_registry(new (thread_registry_placeholder) ThreadRegistry(
- CreateThreadContext, kMaxTid, kThreadQuarantineSize, kMaxTidReuse)),
+ thread_registry([](Tid tid) -> ThreadContextBase* {
+ return new (Alloc(sizeof(ThreadContext))) ThreadContext(tid);
+ }),
racy_mtx(MutexTypeRacy),
racy_stacks(),
- racy_addresses(),
fired_suppressions_mtx(MutexTypeFired),
- clock_alloc(LINKER_INITIALIZED, "clock allocator") {
+ slot_mtx(MutexTypeSlots),
+ resetting() {
fired_suppressions.reserve(8);
+ for (uptr i = 0; i < ARRAY_SIZE(slots); i++) {
+ TidSlot* slot = &slots[i];
+ slot->sid = static_cast<Sid>(i);
+ slot_queue.PushBack(slot);
+ }
+ global_epoch = 1;
}
+TidSlot::TidSlot() : mtx(MutexTypeSlot) {}
+
// The objects are allocated in TLS, so one may rely on zero-initialization.
-ThreadState::ThreadState(Context *ctx, u32 tid, int unique_id, u64 epoch,
- unsigned reuse_count, uptr stk_addr, uptr stk_size,
- uptr tls_addr, uptr tls_size)
- : fast_state(tid, epoch)
- // Do not touch these, rely on zero initialization,
- // they may be accessed before the ctor.
- // , ignore_reads_and_writes()
- // , ignore_interceptors()
- ,
- clock(tid, reuse_count)
-#if !SANITIZER_GO
- ,
- jmp_bufs()
-#endif
- ,
- tid(tid),
- unique_id(unique_id),
- stk_addr(stk_addr),
- stk_size(stk_size),
- tls_addr(tls_addr),
- tls_size(tls_size)
+ThreadState::ThreadState(Tid tid)
+ // Do not touch these, rely on zero initialization,
+ // they may be accessed before the ctor.
+ // ignore_reads_and_writes()
+ // ignore_interceptors()
+ : tid(tid) {
+ CHECK_EQ(reinterpret_cast<uptr>(this) % SANITIZER_CACHE_LINE_SIZE, 0);
#if !SANITIZER_GO
- ,
- last_sleep_clock(tid)
+ // C/C++ uses fixed size shadow stack.
+ const int kInitStackSize = kShadowStackSize;
+ shadow_stack = static_cast<uptr*>(
+ MmapNoReserveOrDie(kInitStackSize * sizeof(uptr), "shadow stack"));
+ SetShadowRegionHugePageMode(reinterpret_cast<uptr>(shadow_stack),
+ kInitStackSize * sizeof(uptr));
+#else
+ // Go uses malloc-allocated shadow stack with dynamic size.
+ const int kInitStackSize = 8;
+ shadow_stack = static_cast<uptr*>(Alloc(kInitStackSize * sizeof(uptr)));
#endif
-{
+ shadow_stack_pos = shadow_stack;
+ shadow_stack_end = shadow_stack + kInitStackSize;
}
#if !SANITIZER_GO
-static void MemoryProfiler(Context *ctx, fd_t fd, int i) {
- uptr n_threads;
- uptr n_running_threads;
- ctx->thread_registry->GetNumberOfThreads(&n_threads, &n_running_threads);
+void MemoryProfiler(u64 uptime) {
+ if (ctx->memprof_fd == kInvalidFd)
+ return;
InternalMmapVector<char> buf(4096);
- WriteMemoryProfile(buf.data(), buf.size(), n_threads, n_running_threads);
- WriteToFile(fd, buf.data(), internal_strlen(buf.data()));
+ WriteMemoryProfile(buf.data(), buf.size(), uptime);
+ WriteToFile(ctx->memprof_fd, buf.data(), internal_strlen(buf.data()));
+}
+
+static bool InitializeMemoryProfiler() {
+ ctx->memprof_fd = kInvalidFd;
+ const char *fname = flags()->profile_memory;
+ if (!fname || !fname[0])
+ return false;
+ if (internal_strcmp(fname, "stdout") == 0) {
+ ctx->memprof_fd = 1;
+ } else if (internal_strcmp(fname, "stderr") == 0) {
+ ctx->memprof_fd = 2;
+ } else {
+ InternalScopedString filename;
+ filename.append("%s.%d", fname, (int)internal_getpid());
+ ctx->memprof_fd = OpenFile(filename.data(), WrOnly);
+ if (ctx->memprof_fd == kInvalidFd) {
+ Printf("ThreadSanitizer: failed to open memory profile file '%s'\n",
+ filename.data());
+ return false;
+ }
+ }
+ MemoryProfiler(0);
+ return true;
}
static void *BackgroundThread(void *arg) {
// We don't use ScopedIgnoreInterceptors, because we want ignores to be
// enabled even when the thread function exits (e.g. during pthread thread
// shutdown code).
- cur_thread_init();
- cur_thread()->ignore_interceptors++;
+ cur_thread_init()->ignore_interceptors++;
const u64 kMs2Ns = 1000 * 1000;
+ const u64 start = NanoTime();
- fd_t mprof_fd = kInvalidFd;
- if (flags()->profile_memory && flags()->profile_memory[0]) {
- if (internal_strcmp(flags()->profile_memory, "stdout") == 0) {
- mprof_fd = 1;
- } else if (internal_strcmp(flags()->profile_memory, "stderr") == 0) {
- mprof_fd = 2;
- } else {
- InternalScopedString filename;
- filename.append("%s.%d", flags()->profile_memory, (int)internal_getpid());
- fd_t fd = OpenFile(filename.data(), WrOnly);
- if (fd == kInvalidFd) {
- Printf("ThreadSanitizer: failed to open memory profile file '%s'\n",
- filename.data());
- } else {
- mprof_fd = fd;
- }
- }
- }
-
- u64 last_flush = NanoTime();
+ u64 last_flush = start;
uptr last_rss = 0;
- for (int i = 0;
- atomic_load(&ctx->stop_background_thread, memory_order_relaxed) == 0;
- i++) {
+ while (!atomic_load_relaxed(&ctx->stop_background_thread)) {
SleepForMillis(100);
u64 now = NanoTime();
// Flush memory if requested.
if (flags()->flush_memory_ms > 0) {
if (last_flush + flags()->flush_memory_ms * kMs2Ns < now) {
- VPrintf(1, "ThreadSanitizer: periodic memory flush\n");
+ VReport(1, "ThreadSanitizer: periodic memory flush\n");
FlushShadowMemory();
- last_flush = NanoTime();
+ now = last_flush = NanoTime();
}
}
- // GetRSS can be expensive on huge programs, so don't do it every 100ms.
if (flags()->memory_limit_mb > 0) {
uptr rss = GetRSS();
uptr limit = uptr(flags()->memory_limit_mb) << 20;
- VPrintf(1, "ThreadSanitizer: memory flush check"
- " RSS=%llu LAST=%llu LIMIT=%llu\n",
+ VReport(1,
+ "ThreadSanitizer: memory flush check"
+ " RSS=%llu LAST=%llu LIMIT=%llu\n",
(u64)rss >> 20, (u64)last_rss >> 20, (u64)limit >> 20);
if (2 * rss > limit + last_rss) {
- VPrintf(1, "ThreadSanitizer: flushing memory due to RSS\n");
+ VReport(1, "ThreadSanitizer: flushing memory due to RSS\n");
FlushShadowMemory();
rss = GetRSS();
- VPrintf(1, "ThreadSanitizer: memory flushed RSS=%llu\n", (u64)rss>>20);
+ now = NanoTime();
+ VReport(1, "ThreadSanitizer: memory flushed RSS=%llu\n",
+ (u64)rss >> 20);
}
last_rss = rss;
}
- // Write memory profile if requested.
- if (mprof_fd != kInvalidFd)
- MemoryProfiler(ctx, mprof_fd, i);
+ MemoryProfiler(now - start);
// Flush symbolizer cache if requested.
if (flags()->flush_symbolizer_ms > 0) {
#endif
void DontNeedShadowFor(uptr addr, uptr size) {
- ReleaseMemoryPagesToOS(MemToShadow(addr), MemToShadow(addr + size));
+ ReleaseMemoryPagesToOS(reinterpret_cast<uptr>(MemToShadow(addr)),
+ reinterpret_cast<uptr>(MemToShadow(addr + size)));
}
#if !SANITIZER_GO
+// We call UnmapShadow before the actual munmap, at that point we don't yet
+// know if the provided address/size are sane. We can't call UnmapShadow
+// after the actual munmap becuase at that point the memory range can
+// already be reused for something else, so we can't rely on the munmap
+// return value to understand is the values are sane.
+// While calling munmap with insane values (non-canonical address, negative
+// size, etc) is an error, the kernel won't crash. We must also try to not
+// crash as the failure mode is very confusing (paging fault inside of the
+// runtime on some derived shadow address).
+static bool IsValidMmapRange(uptr addr, uptr size) {
+ if (size == 0)
+ return true;
+ if (static_cast<sptr>(size) < 0)
+ return false;
+ if (!IsAppMem(addr) || !IsAppMem(addr + size - 1))
+ return false;
+ // Check that if the start of the region belongs to one of app ranges,
+ // end of the region belongs to the same region.
+ const uptr ranges[][2] = {
+ {LoAppMemBeg(), LoAppMemEnd()},
+ {MidAppMemBeg(), MidAppMemEnd()},
+ {HiAppMemBeg(), HiAppMemEnd()},
+ };
+ for (auto range : ranges) {
+ if (addr >= range[0] && addr < range[1])
+ return addr + size <= range[1];
+ }
+ return false;
+}
+
void UnmapShadow(ThreadState *thr, uptr addr, uptr size) {
- if (size == 0) return;
+ if (size == 0 || !IsValidMmapRange(addr, size))
+ return;
DontNeedShadowFor(addr, size);
ScopedGlobalProcessor sgp;
- ctx->metamap.ResetRange(thr->proc(), addr, size);
+ SlotLocker locker(thr, true);
+ ctx->metamap.ResetRange(thr->proc(), addr, size, true);
}
#endif
void MapShadow(uptr addr, uptr size) {
+ // Ensure thead registry lock held, so as to synchronize
+ // with DoReset, which also access the mapped_shadow_* ctxt fields.
+ ThreadRegistryLock lock0(&ctx->thread_registry);
+ static bool data_mapped = false;
+
+#if !SANITIZER_GO
// Global data is not 64K aligned, but there are no adjacent mappings,
// so we can get away with unaligned mapping.
// CHECK_EQ(addr, addr & ~((64 << 10) - 1)); // windows wants 64K alignment
const uptr kPageSize = GetPageSizeCached();
uptr shadow_begin = RoundDownTo((uptr)MemToShadow(addr), kPageSize);
uptr shadow_end = RoundUpTo((uptr)MemToShadow(addr + size), kPageSize);
- if (!MmapFixedSuperNoReserve(shadow_begin, shadow_end - shadow_begin,
- "shadow"))
+ if (!MmapFixedNoReserve(shadow_begin, shadow_end - shadow_begin, "shadow"))
Die();
+#else
+ uptr shadow_begin = RoundDownTo((uptr)MemToShadow(addr), (64 << 10));
+ uptr shadow_end = RoundUpTo((uptr)MemToShadow(addr + size), (64 << 10));
+ VPrintf(2, "MapShadow for (0x%zx-0x%zx), begin/end: (0x%zx-0x%zx)\n",
+ addr, addr + size, shadow_begin, shadow_end);
+
+ if (!data_mapped) {
+ // First call maps data+bss.
+ if (!MmapFixedSuperNoReserve(shadow_begin, shadow_end - shadow_begin, "shadow"))
+ Die();
+ } else {
+ VPrintf(2, "ctx->mapped_shadow_{begin,end} = (0x%zx-0x%zx)\n",
+ ctx->mapped_shadow_begin, ctx->mapped_shadow_end);
+ // Second and subsequent calls map heap.
+ if (shadow_end <= ctx->mapped_shadow_end)
+ return;
+ if (!ctx->mapped_shadow_begin || ctx->mapped_shadow_begin > shadow_begin)
+ ctx->mapped_shadow_begin = shadow_begin;
+ if (shadow_begin < ctx->mapped_shadow_end)
+ shadow_begin = ctx->mapped_shadow_end;
+ VPrintf(2, "MapShadow begin/end = (0x%zx-0x%zx)\n",
+ shadow_begin, shadow_end);
+ if (!MmapFixedSuperNoReserve(shadow_begin, shadow_end - shadow_begin,
+ "shadow"))
+ Die();
+ ctx->mapped_shadow_end = shadow_end;
+ }
+#endif
// Meta shadow is 2:1, so tread carefully.
- static bool data_mapped = false;
static uptr mapped_meta_end = 0;
uptr meta_begin = (uptr)MemToMeta(addr);
uptr meta_end = (uptr)MemToMeta(addr + size);
"meta shadow"))
Die();
} else {
- // Mapping continous heap.
+ // Mapping continuous heap.
// Windows wants 64K alignment.
meta_begin = RoundDownTo(meta_begin, 64 << 10);
meta_end = RoundUpTo(meta_end, 64 << 10);
- if (meta_end <= mapped_meta_end)
- return;
+ CHECK_GT(meta_end, mapped_meta_end);
if (meta_begin < mapped_meta_end)
meta_begin = mapped_meta_end;
if (!MmapFixedSuperNoReserve(meta_begin, meta_end - meta_begin,
Die();
mapped_meta_end = meta_end;
}
- VPrintf(2, "mapped meta shadow for (%p-%p) at (%p-%p)\n",
- addr, addr+size, meta_begin, meta_end);
-}
-
-void MapThreadTrace(uptr addr, uptr size, const char *name) {
- DPrintf("#0: Mapping trace at %p-%p(0x%zx)\n", addr, addr + size, size);
- CHECK_GE(addr, TraceMemBeg());
- CHECK_LE(addr + size, TraceMemEnd());
- CHECK_EQ(addr, addr & ~((64 << 10) - 1)); // windows wants 64K alignment
- if (!MmapFixedSuperNoReserve(addr, size, name)) {
- Printf("FATAL: ThreadSanitizer can not mmap thread trace (%p/%p)\n",
- addr, size);
- Die();
- }
-}
-
-static void CheckShadowMapping() {
- uptr beg, end;
- for (int i = 0; GetUserRegion(i, &beg, &end); i++) {
- // Skip cases for empty regions (heap definition for architectures that
- // do not use 64-bit allocator).
- if (beg == end)
- continue;
- VPrintf(3, "checking shadow region %p-%p\n", beg, end);
- uptr prev = 0;
- for (uptr p0 = beg; p0 <= end; p0 += (end - beg) / 4) {
- for (int x = -(int)kShadowCell; x <= (int)kShadowCell; x += kShadowCell) {
- const uptr p = RoundDown(p0 + x, kShadowCell);
- if (p < beg || p >= end)
- continue;
- const uptr s = MemToShadow(p);
- const uptr m = (uptr)MemToMeta(p);
- VPrintf(3, " checking pointer %p: shadow=%p meta=%p\n", p, s, m);
- CHECK(IsAppMem(p));
- CHECK(IsShadowMem(s));
- CHECK_EQ(p, ShadowToMem(s));
- CHECK(IsMetaMem(m));
- if (prev) {
- // Ensure that shadow and meta mappings are linear within a single
- // user range. Lots of code that processes memory ranges assumes it.
- const uptr prev_s = MemToShadow(prev);
- const uptr prev_m = (uptr)MemToMeta(prev);
- CHECK_EQ(s - prev_s, (p - prev) * kShadowMultiplier);
- CHECK_EQ((m - prev_m) / kMetaShadowSize,
- (p - prev) / kMetaShadowCell);
- }
- prev = p;
- }
- }
- }
+ VPrintf(2, "mapped meta shadow for (0x%zx-0x%zx) at (0x%zx-0x%zx)\n", addr,
+ addr + size, meta_begin, meta_end);
}
#if !SANITIZER_GO
// since we are going to die soon.
ScopedIgnoreInterceptors ignore;
#if !SANITIZER_GO
- cur_thread()->ignore_sync++;
- cur_thread()->ignore_reads_and_writes++;
+ ThreadState* thr = cur_thread();
+ thr->nomalloc = false;
+ thr->ignore_sync++;
+ thr->ignore_reads_and_writes++;
+ atomic_store_relaxed(&thr->in_signal_handler, 0);
#endif
PrintCurrentStackSlow(StackTrace::GetCurrentPc());
}
+bool is_initialized;
+
void Initialize(ThreadState *thr) {
// Thread safe because done before all threads exist.
- static bool is_initialized = false;
if (is_initialized)
return;
is_initialized = true;
__tsan::InitializePlatformEarly();
#if !SANITIZER_GO
- // Re-exec ourselves if we need to set additional env or command line args.
- MaybeReexec();
-
InitializeAllocator();
ReplaceSystemMalloc();
#endif
Processor *proc = ProcCreate();
ProcWire(proc, thr);
InitializeInterceptors();
- CheckShadowMapping();
InitializePlatform();
InitializeDynamicAnnotations();
#if !SANITIZER_GO
Symbolizer::GetOrInit()->AddHooks(EnterSymbolizer, ExitSymbolizer);
#endif
- VPrintf(1, "***** Running under ThreadSanitizer v2 (pid %d) *****\n",
+ VPrintf(1, "***** Running under ThreadSanitizer v3 (pid %d) *****\n",
(int)internal_getpid());
// Initialize thread 0.
- int tid = ThreadCreate(thr, 0, 0, true);
- CHECK_EQ(tid, 0);
+ Tid tid = ThreadCreate(nullptr, 0, 0, true);
+ CHECK_EQ(tid, kMainTid);
ThreadStart(thr, tid, GetTid(), ThreadType::Regular);
#if TSAN_CONTAINS_UBSAN
__ubsan::InitAsPlugin();
#endif
- ctx->initialized = true;
#if !SANITIZER_GO
Symbolizer::LateInitialize();
+ if (InitializeMemoryProfiler() || flags()->force_background_thread)
+ MaybeSpawnBackgroundThread();
#endif
+ ctx->initialized = true;
if (flags()->stop_on_start) {
Printf("ThreadSanitizer is suspended at startup (pid %d)."
#endif
}
-
int Finalize(ThreadState *thr) {
bool failed = false;
+#if !SANITIZER_GO
if (common_flags()->print_module_map == 1)
DumpProcessMap();
+#endif
if (flags()->atexit_sleep_ms > 0 && ThreadCount(thr) > 1)
- SleepForMillis(flags()->atexit_sleep_ms);
+ internal_usleep(u64(flags()->atexit_sleep_ms) * 1000);
- // Wait for pending reports.
- ctx->report_mtx.Lock();
- { ScopedErrorReportLock l; }
- ctx->report_mtx.Unlock();
+ {
+ // Wait for pending reports.
+ ScopedErrorReportLock lock;
+ }
#if !SANITIZER_GO
if (Verbosity()) AllocatorPrintStats();
#endif
}
- if (ctx->nmissed_expected) {
- failed = true;
- Printf("ThreadSanitizer: missed %d expected races\n",
- ctx->nmissed_expected);
- }
-
if (common_flags()->print_suppressions)
PrintMatchedSuppressions();
-#if !SANITIZER_GO
- if (flags()->print_benign)
- PrintMatchedBenignRaces();
-#endif
failed = OnFinalize(failed);
}
#if !SANITIZER_GO
-void ForkBefore(ThreadState *thr, uptr pc) NO_THREAD_SAFETY_ANALYSIS {
- ctx->thread_registry->Lock();
- ctx->report_mtx.Lock();
+void ForkBefore(ThreadState* thr, uptr pc) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
+ GlobalProcessorLock();
+ // Detaching from the slot makes OnUserFree skip writing to the shadow.
+ // The slot will be locked so any attempts to use it will deadlock anyway.
+ SlotDetach(thr);
+ for (auto& slot : ctx->slots) slot.mtx.Lock();
+ ctx->thread_registry.Lock();
+ ctx->slot_mtx.Lock();
ScopedErrorReportLock::Lock();
+ AllocatorLock();
// Suppress all reports in the pthread_atfork callbacks.
// Reports will deadlock on the report_mtx.
// We could ignore sync operations as well,
thr->suppress_reports++;
// On OS X, REAL(fork) can call intercepted functions (OSSpinLockLock), and
// we'll assert in CheckNoLocks() unless we ignore interceptors.
+ // On OS X libSystem_atfork_prepare/parent/child callbacks are called
+ // after/before our callbacks and they call free.
thr->ignore_interceptors++;
+ // Disables memory write in OnUserAlloc/Free.
+ thr->ignore_reads_and_writes++;
+
+ __tsan_test_only_on_fork();
}
-void ForkParentAfter(ThreadState *thr, uptr pc) NO_THREAD_SAFETY_ANALYSIS {
+static void ForkAfter(ThreadState* thr) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
thr->suppress_reports--; // Enabled in ForkBefore.
thr->ignore_interceptors--;
+ thr->ignore_reads_and_writes--;
+ AllocatorUnlock();
ScopedErrorReportLock::Unlock();
- ctx->report_mtx.Unlock();
- ctx->thread_registry->Unlock();
+ ctx->slot_mtx.Unlock();
+ ctx->thread_registry.Unlock();
+ for (auto& slot : ctx->slots) slot.mtx.Unlock();
+ SlotAttachAndLock(thr);
+ SlotUnlock(thr);
+ GlobalProcessorUnlock();
}
-void ForkChildAfter(ThreadState *thr, uptr pc) NO_THREAD_SAFETY_ANALYSIS {
- thr->suppress_reports--; // Enabled in ForkBefore.
- thr->ignore_interceptors--;
- ScopedErrorReportLock::Unlock();
- ctx->report_mtx.Unlock();
- ctx->thread_registry->Unlock();
+void ForkParentAfter(ThreadState* thr, uptr pc) { ForkAfter(thr); }
- uptr nthread = 0;
- ctx->thread_registry->GetNumberOfThreads(0, 0, &nthread /* alive threads */);
- VPrintf(1, "ThreadSanitizer: forked new process with pid %d,"
- " parent had %d threads\n", (int)internal_getpid(), (int)nthread);
+void ForkChildAfter(ThreadState* thr, uptr pc, bool start_thread) {
+ ForkAfter(thr);
+ u32 nthread = ctx->thread_registry.OnFork(thr->tid);
+ VPrintf(1,
+ "ThreadSanitizer: forked new process with pid %d,"
+ " parent had %d threads\n",
+ (int)internal_getpid(), (int)nthread);
if (nthread == 1) {
- StartBackgroundThread();
+ if (start_thread)
+ StartBackgroundThread();
} else {
// We've just forked a multi-threaded process. We cannot reasonably function
// after that (some mutexes may be locked before fork). So just enable
// ignores for everything in the hope that we will exec soon.
ctx->after_multithreaded_fork = true;
thr->ignore_interceptors++;
+ thr->suppress_reports++;
ThreadIgnoreBegin(thr, pc);
ThreadIgnoreSyncBegin(thr, pc);
}
void GrowShadowStack(ThreadState *thr) {
const int sz = thr->shadow_stack_end - thr->shadow_stack;
const int newsz = 2 * sz;
- uptr *newstack = (uptr*)internal_alloc(MBlockShadowStack,
- newsz * sizeof(uptr));
+ auto *newstack = (uptr *)Alloc(newsz * sizeof(uptr));
internal_memcpy(newstack, thr->shadow_stack, sz * sizeof(uptr));
- internal_free(thr->shadow_stack);
+ Free(thr->shadow_stack);
thr->shadow_stack = newstack;
thr->shadow_stack_pos = newstack + sz;
thr->shadow_stack_end = newstack + newsz;
}
#endif
-u32 CurrentStackId(ThreadState *thr, uptr pc) {
+StackID CurrentStackId(ThreadState *thr, uptr pc) {
+#if !SANITIZER_GO
if (!thr->is_inited) // May happen during bootstrap.
- return 0;
+ return kInvalidStackID;
+#endif
if (pc != 0) {
#if !SANITIZER_GO
DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
thr->shadow_stack_pos[0] = pc;
thr->shadow_stack_pos++;
}
- u32 id = StackDepotPut(
+ StackID id = StackDepotPut(
StackTrace(thr->shadow_stack, thr->shadow_stack_pos - thr->shadow_stack));
if (pc != 0)
thr->shadow_stack_pos--;
return id;
}
-void TraceSwitch(ThreadState *thr) {
-#if !SANITIZER_GO
- if (ctx->after_multithreaded_fork)
- return;
-#endif
- thr->nomalloc++;
- Trace *thr_trace = ThreadTrace(thr->tid);
- Lock l(&thr_trace->mtx);
- unsigned trace = (thr->fast_state.epoch() / kTracePartSize) % TraceParts();
- TraceHeader *hdr = &thr_trace->headers[trace];
- hdr->epoch0 = thr->fast_state.epoch();
- ObtainCurrentStack(thr, 0, &hdr->stack0);
- hdr->mset0 = thr->mset;
- thr->nomalloc--;
-}
-
-Trace *ThreadTrace(int tid) {
- return (Trace*)GetThreadTraceHeader(tid);
-}
-
-uptr TraceTopPC(ThreadState *thr) {
- Event *events = (Event*)GetThreadTrace(thr->tid);
- uptr pc = events[thr->fast_state.GetTracePos()];
- return pc;
-}
-
-uptr TraceSize() {
- return (uptr)(1ull << (kTracePartSizeBits + flags()->history_size + 1));
-}
-
-uptr TraceParts() {
- return TraceSize() / kTracePartSize;
-}
-
-#if !SANITIZER_GO
-extern "C" void __tsan_trace_switch() {
- TraceSwitch(cur_thread());
-}
-
-extern "C" void __tsan_report_race() {
- ReportRace(cur_thread());
-}
-#endif
-
-ALWAYS_INLINE
-Shadow LoadShadow(u64 *p) {
- u64 raw = atomic_load((atomic_uint64_t*)p, memory_order_relaxed);
- return Shadow(raw);
-}
-
-ALWAYS_INLINE
-void StoreShadow(u64 *sp, u64 s) {
- atomic_store((atomic_uint64_t*)sp, s, memory_order_relaxed);
-}
-
-ALWAYS_INLINE
-void StoreIfNotYetStored(u64 *sp, u64 *s) {
- StoreShadow(sp, *s);
- *s = 0;
-}
-
-ALWAYS_INLINE
-void HandleRace(ThreadState *thr, u64 *shadow_mem,
- Shadow cur, Shadow old) {
- thr->racy_state[0] = cur.raw();
- thr->racy_state[1] = old.raw();
- thr->racy_shadow_addr = shadow_mem;
-#if !SANITIZER_GO
- HACKY_CALL(__tsan_report_race);
-#else
- ReportRace(thr);
-#endif
-}
-
-static inline bool HappensBefore(Shadow old, ThreadState *thr) {
- return thr->clock.get(old.TidWithIgnore()) >= old.epoch();
-}
-
-ALWAYS_INLINE
-void MemoryAccessImpl1(ThreadState *thr, uptr addr,
- int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic,
- u64 *shadow_mem, Shadow cur) {
-
- // This potentially can live in an MMX/SSE scratch register.
- // The required intrinsics are:
- // __m128i _mm_move_epi64(__m128i*);
- // _mm_storel_epi64(u64*, __m128i);
- u64 store_word = cur.raw();
- bool stored = false;
-
- // scan all the shadow values and dispatch to 4 categories:
- // same, replace, candidate and race (see comments below).
- // we consider only 3 cases regarding access sizes:
- // equal, intersect and not intersect. initially I considered
- // larger and smaller as well, it allowed to replace some
- // 'candidates' with 'same' or 'replace', but I think
- // it's just not worth it (performance- and complexity-wise).
-
- Shadow old(0);
-
- // It release mode we manually unroll the loop,
- // because empirically gcc generates better code this way.
- // However, we can't afford unrolling in debug mode, because the function
- // consumes almost 4K of stack. Gtest gives only 4K of stack to death test
- // threads, which is not enough for the unrolled loop.
-#if SANITIZER_DEBUG
- for (int idx = 0; idx < 4; idx++) {
-#include "tsan_update_shadow_word_inl.h"
- }
-#else
- int idx = 0;
-#include "tsan_update_shadow_word_inl.h"
- idx = 1;
- if (stored) {
-#include "tsan_update_shadow_word_inl.h"
- } else {
-#include "tsan_update_shadow_word_inl.h"
- }
- idx = 2;
- if (stored) {
-#include "tsan_update_shadow_word_inl.h"
- } else {
-#include "tsan_update_shadow_word_inl.h"
- }
- idx = 3;
- if (stored) {
-#include "tsan_update_shadow_word_inl.h"
- } else {
-#include "tsan_update_shadow_word_inl.h"
- }
-#endif
-
- // we did not find any races and had already stored
- // the current access info, so we are done
- if (LIKELY(stored))
- return;
- // choose a random candidate slot and replace it
- StoreShadow(shadow_mem + (cur.epoch() % kShadowCnt), store_word);
- return;
- RACE:
- HandleRace(thr, shadow_mem, cur, old);
- return;
-}
-
-void UnalignedMemoryAccess(ThreadState *thr, uptr pc, uptr addr,
- int size, bool kAccessIsWrite, bool kIsAtomic) {
- while (size) {
- int size1 = 1;
- int kAccessSizeLog = kSizeLog1;
- if (size >= 8 && (addr & ~7) == ((addr + 7) & ~7)) {
- size1 = 8;
- kAccessSizeLog = kSizeLog8;
- } else if (size >= 4 && (addr & ~7) == ((addr + 3) & ~7)) {
- size1 = 4;
- kAccessSizeLog = kSizeLog4;
- } else if (size >= 2 && (addr & ~7) == ((addr + 1) & ~7)) {
- size1 = 2;
- kAccessSizeLog = kSizeLog2;
- }
- MemoryAccess(thr, pc, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic);
- addr += size1;
- size -= size1;
- }
-}
-
-ALWAYS_INLINE
-bool ContainsSameAccessSlow(u64 *s, u64 a, u64 sync_epoch, bool is_write) {
- Shadow cur(a);
- for (uptr i = 0; i < kShadowCnt; i++) {
- Shadow old(LoadShadow(&s[i]));
- if (Shadow::Addr0AndSizeAreEqual(cur, old) &&
- old.TidWithIgnore() == cur.TidWithIgnore() &&
- old.epoch() > sync_epoch &&
- old.IsAtomic() == cur.IsAtomic() &&
- old.IsRead() <= cur.IsRead())
- return true;
+static bool TraceSkipGap(ThreadState* thr) {
+ Trace *trace = &thr->tctx->trace;
+ Event *pos = reinterpret_cast<Event *>(atomic_load_relaxed(&thr->trace_pos));
+ DCHECK_EQ(reinterpret_cast<uptr>(pos + 1) & TracePart::kAlignment, 0);
+ auto *part = trace->parts.Back();
+ DPrintf("#%d: TraceSwitchPart enter trace=%p parts=%p-%p pos=%p\n", thr->tid,
+ trace, trace->parts.Front(), part, pos);
+ if (!part)
+ return false;
+ // We can get here when we still have space in the current trace part.
+ // The fast-path check in TraceAcquire has false positives in the middle of
+ // the part. Check if we are indeed at the end of the current part or not,
+ // and fill any gaps with NopEvent's.
+ Event* end = &part->events[TracePart::kSize];
+ DCHECK_GE(pos, &part->events[0]);
+ DCHECK_LE(pos, end);
+ if (pos + 1 < end) {
+ if ((reinterpret_cast<uptr>(pos) & TracePart::kAlignment) ==
+ TracePart::kAlignment)
+ *pos++ = NopEvent;
+ *pos++ = NopEvent;
+ DCHECK_LE(pos + 2, end);
+ atomic_store_relaxed(&thr->trace_pos, reinterpret_cast<uptr>(pos));
+ return true;
}
+ // We are indeed at the end.
+ for (; pos < end; pos++) *pos = NopEvent;
return false;
}
-#if defined(__SSE3__)
-#define SHUF(v0, v1, i0, i1, i2, i3) _mm_castps_si128(_mm_shuffle_ps( \
- _mm_castsi128_ps(v0), _mm_castsi128_ps(v1), \
- (i0)*1 + (i1)*4 + (i2)*16 + (i3)*64))
-ALWAYS_INLINE
-bool ContainsSameAccessFast(u64 *s, u64 a, u64 sync_epoch, bool is_write) {
- // This is an optimized version of ContainsSameAccessSlow.
- // load current access into access[0:63]
- const m128 access = _mm_cvtsi64_si128(a);
- // duplicate high part of access in addr0:
- // addr0[0:31] = access[32:63]
- // addr0[32:63] = access[32:63]
- // addr0[64:95] = access[32:63]
- // addr0[96:127] = access[32:63]
- const m128 addr0 = SHUF(access, access, 1, 1, 1, 1);
- // load 4 shadow slots
- const m128 shadow0 = _mm_load_si128((__m128i*)s);
- const m128 shadow1 = _mm_load_si128((__m128i*)s + 1);
- // load high parts of 4 shadow slots into addr_vect:
- // addr_vect[0:31] = shadow0[32:63]
- // addr_vect[32:63] = shadow0[96:127]
- // addr_vect[64:95] = shadow1[32:63]
- // addr_vect[96:127] = shadow1[96:127]
- m128 addr_vect = SHUF(shadow0, shadow1, 1, 3, 1, 3);
- if (!is_write) {
- // set IsRead bit in addr_vect
- const m128 rw_mask1 = _mm_cvtsi64_si128(1<<15);
- const m128 rw_mask = SHUF(rw_mask1, rw_mask1, 0, 0, 0, 0);
- addr_vect = _mm_or_si128(addr_vect, rw_mask);
- }
- // addr0 == addr_vect?
- const m128 addr_res = _mm_cmpeq_epi32(addr0, addr_vect);
- // epoch1[0:63] = sync_epoch
- const m128 epoch1 = _mm_cvtsi64_si128(sync_epoch);
- // epoch[0:31] = sync_epoch[0:31]
- // epoch[32:63] = sync_epoch[0:31]
- // epoch[64:95] = sync_epoch[0:31]
- // epoch[96:127] = sync_epoch[0:31]
- const m128 epoch = SHUF(epoch1, epoch1, 0, 0, 0, 0);
- // load low parts of shadow cell epochs into epoch_vect:
- // epoch_vect[0:31] = shadow0[0:31]
- // epoch_vect[32:63] = shadow0[64:95]
- // epoch_vect[64:95] = shadow1[0:31]
- // epoch_vect[96:127] = shadow1[64:95]
- const m128 epoch_vect = SHUF(shadow0, shadow1, 0, 2, 0, 2);
- // epoch_vect >= sync_epoch?
- const m128 epoch_res = _mm_cmpgt_epi32(epoch_vect, epoch);
- // addr_res & epoch_res
- const m128 res = _mm_and_si128(addr_res, epoch_res);
- // mask[0] = res[7]
- // mask[1] = res[15]
- // ...
- // mask[15] = res[127]
- const int mask = _mm_movemask_epi8(res);
- return mask != 0;
-}
-#endif
-
-ALWAYS_INLINE
-bool ContainsSameAccess(u64 *s, u64 a, u64 sync_epoch, bool is_write) {
-#if defined(__SSE3__)
- bool res = ContainsSameAccessFast(s, a, sync_epoch, is_write);
- // NOTE: this check can fail if the shadow is concurrently mutated
- // by other threads. But it still can be useful if you modify
- // ContainsSameAccessFast and want to ensure that it's not completely broken.
- // DCHECK_EQ(res, ContainsSameAccessSlow(s, a, sync_epoch, is_write));
- return res;
-#else
- return ContainsSameAccessSlow(s, a, sync_epoch, is_write);
-#endif
-}
-
-ALWAYS_INLINE USED
-void MemoryAccess(ThreadState *thr, uptr pc, uptr addr,
- int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic) {
- u64 *shadow_mem = (u64*)MemToShadow(addr);
- DPrintf2("#%d: MemoryAccess: @%p %p size=%d"
- " is_write=%d shadow_mem=%p {%zx, %zx, %zx, %zx}\n",
- (int)thr->fast_state.tid(), (void*)pc, (void*)addr,
- (int)(1 << kAccessSizeLog), kAccessIsWrite, shadow_mem,
- (uptr)shadow_mem[0], (uptr)shadow_mem[1],
- (uptr)shadow_mem[2], (uptr)shadow_mem[3]);
-#if SANITIZER_DEBUG
- if (!IsAppMem(addr)) {
- Printf("Access to non app mem %zx\n", addr);
- DCHECK(IsAppMem(addr));
- }
- if (!IsShadowMem((uptr)shadow_mem)) {
- Printf("Bad shadow addr %p (%zx)\n", shadow_mem, addr);
- DCHECK(IsShadowMem((uptr)shadow_mem));
- }
-#endif
-
- if (!SANITIZER_GO && !kAccessIsWrite && *shadow_mem == kShadowRodata) {
- // Access to .rodata section, no races here.
- // Measurements show that it can be 10-20% of all memory accesses.
- return;
- }
-
- FastState fast_state = thr->fast_state;
- if (UNLIKELY(fast_state.GetIgnoreBit())) {
- return;
- }
-
- Shadow cur(fast_state);
- cur.SetAddr0AndSizeLog(addr & 7, kAccessSizeLog);
- cur.SetWrite(kAccessIsWrite);
- cur.SetAtomic(kIsAtomic);
-
- if (LIKELY(ContainsSameAccess(shadow_mem, cur.raw(),
- thr->fast_synch_epoch, kAccessIsWrite))) {
- return;
- }
-
- if (kCollectHistory) {
- fast_state.IncrementEpoch();
- thr->fast_state = fast_state;
- TraceAddEvent(thr, fast_state, EventTypeMop, pc);
- cur.IncrementEpoch();
- }
-
- MemoryAccessImpl1(thr, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic,
- shadow_mem, cur);
-}
-
-// Called by MemoryAccessRange in tsan_rtl_thread.cpp
-ALWAYS_INLINE USED
-void MemoryAccessImpl(ThreadState *thr, uptr addr,
- int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic,
- u64 *shadow_mem, Shadow cur) {
- if (LIKELY(ContainsSameAccess(shadow_mem, cur.raw(),
- thr->fast_synch_epoch, kAccessIsWrite))) {
- return;
- }
-
- MemoryAccessImpl1(thr, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic,
- shadow_mem, cur);
-}
-
-static void MemoryRangeSet(ThreadState *thr, uptr pc, uptr addr, uptr size,
- u64 val) {
- (void)thr;
- (void)pc;
- if (size == 0)
+NOINLINE
+void TraceSwitchPart(ThreadState* thr) {
+ if (TraceSkipGap(thr))
return;
- // FIXME: fix me.
- uptr offset = addr % kShadowCell;
- if (offset) {
- offset = kShadowCell - offset;
- if (size <= offset)
+#if !SANITIZER_GO
+ if (ctx->after_multithreaded_fork) {
+ // We just need to survive till exec.
+ TracePart* part = thr->tctx->trace.parts.Back();
+ if (part) {
+ atomic_store_relaxed(&thr->trace_pos,
+ reinterpret_cast<uptr>(&part->events[0]));
return;
- addr += offset;
- size -= offset;
- }
- DCHECK_EQ(addr % 8, 0);
- // If a user passes some insane arguments (memset(0)),
- // let it just crash as usual.
- if (!IsAppMem(addr) || !IsAppMem(addr + size - 1))
- return;
- // Don't want to touch lots of shadow memory.
- // If a program maps 10MB stack, there is no need reset the whole range.
- size = (size + (kShadowCell - 1)) & ~(kShadowCell - 1);
- // UnmapOrDie/MmapFixedNoReserve does not work on Windows.
- if (SANITIZER_WINDOWS || size < common_flags()->clear_shadow_mmap_threshold) {
- u64 *p = (u64*)MemToShadow(addr);
- CHECK(IsShadowMem((uptr)p));
- CHECK(IsShadowMem((uptr)(p + size * kShadowCnt / kShadowCell - 1)));
- // FIXME: may overwrite a part outside the region
- for (uptr i = 0; i < size / kShadowCell * kShadowCnt;) {
- p[i++] = val;
- for (uptr j = 1; j < kShadowCnt; j++)
- p[i++] = 0;
- }
- } else {
- // The region is big, reset only beginning and end.
- const uptr kPageSize = GetPageSizeCached();
- u64 *begin = (u64*)MemToShadow(addr);
- u64 *end = begin + size / kShadowCell * kShadowCnt;
- u64 *p = begin;
- // Set at least first kPageSize/2 to page boundary.
- while ((p < begin + kPageSize / kShadowSize / 2) || ((uptr)p % kPageSize)) {
- *p++ = val;
- for (uptr j = 1; j < kShadowCnt; j++)
- *p++ = 0;
- }
- // Reset middle part.
- u64 *p1 = p;
- p = RoundDown(end, kPageSize);
- if (!MmapFixedSuperNoReserve((uptr)p1, (uptr)p - (uptr)p1))
- Die();
- // Set the ending.
- while (p < end) {
- *p++ = val;
- for (uptr j = 1; j < kShadowCnt; j++)
- *p++ = 0;
}
}
+#endif
+ TraceSwitchPartImpl(thr);
}
-void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size) {
- MemoryRangeSet(thr, pc, addr, size, 0);
-}
-
-void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size) {
- // Processing more than 1k (4k of shadow) is expensive,
- // can cause excessive memory consumption (user does not necessary touch
- // the whole range) and most likely unnecessary.
- if (size > 1024)
- size = 1024;
- CHECK_EQ(thr->is_freeing, false);
- thr->is_freeing = true;
- MemoryAccessRange(thr, pc, addr, size, true);
- thr->is_freeing = false;
- if (kCollectHistory) {
- thr->fast_state.IncrementEpoch();
- TraceAddEvent(thr, thr->fast_state, EventTypeMop, pc);
+void TraceSwitchPartImpl(ThreadState* thr) {
+ SlotLocker locker(thr, true);
+ Trace* trace = &thr->tctx->trace;
+ TracePart* part = TracePartAlloc(thr);
+ part->trace = trace;
+ thr->trace_prev_pc = 0;
+ TracePart* recycle = nullptr;
+ // Keep roughly half of parts local to the thread
+ // (not queued into the recycle queue).
+ uptr local_parts = (Trace::kMinParts + flags()->history_size + 1) / 2;
+ {
+ Lock lock(&trace->mtx);
+ if (trace->parts.Empty())
+ trace->local_head = part;
+ if (trace->parts.Size() >= local_parts) {
+ recycle = trace->local_head;
+ trace->local_head = trace->parts.Next(recycle);
+ }
+ trace->parts.PushBack(part);
+ atomic_store_relaxed(&thr->trace_pos,
+ reinterpret_cast<uptr>(&part->events[0]));
}
- Shadow s(thr->fast_state);
- s.ClearIgnoreBit();
- s.MarkAsFreed();
- s.SetWrite(true);
- s.SetAddr0AndSizeLog(0, 3);
- MemoryRangeSet(thr, pc, addr, size, s.raw());
-}
-
-void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size) {
- if (kCollectHistory) {
- thr->fast_state.IncrementEpoch();
- TraceAddEvent(thr, thr->fast_state, EventTypeMop, pc);
+ // Make this part self-sufficient by restoring the current stack
+ // and mutex set in the beginning of the trace.
+ TraceTime(thr);
+ {
+ // Pathologically large stacks may not fit into the part.
+ // In these cases we log only fixed number of top frames.
+ const uptr kMaxFrames = 1000;
+ // Check that kMaxFrames won't consume the whole part.
+ static_assert(kMaxFrames < TracePart::kSize / 2, "kMaxFrames is too big");
+ uptr* pos = Max(&thr->shadow_stack[0], thr->shadow_stack_pos - kMaxFrames);
+ for (; pos < thr->shadow_stack_pos; pos++) {
+ if (TryTraceFunc(thr, *pos))
+ continue;
+ CHECK(TraceSkipGap(thr));
+ CHECK(TryTraceFunc(thr, *pos));
+ }
}
- Shadow s(thr->fast_state);
- s.ClearIgnoreBit();
- s.SetWrite(true);
- s.SetAddr0AndSizeLog(0, 3);
- MemoryRangeSet(thr, pc, addr, size, s.raw());
-}
-
-void MemoryRangeImitateWriteOrResetRange(ThreadState *thr, uptr pc, uptr addr,
- uptr size) {
- if (thr->ignore_reads_and_writes == 0)
- MemoryRangeImitateWrite(thr, pc, addr, size);
- else
- MemoryResetRange(thr, pc, addr, size);
-}
-
-ALWAYS_INLINE USED
-void FuncEntry(ThreadState *thr, uptr pc) {
- DPrintf2("#%d: FuncEntry %p\n", (int)thr->fast_state.tid(), (void*)pc);
- if (kCollectHistory) {
- thr->fast_state.IncrementEpoch();
- TraceAddEvent(thr, thr->fast_state, EventTypeFuncEnter, pc);
+ for (uptr i = 0; i < thr->mset.Size(); i++) {
+ MutexSet::Desc d = thr->mset.Get(i);
+ for (uptr i = 0; i < d.count; i++)
+ TraceMutexLock(thr, d.write ? EventType::kLock : EventType::kRLock, 0,
+ d.addr, d.stack_id);
}
-
- // Shadow stack maintenance can be replaced with
- // stack unwinding during trace switch (which presumably must be faster).
- DCHECK_GE(thr->shadow_stack_pos, thr->shadow_stack);
-#if !SANITIZER_GO
- DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
-#else
- if (thr->shadow_stack_pos == thr->shadow_stack_end)
- GrowShadowStack(thr);
-#endif
- thr->shadow_stack_pos[0] = pc;
- thr->shadow_stack_pos++;
-}
-
-ALWAYS_INLINE USED
-void FuncExit(ThreadState *thr) {
- DPrintf2("#%d: FuncExit\n", (int)thr->fast_state.tid());
- if (kCollectHistory) {
- thr->fast_state.IncrementEpoch();
- TraceAddEvent(thr, thr->fast_state, EventTypeFuncExit, 0);
+ // Callers of TraceSwitchPart expect that TraceAcquire will always succeed
+ // after the call. It's possible that TryTraceFunc/TraceMutexLock above
+ // filled the trace part exactly up to the TracePart::kAlignment gap
+ // and the next TraceAcquire won't succeed. Skip the gap to avoid that.
+ EventFunc *ev;
+ if (!TraceAcquire(thr, &ev)) {
+ CHECK(TraceSkipGap(thr));
+ CHECK(TraceAcquire(thr, &ev));
}
-
- DCHECK_GT(thr->shadow_stack_pos, thr->shadow_stack);
-#if !SANITIZER_GO
- DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
-#endif
- thr->shadow_stack_pos--;
+ {
+ Lock lock(&ctx->slot_mtx);
+ // There is a small chance that the slot may be not queued at this point.
+ // This can happen if the slot has kEpochLast epoch and another thread
+ // in FindSlotAndLock discovered that it's exhausted and removed it from
+ // the slot queue. kEpochLast can happen in 2 cases: (1) if TraceSwitchPart
+ // was called with the slot locked and epoch already at kEpochLast,
+ // or (2) if we've acquired a new slot in SlotLock in the beginning
+ // of the function and the slot was at kEpochLast - 1, so after increment
+ // in SlotAttachAndLock it become kEpochLast.
+ if (ctx->slot_queue.Queued(thr->slot)) {
+ ctx->slot_queue.Remove(thr->slot);
+ ctx->slot_queue.PushBack(thr->slot);
+ }
+ if (recycle)
+ ctx->trace_part_recycle.PushBack(recycle);
+ }
+ DPrintf("#%d: TraceSwitchPart exit parts=%p-%p pos=0x%zx\n", thr->tid,
+ trace->parts.Front(), trace->parts.Back(),
+ atomic_load_relaxed(&thr->trace_pos));
}
-void ThreadIgnoreBegin(ThreadState *thr, uptr pc, bool save_stack) {
+void ThreadIgnoreBegin(ThreadState* thr, uptr pc) {
DPrintf("#%d: ThreadIgnoreBegin\n", thr->tid);
thr->ignore_reads_and_writes++;
CHECK_GT(thr->ignore_reads_and_writes, 0);
thr->fast_state.SetIgnoreBit();
#if !SANITIZER_GO
- if (save_stack && !ctx->after_multithreaded_fork)
+ if (pc && !ctx->after_multithreaded_fork)
thr->mop_ignore_set.Add(CurrentStackId(thr, pc));
#endif
}
-void ThreadIgnoreEnd(ThreadState *thr, uptr pc) {
+void ThreadIgnoreEnd(ThreadState *thr) {
DPrintf("#%d: ThreadIgnoreEnd\n", thr->tid);
CHECK_GT(thr->ignore_reads_and_writes, 0);
thr->ignore_reads_and_writes--;
}
#endif
-void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc, bool save_stack) {
+void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc) {
DPrintf("#%d: ThreadIgnoreSyncBegin\n", thr->tid);
thr->ignore_sync++;
CHECK_GT(thr->ignore_sync, 0);
#if !SANITIZER_GO
- if (save_stack && !ctx->after_multithreaded_fork)
+ if (pc && !ctx->after_multithreaded_fork)
thr->sync_ignore_set.Add(CurrentStackId(thr, pc));
#endif
}
-void ThreadIgnoreSyncEnd(ThreadState *thr, uptr pc) {
+void ThreadIgnoreSyncEnd(ThreadState *thr) {
DPrintf("#%d: ThreadIgnoreSyncEnd\n", thr->tid);
CHECK_GT(thr->ignore_sync, 0);
thr->ignore_sync--;
#else
void build_consistency_release() {}
#endif
-
} // namespace __tsan
#if SANITIZER_CHECK_DEADLOCKS
using namespace __tsan;
MutexMeta mutex_meta[] = {
{MutexInvalid, "Invalid", {}},
- {MutexThreadRegistry, "ThreadRegistry", {}},
- {MutexTypeTrace, "Trace", {MutexLeaf}},
- {MutexTypeReport, "Report", {MutexTypeSyncVar}},
- {MutexTypeSyncVar, "SyncVar", {}},
+ {MutexThreadRegistry,
+ "ThreadRegistry",
+ {MutexTypeSlots, MutexTypeTrace, MutexTypeReport}},
+ {MutexTypeReport, "Report", {MutexTypeTrace}},
+ {MutexTypeSyncVar, "SyncVar", {MutexTypeReport, MutexTypeTrace}},
{MutexTypeAnnotations, "Annotations", {}},
- {MutexTypeAtExit, "AtExit", {MutexTypeSyncVar}},
+ {MutexTypeAtExit, "AtExit", {}},
{MutexTypeFired, "Fired", {MutexLeaf}},
{MutexTypeRacy, "Racy", {MutexLeaf}},
- {MutexTypeGlobalProc, "GlobalProc", {}},
+ {MutexTypeGlobalProc, "GlobalProc", {MutexTypeSlot, MutexTypeSlots}},
+ {MutexTypeInternalAlloc, "InternalAlloc", {MutexLeaf}},
+ {MutexTypeTrace, "Trace", {}},
+ {MutexTypeSlot,
+ "Slot",
+ {MutexMulti, MutexTypeTrace, MutexTypeSyncVar, MutexThreadRegistry,
+ MutexTypeSlots}},
+ {MutexTypeSlots, "Slots", {MutexTypeTrace, MutexTypeReport}},
{},
};
void PrintMutexPC(uptr pc) { StackTrace(&pc, 1).Print(); }
-} // namespace __sanitizer
-#endif
-#if !SANITIZER_GO
-// Must be included in this file to make sure everything is inlined.
-# include "tsan_interface_inl.h"
+} // namespace __sanitizer
#endif
#include "sanitizer_common/sanitizer_suppressions.h"
#include "sanitizer_common/sanitizer_thread_registry.h"
#include "sanitizer_common/sanitizer_vector.h"
-#include "tsan_clock.h"
#include "tsan_defs.h"
#include "tsan_flags.h"
+#include "tsan_ignoreset.h"
+#include "tsan_ilist.h"
#include "tsan_mman.h"
-#include "tsan_sync.h"
-#include "tsan_trace.h"
-#include "tsan_report.h"
-#include "tsan_platform.h"
#include "tsan_mutexset.h"
-#include "tsan_ignoreset.h"
+#include "tsan_platform.h"
+#include "tsan_report.h"
+#include "tsan_shadow.h"
#include "tsan_stack_trace.h"
+#include "tsan_sync.h"
+#include "tsan_trace.h"
+#include "tsan_vector_clock.h"
#if SANITIZER_WORDSIZE != 64
# error "ThreadSanitizer is supported only on 64-bit platforms"
#if !SANITIZER_GO
struct MapUnmapCallback;
-#if defined(__mips64) || defined(__aarch64__) || defined(__powerpc__)
+#if defined(__mips64) || defined(__aarch64__) || defined(__loongarch__) || \
+ defined(__powerpc__)
struct AP32 {
static const uptr kSpaceBeg = 0;
typedef SizeClassAllocator32<AP32> PrimaryAllocator;
#else
struct AP64 { // Allocator64 parameters. Deliberately using a short name.
+# if defined(__s390x__)
+ typedef MappingS390x Mapping;
+# else
+ typedef Mapping48AddressSpace Mapping;
+# endif
static const uptr kSpaceBeg = Mapping::kHeapMemBeg;
static const uptr kSpaceSize = Mapping::kHeapMemEnd - Mapping::kHeapMemBeg;
static const uptr kMetadataSize = 0;
Allocator *allocator();
#endif
-const u64 kShadowRodata = (u64)-1; // .rodata shadow marker
-
-// FastState (from most significant bit):
-// ignore : 1
-// tid : kTidBits
-// unused : -
-// history_size : 3
-// epoch : kClkBits
-class FastState {
- public:
- FastState(u64 tid, u64 epoch) {
- x_ = tid << kTidShift;
- x_ |= epoch;
- DCHECK_EQ(tid, this->tid());
- DCHECK_EQ(epoch, this->epoch());
- DCHECK_EQ(GetIgnoreBit(), false);
- }
-
- explicit FastState(u64 x)
- : x_(x) {
- }
-
- u64 raw() const {
- return x_;
- }
-
- u64 tid() const {
- u64 res = (x_ & ~kIgnoreBit) >> kTidShift;
- return res;
- }
-
- u64 TidWithIgnore() const {
- u64 res = x_ >> kTidShift;
- return res;
- }
-
- u64 epoch() const {
- u64 res = x_ & ((1ull << kClkBits) - 1);
- return res;
- }
-
- void IncrementEpoch() {
- u64 old_epoch = epoch();
- x_ += 1;
- DCHECK_EQ(old_epoch + 1, epoch());
- (void)old_epoch;
- }
-
- void SetIgnoreBit() { x_ |= kIgnoreBit; }
- void ClearIgnoreBit() { x_ &= ~kIgnoreBit; }
- bool GetIgnoreBit() const { return (s64)x_ < 0; }
-
- void SetHistorySize(int hs) {
- CHECK_GE(hs, 0);
- CHECK_LE(hs, 7);
- x_ = (x_ & ~(kHistoryMask << kHistoryShift)) | (u64(hs) << kHistoryShift);
- }
-
- ALWAYS_INLINE
- int GetHistorySize() const {
- return (int)((x_ >> kHistoryShift) & kHistoryMask);
- }
-
- void ClearHistorySize() {
- SetHistorySize(0);
- }
-
- ALWAYS_INLINE
- u64 GetTracePos() const {
- const int hs = GetHistorySize();
- // When hs == 0, the trace consists of 2 parts.
- const u64 mask = (1ull << (kTracePartSizeBits + hs + 1)) - 1;
- return epoch() & mask;
- }
-
- private:
- friend class Shadow;
- static const int kTidShift = 64 - kTidBits - 1;
- static const u64 kIgnoreBit = 1ull << 63;
- static const u64 kFreedBit = 1ull << 63;
- static const u64 kHistoryShift = kClkBits;
- static const u64 kHistoryMask = 7;
- u64 x_;
-};
-
-// Shadow (from most significant bit):
-// freed : 1
-// tid : kTidBits
-// is_atomic : 1
-// is_read : 1
-// size_log : 2
-// addr0 : 3
-// epoch : kClkBits
-class Shadow : public FastState {
- public:
- explicit Shadow(u64 x)
- : FastState(x) {
- }
-
- explicit Shadow(const FastState &s)
- : FastState(s.x_) {
- ClearHistorySize();
- }
-
- void SetAddr0AndSizeLog(u64 addr0, unsigned kAccessSizeLog) {
- DCHECK_EQ((x_ >> kClkBits) & 31, 0);
- DCHECK_LE(addr0, 7);
- DCHECK_LE(kAccessSizeLog, 3);
- x_ |= ((kAccessSizeLog << 3) | addr0) << kClkBits;
- DCHECK_EQ(kAccessSizeLog, size_log());
- DCHECK_EQ(addr0, this->addr0());
- }
-
- void SetWrite(unsigned kAccessIsWrite) {
- DCHECK_EQ(x_ & kReadBit, 0);
- if (!kAccessIsWrite)
- x_ |= kReadBit;
- DCHECK_EQ(kAccessIsWrite, IsWrite());
- }
-
- void SetAtomic(bool kIsAtomic) {
- DCHECK(!IsAtomic());
- if (kIsAtomic)
- x_ |= kAtomicBit;
- DCHECK_EQ(IsAtomic(), kIsAtomic);
- }
-
- bool IsAtomic() const {
- return x_ & kAtomicBit;
- }
-
- bool IsZero() const {
- return x_ == 0;
- }
-
- static inline bool TidsAreEqual(const Shadow s1, const Shadow s2) {
- u64 shifted_xor = (s1.x_ ^ s2.x_) >> kTidShift;
- DCHECK_EQ(shifted_xor == 0, s1.TidWithIgnore() == s2.TidWithIgnore());
- return shifted_xor == 0;
- }
-
- static ALWAYS_INLINE
- bool Addr0AndSizeAreEqual(const Shadow s1, const Shadow s2) {
- u64 masked_xor = ((s1.x_ ^ s2.x_) >> kClkBits) & 31;
- return masked_xor == 0;
- }
-
- static ALWAYS_INLINE bool TwoRangesIntersect(Shadow s1, Shadow s2,
- unsigned kS2AccessSize) {
- bool res = false;
- u64 diff = s1.addr0() - s2.addr0();
- if ((s64)diff < 0) { // s1.addr0 < s2.addr0
- // if (s1.addr0() + size1) > s2.addr0()) return true;
- if (s1.size() > -diff)
- res = true;
- } else {
- // if (s2.addr0() + kS2AccessSize > s1.addr0()) return true;
- if (kS2AccessSize > diff)
- res = true;
- }
- DCHECK_EQ(res, TwoRangesIntersectSlow(s1, s2));
- DCHECK_EQ(res, TwoRangesIntersectSlow(s2, s1));
- return res;
- }
-
- u64 ALWAYS_INLINE addr0() const { return (x_ >> kClkBits) & 7; }
- u64 ALWAYS_INLINE size() const { return 1ull << size_log(); }
- bool ALWAYS_INLINE IsWrite() const { return !IsRead(); }
- bool ALWAYS_INLINE IsRead() const { return x_ & kReadBit; }
-
- // The idea behind the freed bit is as follows.
- // When the memory is freed (or otherwise unaccessible) we write to the shadow
- // values with tid/epoch related to the free and the freed bit set.
- // During memory accesses processing the freed bit is considered
- // as msb of tid. So any access races with shadow with freed bit set
- // (it is as if write from a thread with which we never synchronized before).
- // This allows us to detect accesses to freed memory w/o additional
- // overheads in memory access processing and at the same time restore
- // tid/epoch of free.
- void MarkAsFreed() {
- x_ |= kFreedBit;
- }
-
- bool IsFreed() const {
- return x_ & kFreedBit;
- }
-
- bool GetFreedAndReset() {
- bool res = x_ & kFreedBit;
- x_ &= ~kFreedBit;
- return res;
- }
-
- bool ALWAYS_INLINE IsBothReadsOrAtomic(bool kIsWrite, bool kIsAtomic) const {
- bool v = x_ & ((u64(kIsWrite ^ 1) << kReadShift)
- | (u64(kIsAtomic) << kAtomicShift));
- DCHECK_EQ(v, (!IsWrite() && !kIsWrite) || (IsAtomic() && kIsAtomic));
- return v;
- }
-
- bool ALWAYS_INLINE IsRWNotWeaker(bool kIsWrite, bool kIsAtomic) const {
- bool v = ((x_ >> kReadShift) & 3)
- <= u64((kIsWrite ^ 1) | (kIsAtomic << 1));
- DCHECK_EQ(v, (IsAtomic() < kIsAtomic) ||
- (IsAtomic() == kIsAtomic && !IsWrite() <= !kIsWrite));
- return v;
- }
-
- bool ALWAYS_INLINE IsRWWeakerOrEqual(bool kIsWrite, bool kIsAtomic) const {
- bool v = ((x_ >> kReadShift) & 3)
- >= u64((kIsWrite ^ 1) | (kIsAtomic << 1));
- DCHECK_EQ(v, (IsAtomic() > kIsAtomic) ||
- (IsAtomic() == kIsAtomic && !IsWrite() >= !kIsWrite));
- return v;
- }
-
- private:
- static const u64 kReadShift = 5 + kClkBits;
- static const u64 kReadBit = 1ull << kReadShift;
- static const u64 kAtomicShift = 6 + kClkBits;
- static const u64 kAtomicBit = 1ull << kAtomicShift;
-
- u64 size_log() const { return (x_ >> (3 + kClkBits)) & 3; }
-
- static bool TwoRangesIntersectSlow(const Shadow s1, const Shadow s2) {
- if (s1.addr0() == s2.addr0()) return true;
- if (s1.addr0() < s2.addr0() && s1.addr0() + s1.size() > s2.addr0())
- return true;
- if (s2.addr0() < s1.addr0() && s2.addr0() + s2.size() > s1.addr0())
- return true;
- return false;
- }
-};
-
struct ThreadSignalContext;
struct JmpBuf {
#endif
DenseSlabAllocCache block_cache;
DenseSlabAllocCache sync_cache;
- DenseSlabAllocCache clock_cache;
DDPhysicalThread *dd_pt;
};
};
#endif
+struct TidEpoch {
+ Tid tid;
+ Epoch epoch;
+};
+
+struct TidSlot {
+ Mutex mtx;
+ Sid sid;
+ atomic_uint32_t raw_epoch;
+ ThreadState *thr;
+ Vector<TidEpoch> journal;
+ INode node;
+
+ Epoch epoch() const {
+ return static_cast<Epoch>(atomic_load(&raw_epoch, memory_order_relaxed));
+ }
+
+ void SetEpoch(Epoch v) {
+ atomic_store(&raw_epoch, static_cast<u32>(v), memory_order_relaxed);
+ }
+
+ TidSlot();
+} ALIGNED(SANITIZER_CACHE_LINE_SIZE);
+
// This struct is stored in TLS.
struct ThreadState {
FastState fast_state;
- // Synch epoch represents the threads's epoch before the last synchronization
- // action. It allows to reduce number of shadow state updates.
- // For example, fast_synch_epoch=100, last write to addr X was at epoch=150,
- // if we are processing write to X from the same thread at epoch=200,
- // we do nothing, because both writes happen in the same 'synch epoch'.
- // That is, if another memory access does not race with the former write,
- // it does not race with the latter as well.
- // QUESTION: can we can squeeze this into ThreadState::Fast?
- // E.g. ThreadState::Fast is a 44-bit, 32 are taken by synch_epoch and 12 are
- // taken by epoch between synchs.
- // This way we can save one load from tls.
- u64 fast_synch_epoch;
+ int ignore_sync;
+#if !SANITIZER_GO
+ int ignore_interceptors;
+#endif
+ uptr *shadow_stack_pos;
+
+ // Current position in tctx->trace.Back()->events (Event*).
+ atomic_uintptr_t trace_pos;
+ // PC of the last memory access, used to compute PC deltas in the trace.
+ uptr trace_prev_pc;
+
// Technically `current` should be a separate THREADLOCAL variable;
// but it is placed here in order to share cache line with previous fields.
ThreadState* current;
+
+ atomic_sint32_t pending_signals;
+
+ VectorClock clock;
+
// This is a slow path flag. On fast path, fast_state.GetIgnoreBit() is read.
// We do not distinguish beteween ignoring reads and writes
// for better performance.
int ignore_reads_and_writes;
- int ignore_sync;
int suppress_reports;
// Go does not support ignores.
#if !SANITIZER_GO
IgnoreSet mop_ignore_set;
IgnoreSet sync_ignore_set;
#endif
- // C/C++ uses fixed size shadow stack embed into Trace.
- // Go uses malloc-allocated shadow stack with dynamic size.
uptr *shadow_stack;
uptr *shadow_stack_end;
- uptr *shadow_stack_pos;
- u64 *racy_shadow_addr;
- u64 racy_state[2];
- MutexSet mset;
- ThreadClock clock;
#if !SANITIZER_GO
Vector<JmpBuf> jmp_bufs;
- int ignore_interceptors;
-#endif
- const u32 tid;
- const int unique_id;
- bool in_symbolizer;
+ int in_symbolizer;
+ atomic_uintptr_t in_blocking_func;
bool in_ignored_lib;
bool is_inited;
+#endif
+ MutexSet mset;
bool is_dead;
- bool is_freeing;
- bool is_vptr_access;
- const uptr stk_addr;
- const uptr stk_size;
- const uptr tls_addr;
- const uptr tls_size;
+ const Tid tid;
+ uptr stk_addr;
+ uptr stk_size;
+ uptr tls_addr;
+ uptr tls_size;
ThreadContext *tctx;
DDLogicalThread *dd_lt;
+ TidSlot *slot;
+ uptr slot_epoch;
+ bool slot_locked;
+
// Current wired Processor, or nullptr. Required to handle any events.
Processor *proc1;
#if !SANITIZER_GO
#endif
atomic_uintptr_t in_signal_handler;
- ThreadSignalContext *signal_ctx;
+ atomic_uintptr_t signal_ctx;
#if !SANITIZER_GO
- u32 last_sleep_stack_id;
- ThreadClock last_sleep_clock;
+ StackID last_sleep_stack_id;
+ VectorClock last_sleep_clock;
#endif
// Set in regions of runtime that must be signal-safe and fork-safe.
const ReportDesc *current_report;
- explicit ThreadState(Context *ctx, u32 tid, int unique_id, u64 epoch,
- unsigned reuse_count, uptr stk_addr, uptr stk_size,
- uptr tls_addr, uptr tls_size);
-};
+ explicit ThreadState(Tid tid);
+} ALIGNED(SANITIZER_CACHE_LINE_SIZE);
#if !SANITIZER_GO
-#if SANITIZER_MAC || SANITIZER_ANDROID
+#if SANITIZER_APPLE || SANITIZER_ANDROID
ThreadState *cur_thread();
void set_cur_thread(ThreadState *thr);
void cur_thread_finalize();
-inline void cur_thread_init() { }
-#else
+inline ThreadState *cur_thread_init() { return cur_thread(); }
+# else
__attribute__((tls_model("initial-exec")))
extern THREADLOCAL char cur_thread_placeholder[];
inline ThreadState *cur_thread() {
return reinterpret_cast<ThreadState *>(cur_thread_placeholder)->current;
}
-inline void cur_thread_init() {
+inline ThreadState *cur_thread_init() {
ThreadState *thr = reinterpret_cast<ThreadState *>(cur_thread_placeholder);
if (UNLIKELY(!thr->current))
thr->current = thr;
+ return thr->current;
}
inline void set_cur_thread(ThreadState *thr) {
reinterpret_cast<ThreadState *>(cur_thread_placeholder)->current = thr;
}
inline void cur_thread_finalize() { }
-#endif // SANITIZER_MAC || SANITIZER_ANDROID
+# endif // SANITIZER_APPLE || SANITIZER_ANDROID
#endif // SANITIZER_GO
class ThreadContext final : public ThreadContextBase {
public:
- explicit ThreadContext(int tid);
+ explicit ThreadContext(Tid tid);
~ThreadContext();
ThreadState *thr;
- u32 creation_stack_id;
- SyncClock sync;
- // Epoch at which the thread had started.
- // If we see an event from the thread stamped by an older epoch,
- // the event is from a dead thread that shared tid with this thread.
- u64 epoch0;
- u64 epoch1;
+ StackID creation_stack_id;
+ VectorClock *sync;
+ uptr sync_epoch;
+ Trace trace;
// Override superclass callbacks.
void OnDead() override;
struct RacyStacks {
MD5Hash hash[2];
- bool operator==(const RacyStacks &other) const {
- if (hash[0] == other.hash[0] && hash[1] == other.hash[1])
- return true;
- if (hash[0] == other.hash[1] && hash[1] == other.hash[0])
- return true;
- return false;
- }
+ bool operator==(const RacyStacks &other) const;
};
struct RacyAddress {
Mutex report_mtx;
int nreported;
- int nmissed_expected;
atomic_uint64_t last_symbolize_time_ns;
void *background_thread;
atomic_uint32_t stop_background_thread;
- ThreadRegistry *thread_registry;
+ ThreadRegistry thread_registry;
+
+ // This is used to prevent a very unlikely but very pathological behavior.
+ // Since memory access handling is not synchronized with DoReset,
+ // a thread running concurrently with DoReset can leave a bogus shadow value
+ // that will be later falsely detected as a race. For such false races
+ // RestoreStack will return false and we will not report it.
+ // However, consider that a thread leaves a whole lot of such bogus values
+ // and these values are later read by a whole lot of threads.
+ // This will cause massive amounts of ReportRace calls and lots of
+ // serialization. In very pathological cases the resulting slowdown
+ // can be >100x. This is very unlikely, but it was presumably observed
+ // in practice: https://github.com/google/sanitizers/issues/1552
+ // If this happens, previous access sid+epoch will be the same for all of
+ // these false races b/c if the thread will try to increment epoch, it will
+ // notice that DoReset has happened and will stop producing bogus shadow
+ // values. So, last_spurious_race is used to remember the last sid+epoch
+ // for which RestoreStack returned false. Then it is used to filter out
+ // races with the same sid+epoch very early and quickly.
+ // It is of course possible that multiple threads left multiple bogus shadow
+ // values and all of them are read by lots of threads at the same time.
+ // In such case last_spurious_race will only be able to deduplicate a few
+ // races from one thread, then few from another and so on. An alternative
+ // would be to hold an array of such sid+epoch, but we consider such scenario
+ // as even less likely.
+ // Note: this can lead to some rare false negatives as well:
+ // 1. When a legit access with the same sid+epoch participates in a race
+ // as the "previous" memory access, it will be wrongly filtered out.
+ // 2. When RestoreStack returns false for a legit memory access because it
+ // was already evicted from the thread trace, we will still remember it in
+ // last_spurious_race. Then if there is another racing memory access from
+ // the same thread that happened in the same epoch, but was stored in the
+ // next thread trace part (which is still preserved in the thread trace),
+ // we will also wrongly filter it out while RestoreStack would actually
+ // succeed for that second memory access.
+ RawShadow last_spurious_race;
Mutex racy_mtx;
Vector<RacyStacks> racy_stacks;
- Vector<RacyAddress> racy_addresses;
// Number of fired suppressions may be large enough.
Mutex fired_suppressions_mtx;
InternalMmapVector<FiredSuppression> fired_suppressions;
DDetector *dd;
- ClockAlloc clock_alloc;
-
Flags flags;
-
- u64 int_alloc_cnt[MBlockTypeCount];
- u64 int_alloc_siz[MBlockTypeCount];
+ fd_t memprof_fd;
+
+ // The last slot index (kFreeSid) is used to denote freed memory.
+ TidSlot slots[kThreadSlotCount - 1];
+
+ // Protects global_epoch, slot_queue, trace_part_recycle.
+ Mutex slot_mtx;
+ uptr global_epoch; // guarded by slot_mtx and by all slot mutexes
+ bool resetting; // global reset is in progress
+ IList<TidSlot, &TidSlot::node> slot_queue SANITIZER_GUARDED_BY(slot_mtx);
+ IList<TraceHeader, &TraceHeader::global, TracePart> trace_part_recycle
+ SANITIZER_GUARDED_BY(slot_mtx);
+ uptr trace_part_total_allocated SANITIZER_GUARDED_BY(slot_mtx);
+ uptr trace_part_recycle_finished SANITIZER_GUARDED_BY(slot_mtx);
+ uptr trace_part_finished_excess SANITIZER_GUARDED_BY(slot_mtx);
+#if SANITIZER_GO
+ uptr mapped_shadow_begin;
+ uptr mapped_shadow_end;
+#endif
};
extern Context *ctx; // The one and the only global runtime context.
class ScopedReportBase {
public:
- void AddMemoryAccess(uptr addr, uptr external_tag, Shadow s, StackTrace stack,
- const MutexSet *mset);
+ void AddMemoryAccess(uptr addr, uptr external_tag, Shadow s, Tid tid,
+ StackTrace stack, const MutexSet *mset);
void AddStack(StackTrace stack, bool suppressable = false);
void AddThread(const ThreadContext *tctx, bool suppressable = false);
- void AddThread(int unique_tid, bool suppressable = false);
- void AddUniqueTid(int unique_tid);
- void AddMutex(const SyncVar *s);
- u64 AddMutex(u64 id);
+ void AddThread(Tid tid, bool suppressable = false);
+ void AddUniqueTid(Tid unique_tid);
+ int AddMutex(uptr addr, StackID creation_stack_id);
void AddLocation(uptr addr, uptr size);
- void AddSleep(u32 stack_id);
+ void AddSleep(StackID stack_id);
void SetCount(int count);
+ void SetSigNum(int sig);
const ReportDesc *GetReport() const;
// at best it will cause deadlocks on internal mutexes.
ScopedIgnoreInterceptors ignore_interceptors_;
- void AddDeadMutex(u64 id);
-
ScopedReportBase(const ScopedReportBase &) = delete;
void operator=(const ScopedReportBase &) = delete;
};
bool ShouldReport(ThreadState *thr, ReportType typ);
ThreadContext *IsThreadStackOrTls(uptr addr, bool *is_stack);
-void RestoreStack(int tid, const u64 epoch, VarSizeStackTrace *stk,
- MutexSet *mset, uptr *tag = nullptr);
// The stack could look like:
// <start> | <main> | <foo> | tag | <bar>
void DontNeedShadowFor(uptr addr, uptr size);
void UnmapShadow(ThreadState *thr, uptr addr, uptr size);
void InitializeShadowMemory();
+void DontDumpShadow(uptr addr, uptr size);
void InitializeInterceptors();
void InitializeLibIgnore();
void InitializeDynamicAnnotations();
void ForkBefore(ThreadState *thr, uptr pc);
void ForkParentAfter(ThreadState *thr, uptr pc);
-void ForkChildAfter(ThreadState *thr, uptr pc);
+void ForkChildAfter(ThreadState *thr, uptr pc, bool start_thread);
-void ReportRace(ThreadState *thr);
+void ReportRace(ThreadState *thr, RawShadow *shadow_mem, Shadow cur, Shadow old,
+ AccessType typ);
bool OutputReport(ThreadState *thr, const ScopedReport &srep);
bool IsFiredSuppression(Context *ctx, ReportType type, StackTrace trace);
bool IsExpectedReport(uptr addr, uptr size);
-void PrintMatchedBenignRaces();
#if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 1
# define DPrintf Printf
# define DPrintf2(...)
#endif
-u32 CurrentStackId(ThreadState *thr, uptr pc);
-ReportStack *SymbolizeStackId(u32 stack_id);
+StackID CurrentStackId(ThreadState *thr, uptr pc);
+ReportStack *SymbolizeStackId(StackID stack_id);
void PrintCurrentStack(ThreadState *thr, uptr pc);
void PrintCurrentStackSlow(uptr pc); // uses libunwind
+MBlock *JavaHeapBlock(uptr addr, uptr *start);
void Initialize(ThreadState *thr);
void MaybeSpawnBackgroundThread();
void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write);
void OnUserFree(ThreadState *thr, uptr pc, uptr p, bool write);
-void MemoryAccess(ThreadState *thr, uptr pc, uptr addr,
- int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic);
-void MemoryAccessImpl(ThreadState *thr, uptr addr,
- int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic,
- u64 *shadow_mem, Shadow cur);
-void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr,
- uptr size, bool is_write);
-void MemoryAccessRangeStep(ThreadState *thr, uptr pc, uptr addr,
- uptr size, uptr step, bool is_write);
-void UnalignedMemoryAccess(ThreadState *thr, uptr pc, uptr addr,
- int size, bool kAccessIsWrite, bool kIsAtomic);
-
-const int kSizeLog1 = 0;
-const int kSizeLog2 = 1;
-const int kSizeLog4 = 2;
-const int kSizeLog8 = 3;
-
-void ALWAYS_INLINE MemoryRead(ThreadState *thr, uptr pc,
- uptr addr, int kAccessSizeLog) {
- MemoryAccess(thr, pc, addr, kAccessSizeLog, false, false);
-}
-
-void ALWAYS_INLINE MemoryWrite(ThreadState *thr, uptr pc,
- uptr addr, int kAccessSizeLog) {
- MemoryAccess(thr, pc, addr, kAccessSizeLog, true, false);
-}
-
-void ALWAYS_INLINE MemoryReadAtomic(ThreadState *thr, uptr pc,
- uptr addr, int kAccessSizeLog) {
- MemoryAccess(thr, pc, addr, kAccessSizeLog, false, true);
-}
-
-void ALWAYS_INLINE MemoryWriteAtomic(ThreadState *thr, uptr pc,
- uptr addr, int kAccessSizeLog) {
- MemoryAccess(thr, pc, addr, kAccessSizeLog, true, true);
+void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size,
+ AccessType typ);
+void UnalignedMemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size,
+ AccessType typ);
+// This creates 2 non-inlined specialized versions of MemoryAccessRange.
+template <bool is_read>
+void MemoryAccessRangeT(ThreadState *thr, uptr pc, uptr addr, uptr size);
+
+ALWAYS_INLINE
+void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size,
+ bool is_write) {
+ if (size == 0)
+ return;
+ if (is_write)
+ MemoryAccessRangeT<false>(thr, pc, addr, size);
+ else
+ MemoryAccessRangeT<true>(thr, pc, addr, size);
}
-void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size);
+void ShadowSet(RawShadow *p, RawShadow *end, RawShadow v);
void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size);
+void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size);
void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size);
void MemoryRangeImitateWriteOrResetRange(ThreadState *thr, uptr pc, uptr addr,
uptr size);
-void ThreadIgnoreBegin(ThreadState *thr, uptr pc, bool save_stack = true);
-void ThreadIgnoreEnd(ThreadState *thr, uptr pc);
-void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc, bool save_stack = true);
-void ThreadIgnoreSyncEnd(ThreadState *thr, uptr pc);
-
-void FuncEntry(ThreadState *thr, uptr pc);
-void FuncExit(ThreadState *thr);
+void ThreadIgnoreBegin(ThreadState *thr, uptr pc);
+void ThreadIgnoreEnd(ThreadState *thr);
+void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc);
+void ThreadIgnoreSyncEnd(ThreadState *thr);
-int ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached);
-void ThreadStart(ThreadState *thr, int tid, tid_t os_id,
+Tid ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached);
+void ThreadStart(ThreadState *thr, Tid tid, tid_t os_id,
ThreadType thread_type);
void ThreadFinish(ThreadState *thr);
-int ThreadConsumeTid(ThreadState *thr, uptr pc, uptr uid);
-void ThreadJoin(ThreadState *thr, uptr pc, int tid);
-void ThreadDetach(ThreadState *thr, uptr pc, int tid);
+Tid ThreadConsumeTid(ThreadState *thr, uptr pc, uptr uid);
+void ThreadJoin(ThreadState *thr, uptr pc, Tid tid);
+void ThreadDetach(ThreadState *thr, uptr pc, Tid tid);
void ThreadFinalize(ThreadState *thr);
void ThreadSetName(ThreadState *thr, const char *name);
int ThreadCount(ThreadState *thr);
-void ProcessPendingSignals(ThreadState *thr);
-void ThreadNotJoined(ThreadState *thr, uptr pc, int tid, uptr uid);
+void ProcessPendingSignalsImpl(ThreadState *thr);
+void ThreadNotJoined(ThreadState *thr, uptr pc, Tid tid, uptr uid);
Processor *ProcCreate();
void ProcDestroy(Processor *proc);
// handle Go finalizers. Namely, finalizer goroutine executes AcquireGlobal
// right before executing finalizers. This provides a coarse, but simple
// approximation of the actual required synchronization.
-void AcquireGlobal(ThreadState *thr, uptr pc);
+void AcquireGlobal(ThreadState *thr);
void Release(ThreadState *thr, uptr pc, uptr addr);
void ReleaseStoreAcquire(ThreadState *thr, uptr pc, uptr addr);
void ReleaseStore(ThreadState *thr, uptr pc, uptr addr);
void AfterSleep(ThreadState *thr, uptr pc);
-void AcquireImpl(ThreadState *thr, uptr pc, SyncClock *c);
-void ReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c);
-void ReleaseStoreAcquireImpl(ThreadState *thr, uptr pc, SyncClock *c);
-void ReleaseStoreImpl(ThreadState *thr, uptr pc, SyncClock *c);
-void AcquireReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c);
-
-// The hacky call uses custom calling convention and an assembly thunk.
-// It is considerably faster that a normal call for the caller
-// if it is not executed (it is intended for slow paths from hot functions).
-// The trick is that the call preserves all registers and the compiler
-// does not treat it as a call.
-// If it does not work for you, use normal call.
-#if !SANITIZER_DEBUG && defined(__x86_64__) && !SANITIZER_MAC
-// The caller may not create the stack frame for itself at all,
-// so we create a reserve stack frame for it (1024b must be enough).
-#define HACKY_CALL(f) \
- __asm__ __volatile__("sub $1024, %%rsp;" \
- CFI_INL_ADJUST_CFA_OFFSET(1024) \
- ".hidden " #f "_thunk;" \
- "call " #f "_thunk;" \
- "add $1024, %%rsp;" \
- CFI_INL_ADJUST_CFA_OFFSET(-1024) \
- ::: "memory", "cc");
-#else
-#define HACKY_CALL(f) f()
-#endif
-
-void TraceSwitch(ThreadState *thr);
-uptr TraceTopPC(ThreadState *thr);
-uptr TraceSize();
-uptr TraceParts();
-Trace *ThreadTrace(int tid);
-
-extern "C" void __tsan_trace_switch();
-void ALWAYS_INLINE TraceAddEvent(ThreadState *thr, FastState fs,
- EventType typ, u64 addr) {
- if (!kCollectHistory)
- return;
- DCHECK_GE((int)typ, 0);
- DCHECK_LE((int)typ, 7);
- DCHECK_EQ(GetLsb(addr, kEventPCBits), addr);
- u64 pos = fs.GetTracePos();
- if (UNLIKELY((pos % kTracePartSize) == 0)) {
-#if !SANITIZER_GO
- HACKY_CALL(__tsan_trace_switch);
-#else
- TraceSwitch(thr);
-#endif
- }
- Event *trace = (Event*)GetThreadTrace(fs.tid());
- Event *evp = &trace[pos];
- Event ev = (u64)addr | ((u64)typ << kEventPCBits);
- *evp = ev;
-}
+void IncrementEpoch(ThreadState *thr);
#if !SANITIZER_GO
uptr ALWAYS_INLINE HeapEnd() {
}
#endif
+void SlotAttachAndLock(ThreadState *thr) SANITIZER_ACQUIRE(thr->slot->mtx);
+void SlotDetach(ThreadState *thr);
+void SlotLock(ThreadState *thr) SANITIZER_ACQUIRE(thr->slot->mtx);
+void SlotUnlock(ThreadState *thr) SANITIZER_RELEASE(thr->slot->mtx);
+void DoReset(ThreadState *thr, uptr epoch);
+void FlushShadowMemory();
+
ThreadState *FiberCreate(ThreadState *thr, uptr pc, unsigned flags);
void FiberDestroy(ThreadState *thr, uptr pc, ThreadState *fiber);
void FiberSwitch(ThreadState *thr, uptr pc, ThreadState *fiber, unsigned flags);
FiberSwitchFlagNoSync = 1 << 0, // __tsan_switch_to_fiber_no_sync
};
+class SlotLocker {
+ public:
+ ALWAYS_INLINE
+ SlotLocker(ThreadState *thr, bool recursive = false)
+ : thr_(thr), locked_(recursive ? thr->slot_locked : false) {
+#if !SANITIZER_GO
+ // We are in trouble if we are here with in_blocking_func set.
+ // If in_blocking_func is set, all signals will be delivered synchronously,
+ // which means we can't lock slots since the signal handler will try
+ // to lock it recursively and deadlock.
+ DCHECK(!atomic_load(&thr->in_blocking_func, memory_order_relaxed));
+#endif
+ if (!locked_)
+ SlotLock(thr_);
+ }
+
+ ALWAYS_INLINE
+ ~SlotLocker() {
+ if (!locked_)
+ SlotUnlock(thr_);
+ }
+
+ private:
+ ThreadState *thr_;
+ bool locked_;
+};
+
+class SlotUnlocker {
+ public:
+ SlotUnlocker(ThreadState *thr) : thr_(thr), locked_(thr->slot_locked) {
+ if (locked_)
+ SlotUnlock(thr_);
+ }
+
+ ~SlotUnlocker() {
+ if (locked_)
+ SlotLock(thr_);
+ }
+
+ private:
+ ThreadState *thr_;
+ bool locked_;
+};
+
+ALWAYS_INLINE void ProcessPendingSignals(ThreadState *thr) {
+ if (UNLIKELY(atomic_load_relaxed(&thr->pending_signals)))
+ ProcessPendingSignalsImpl(thr);
+}
+
+extern bool is_initialized;
+
+ALWAYS_INLINE
+void LazyInitialize(ThreadState *thr) {
+ // If we can use .preinit_array, assume that __tsan_init
+ // called from .preinit_array initializes runtime before
+ // any instrumented code except when tsan is used as a
+ // shared library.
+#if (!SANITIZER_CAN_USE_PREINIT_ARRAY || defined(SANITIZER_SHARED))
+ if (UNLIKELY(!is_initialized))
+ Initialize(thr);
+#endif
+}
+
+void TraceResetForTesting();
+void TraceSwitchPart(ThreadState *thr);
+void TraceSwitchPartImpl(ThreadState *thr);
+bool RestoreStack(EventType type, Sid sid, Epoch epoch, uptr addr, uptr size,
+ AccessType typ, Tid *ptid, VarSizeStackTrace *pstk,
+ MutexSet *pmset, uptr *ptag);
+
+template <typename EventT>
+ALWAYS_INLINE WARN_UNUSED_RESULT bool TraceAcquire(ThreadState *thr,
+ EventT **ev) {
+ // TraceSwitchPart accesses shadow_stack, but it's called infrequently,
+ // so we check it here proactively.
+ DCHECK(thr->shadow_stack);
+ Event *pos = reinterpret_cast<Event *>(atomic_load_relaxed(&thr->trace_pos));
+#if SANITIZER_DEBUG
+ // TraceSwitch acquires these mutexes,
+ // so we lock them here to detect deadlocks more reliably.
+ { Lock lock(&ctx->slot_mtx); }
+ { Lock lock(&thr->tctx->trace.mtx); }
+ TracePart *current = thr->tctx->trace.parts.Back();
+ if (current) {
+ DCHECK_GE(pos, ¤t->events[0]);
+ DCHECK_LE(pos, ¤t->events[TracePart::kSize]);
+ } else {
+ DCHECK_EQ(pos, nullptr);
+ }
+#endif
+ // TracePart is allocated with mmap and is at least 4K aligned.
+ // So the following check is a faster way to check for part end.
+ // It may have false positives in the middle of the trace,
+ // they are filtered out in TraceSwitch.
+ if (UNLIKELY(((uptr)(pos + 1) & TracePart::kAlignment) == 0))
+ return false;
+ *ev = reinterpret_cast<EventT *>(pos);
+ return true;
+}
+
+template <typename EventT>
+ALWAYS_INLINE void TraceRelease(ThreadState *thr, EventT *evp) {
+ DCHECK_LE(evp + 1, &thr->tctx->trace.parts.Back()->events[TracePart::kSize]);
+ atomic_store_relaxed(&thr->trace_pos, (uptr)(evp + 1));
+}
+
+template <typename EventT>
+void TraceEvent(ThreadState *thr, EventT ev) {
+ EventT *evp;
+ if (!TraceAcquire(thr, &evp)) {
+ TraceSwitchPart(thr);
+ UNUSED bool res = TraceAcquire(thr, &evp);
+ DCHECK(res);
+ }
+ *evp = ev;
+ TraceRelease(thr, evp);
+}
+
+ALWAYS_INLINE WARN_UNUSED_RESULT bool TryTraceFunc(ThreadState *thr,
+ uptr pc = 0) {
+ if (!kCollectHistory)
+ return true;
+ EventFunc *ev;
+ if (UNLIKELY(!TraceAcquire(thr, &ev)))
+ return false;
+ ev->is_access = 0;
+ ev->is_func = 1;
+ ev->pc = pc;
+ TraceRelease(thr, ev);
+ return true;
+}
+
+WARN_UNUSED_RESULT
+bool TryTraceMemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size,
+ AccessType typ);
+WARN_UNUSED_RESULT
+bool TryTraceMemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size,
+ AccessType typ);
+void TraceMemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size,
+ AccessType typ);
+void TraceFunc(ThreadState *thr, uptr pc = 0);
+void TraceMutexLock(ThreadState *thr, EventType type, uptr pc, uptr addr,
+ StackID stk);
+void TraceMutexUnlock(ThreadState *thr, uptr addr);
+void TraceTime(ThreadState *thr);
+
+void TraceRestartFuncExit(ThreadState *thr);
+void TraceRestartFuncEntry(ThreadState *thr, uptr pc);
+
+void GrowShadowStack(ThreadState *thr);
+
+ALWAYS_INLINE
+void FuncEntry(ThreadState *thr, uptr pc) {
+ DPrintf2("#%d: FuncEntry %p\n", (int)thr->fast_state.sid(), (void *)pc);
+ if (UNLIKELY(!TryTraceFunc(thr, pc)))
+ return TraceRestartFuncEntry(thr, pc);
+ DCHECK_GE(thr->shadow_stack_pos, thr->shadow_stack);
+#if !SANITIZER_GO
+ DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
+#else
+ if (thr->shadow_stack_pos == thr->shadow_stack_end)
+ GrowShadowStack(thr);
+#endif
+ thr->shadow_stack_pos[0] = pc;
+ thr->shadow_stack_pos++;
+}
+
+ALWAYS_INLINE
+void FuncExit(ThreadState *thr) {
+ DPrintf2("#%d: FuncExit\n", (int)thr->fast_state.sid());
+ if (UNLIKELY(!TryTraceFunc(thr, 0)))
+ return TraceRestartFuncExit(thr);
+ DCHECK_GT(thr->shadow_stack_pos, thr->shadow_stack);
+#if !SANITIZER_GO
+ DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
+#endif
+ thr->shadow_stack_pos--;
+}
+
+#if !SANITIZER_GO
+extern void (*on_initialize)(void);
+extern int (*on_finalize)(int);
+#endif
} // namespace __tsan
#endif // TSAN_RTL_H
--- /dev/null
+//===-- tsan_rtl_access.cpp -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// Definitions of memory access and function entry/exit entry points.
+//===----------------------------------------------------------------------===//
+
+#include "tsan_rtl.h"
+
+namespace __tsan {
+
+ALWAYS_INLINE USED bool TryTraceMemoryAccess(ThreadState* thr, uptr pc,
+ uptr addr, uptr size,
+ AccessType typ) {
+ DCHECK(size == 1 || size == 2 || size == 4 || size == 8);
+ if (!kCollectHistory)
+ return true;
+ EventAccess* ev;
+ if (UNLIKELY(!TraceAcquire(thr, &ev)))
+ return false;
+ u64 size_log = size == 1 ? 0 : size == 2 ? 1 : size == 4 ? 2 : 3;
+ uptr pc_delta = pc - thr->trace_prev_pc + (1 << (EventAccess::kPCBits - 1));
+ thr->trace_prev_pc = pc;
+ if (LIKELY(pc_delta < (1 << EventAccess::kPCBits))) {
+ ev->is_access = 1;
+ ev->is_read = !!(typ & kAccessRead);
+ ev->is_atomic = !!(typ & kAccessAtomic);
+ ev->size_log = size_log;
+ ev->pc_delta = pc_delta;
+ DCHECK_EQ(ev->pc_delta, pc_delta);
+ ev->addr = CompressAddr(addr);
+ TraceRelease(thr, ev);
+ return true;
+ }
+ auto* evex = reinterpret_cast<EventAccessExt*>(ev);
+ evex->is_access = 0;
+ evex->is_func = 0;
+ evex->type = EventType::kAccessExt;
+ evex->is_read = !!(typ & kAccessRead);
+ evex->is_atomic = !!(typ & kAccessAtomic);
+ evex->size_log = size_log;
+ // Note: this is important, see comment in EventAccessExt.
+ evex->_ = 0;
+ evex->addr = CompressAddr(addr);
+ evex->pc = pc;
+ TraceRelease(thr, evex);
+ return true;
+}
+
+ALWAYS_INLINE
+bool TryTraceMemoryAccessRange(ThreadState* thr, uptr pc, uptr addr, uptr size,
+ AccessType typ) {
+ if (!kCollectHistory)
+ return true;
+ EventAccessRange* ev;
+ if (UNLIKELY(!TraceAcquire(thr, &ev)))
+ return false;
+ thr->trace_prev_pc = pc;
+ ev->is_access = 0;
+ ev->is_func = 0;
+ ev->type = EventType::kAccessRange;
+ ev->is_read = !!(typ & kAccessRead);
+ ev->is_free = !!(typ & kAccessFree);
+ ev->size_lo = size;
+ ev->pc = CompressAddr(pc);
+ ev->addr = CompressAddr(addr);
+ ev->size_hi = size >> EventAccessRange::kSizeLoBits;
+ TraceRelease(thr, ev);
+ return true;
+}
+
+void TraceMemoryAccessRange(ThreadState* thr, uptr pc, uptr addr, uptr size,
+ AccessType typ) {
+ if (LIKELY(TryTraceMemoryAccessRange(thr, pc, addr, size, typ)))
+ return;
+ TraceSwitchPart(thr);
+ UNUSED bool res = TryTraceMemoryAccessRange(thr, pc, addr, size, typ);
+ DCHECK(res);
+}
+
+void TraceFunc(ThreadState* thr, uptr pc) {
+ if (LIKELY(TryTraceFunc(thr, pc)))
+ return;
+ TraceSwitchPart(thr);
+ UNUSED bool res = TryTraceFunc(thr, pc);
+ DCHECK(res);
+}
+
+NOINLINE void TraceRestartFuncEntry(ThreadState* thr, uptr pc) {
+ TraceSwitchPart(thr);
+ FuncEntry(thr, pc);
+}
+
+NOINLINE void TraceRestartFuncExit(ThreadState* thr) {
+ TraceSwitchPart(thr);
+ FuncExit(thr);
+}
+
+void TraceMutexLock(ThreadState* thr, EventType type, uptr pc, uptr addr,
+ StackID stk) {
+ DCHECK(type == EventType::kLock || type == EventType::kRLock);
+ if (!kCollectHistory)
+ return;
+ EventLock ev;
+ ev.is_access = 0;
+ ev.is_func = 0;
+ ev.type = type;
+ ev.pc = CompressAddr(pc);
+ ev.stack_lo = stk;
+ ev.stack_hi = stk >> EventLock::kStackIDLoBits;
+ ev._ = 0;
+ ev.addr = CompressAddr(addr);
+ TraceEvent(thr, ev);
+}
+
+void TraceMutexUnlock(ThreadState* thr, uptr addr) {
+ if (!kCollectHistory)
+ return;
+ EventUnlock ev;
+ ev.is_access = 0;
+ ev.is_func = 0;
+ ev.type = EventType::kUnlock;
+ ev._ = 0;
+ ev.addr = CompressAddr(addr);
+ TraceEvent(thr, ev);
+}
+
+void TraceTime(ThreadState* thr) {
+ if (!kCollectHistory)
+ return;
+ FastState fast_state = thr->fast_state;
+ EventTime ev;
+ ev.is_access = 0;
+ ev.is_func = 0;
+ ev.type = EventType::kTime;
+ ev.sid = static_cast<u64>(fast_state.sid());
+ ev.epoch = static_cast<u64>(fast_state.epoch());
+ ev._ = 0;
+ TraceEvent(thr, ev);
+}
+
+NOINLINE void DoReportRace(ThreadState* thr, RawShadow* shadow_mem, Shadow cur,
+ Shadow old,
+ AccessType typ) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
+ // For the free shadow markers the first element (that contains kFreeSid)
+ // triggers the race, but the second element contains info about the freeing
+ // thread, take it.
+ if (old.sid() == kFreeSid)
+ old = Shadow(LoadShadow(&shadow_mem[1]));
+ // This prevents trapping on this address in future.
+ for (uptr i = 0; i < kShadowCnt; i++)
+ StoreShadow(&shadow_mem[i], i == 0 ? Shadow::kRodata : Shadow::kEmpty);
+ // See the comment in MemoryRangeFreed as to why the slot is locked
+ // for free memory accesses. ReportRace must not be called with
+ // the slot locked because of the fork. But MemoryRangeFreed is not
+ // called during fork because fork sets ignore_reads_and_writes,
+ // so simply unlocking the slot should be fine.
+ if (typ & kAccessSlotLocked)
+ SlotUnlock(thr);
+ ReportRace(thr, shadow_mem, cur, Shadow(old), typ);
+ if (typ & kAccessSlotLocked)
+ SlotLock(thr);
+}
+
+#if !TSAN_VECTORIZE
+ALWAYS_INLINE
+bool ContainsSameAccess(RawShadow* s, Shadow cur, int unused0, int unused1,
+ AccessType typ) {
+ for (uptr i = 0; i < kShadowCnt; i++) {
+ auto old = LoadShadow(&s[i]);
+ if (!(typ & kAccessRead)) {
+ if (old == cur.raw())
+ return true;
+ continue;
+ }
+ auto masked = static_cast<RawShadow>(static_cast<u32>(old) |
+ static_cast<u32>(Shadow::kRodata));
+ if (masked == cur.raw())
+ return true;
+ if (!(typ & kAccessNoRodata) && !SANITIZER_GO) {
+ if (old == Shadow::kRodata)
+ return true;
+ }
+ }
+ return false;
+}
+
+ALWAYS_INLINE
+bool CheckRaces(ThreadState* thr, RawShadow* shadow_mem, Shadow cur,
+ int unused0, int unused1, AccessType typ) {
+ bool stored = false;
+ for (uptr idx = 0; idx < kShadowCnt; idx++) {
+ RawShadow* sp = &shadow_mem[idx];
+ Shadow old(LoadShadow(sp));
+ if (LIKELY(old.raw() == Shadow::kEmpty)) {
+ if (!(typ & kAccessCheckOnly) && !stored)
+ StoreShadow(sp, cur.raw());
+ return false;
+ }
+ if (LIKELY(!(cur.access() & old.access())))
+ continue;
+ if (LIKELY(cur.sid() == old.sid())) {
+ if (!(typ & kAccessCheckOnly) &&
+ LIKELY(cur.access() == old.access() && old.IsRWWeakerOrEqual(typ))) {
+ StoreShadow(sp, cur.raw());
+ stored = true;
+ }
+ continue;
+ }
+ if (LIKELY(old.IsBothReadsOrAtomic(typ)))
+ continue;
+ if (LIKELY(thr->clock.Get(old.sid()) >= old.epoch()))
+ continue;
+ DoReportRace(thr, shadow_mem, cur, old, typ);
+ return true;
+ }
+ // We did not find any races and had already stored
+ // the current access info, so we are done.
+ if (LIKELY(stored))
+ return false;
+ // Choose a random candidate slot and replace it.
+ uptr index =
+ atomic_load_relaxed(&thr->trace_pos) / sizeof(Event) % kShadowCnt;
+ StoreShadow(&shadow_mem[index], cur.raw());
+ return false;
+}
+
+# define LOAD_CURRENT_SHADOW(cur, shadow_mem) UNUSED int access = 0, shadow = 0
+
+#else /* !TSAN_VECTORIZE */
+
+ALWAYS_INLINE
+bool ContainsSameAccess(RawShadow* unused0, Shadow unused1, m128 shadow,
+ m128 access, AccessType typ) {
+ // Note: we could check if there is a larger access of the same type,
+ // e.g. we just allocated/memset-ed a block (so it contains 8 byte writes)
+ // and now do smaller reads/writes, these can also be considered as "same
+ // access". However, it will make the check more expensive, so it's unclear
+ // if it's worth it. But this would conserve trace space, so it's useful
+ // besides potential speed up.
+ if (!(typ & kAccessRead)) {
+ const m128 same = _mm_cmpeq_epi32(shadow, access);
+ return _mm_movemask_epi8(same);
+ }
+ // For reads we need to reset read bit in the shadow,
+ // because we need to match read with both reads and writes.
+ // Shadow::kRodata has only read bit set, so it does what we want.
+ // We also abuse it for rodata check to save few cycles
+ // since we already loaded Shadow::kRodata into a register.
+ // Reads from rodata can't race.
+ // Measurements show that they can be 10-20% of all memory accesses.
+ // Shadow::kRodata has epoch 0 which cannot appear in shadow normally
+ // (thread epochs start from 1). So the same read bit mask
+ // serves as rodata indicator.
+ const m128 read_mask = _mm_set1_epi32(static_cast<u32>(Shadow::kRodata));
+ const m128 masked_shadow = _mm_or_si128(shadow, read_mask);
+ m128 same = _mm_cmpeq_epi32(masked_shadow, access);
+ // Range memory accesses check Shadow::kRodata before calling this,
+ // Shadow::kRodatas is not possible for free memory access
+ // and Go does not use Shadow::kRodata.
+ if (!(typ & kAccessNoRodata) && !SANITIZER_GO) {
+ const m128 ro = _mm_cmpeq_epi32(shadow, read_mask);
+ same = _mm_or_si128(ro, same);
+ }
+ return _mm_movemask_epi8(same);
+}
+
+NOINLINE void DoReportRaceV(ThreadState* thr, RawShadow* shadow_mem, Shadow cur,
+ u32 race_mask, m128 shadow, AccessType typ) {
+ // race_mask points which of the shadow elements raced with the current
+ // access. Extract that element.
+ CHECK_NE(race_mask, 0);
+ u32 old;
+ // Note: _mm_extract_epi32 index must be a constant value.
+ switch (__builtin_ffs(race_mask) / 4) {
+ case 0:
+ old = _mm_extract_epi32(shadow, 0);
+ break;
+ case 1:
+ old = _mm_extract_epi32(shadow, 1);
+ break;
+ case 2:
+ old = _mm_extract_epi32(shadow, 2);
+ break;
+ case 3:
+ old = _mm_extract_epi32(shadow, 3);
+ break;
+ }
+ Shadow prev(static_cast<RawShadow>(old));
+ // For the free shadow markers the first element (that contains kFreeSid)
+ // triggers the race, but the second element contains info about the freeing
+ // thread, take it.
+ if (prev.sid() == kFreeSid)
+ prev = Shadow(static_cast<RawShadow>(_mm_extract_epi32(shadow, 1)));
+ DoReportRace(thr, shadow_mem, cur, prev, typ);
+}
+
+ALWAYS_INLINE
+bool CheckRaces(ThreadState* thr, RawShadow* shadow_mem, Shadow cur,
+ m128 shadow, m128 access, AccessType typ) {
+ // Note: empty/zero slots don't intersect with any access.
+ const m128 zero = _mm_setzero_si128();
+ const m128 mask_access = _mm_set1_epi32(0x000000ff);
+ const m128 mask_sid = _mm_set1_epi32(0x0000ff00);
+ const m128 mask_read_atomic = _mm_set1_epi32(0xc0000000);
+ const m128 access_and = _mm_and_si128(access, shadow);
+ const m128 access_xor = _mm_xor_si128(access, shadow);
+ const m128 intersect = _mm_and_si128(access_and, mask_access);
+ const m128 not_intersect = _mm_cmpeq_epi32(intersect, zero);
+ const m128 not_same_sid = _mm_and_si128(access_xor, mask_sid);
+ const m128 same_sid = _mm_cmpeq_epi32(not_same_sid, zero);
+ const m128 both_read_or_atomic = _mm_and_si128(access_and, mask_read_atomic);
+ const m128 no_race =
+ _mm_or_si128(_mm_or_si128(not_intersect, same_sid), both_read_or_atomic);
+ const int race_mask = _mm_movemask_epi8(_mm_cmpeq_epi32(no_race, zero));
+ if (UNLIKELY(race_mask))
+ goto SHARED;
+
+STORE : {
+ if (typ & kAccessCheckOnly)
+ return false;
+ // We could also replace different sid's if access is the same,
+ // rw weaker and happens before. However, just checking access below
+ // is not enough because we also need to check that !both_read_or_atomic
+ // (reads from different sids can be concurrent).
+ // Theoretically we could replace smaller accesses with larger accesses,
+ // but it's unclear if it's worth doing.
+ const m128 mask_access_sid = _mm_set1_epi32(0x0000ffff);
+ const m128 not_same_sid_access = _mm_and_si128(access_xor, mask_access_sid);
+ const m128 same_sid_access = _mm_cmpeq_epi32(not_same_sid_access, zero);
+ const m128 access_read_atomic =
+ _mm_set1_epi32((typ & (kAccessRead | kAccessAtomic)) << 30);
+ const m128 rw_weaker =
+ _mm_cmpeq_epi32(_mm_max_epu32(shadow, access_read_atomic), shadow);
+ const m128 rewrite = _mm_and_si128(same_sid_access, rw_weaker);
+ const int rewrite_mask = _mm_movemask_epi8(rewrite);
+ int index = __builtin_ffs(rewrite_mask);
+ if (UNLIKELY(index == 0)) {
+ const m128 empty = _mm_cmpeq_epi32(shadow, zero);
+ const int empty_mask = _mm_movemask_epi8(empty);
+ index = __builtin_ffs(empty_mask);
+ if (UNLIKELY(index == 0))
+ index = (atomic_load_relaxed(&thr->trace_pos) / 2) % 16;
+ }
+ StoreShadow(&shadow_mem[index / 4], cur.raw());
+ // We could zero other slots determined by rewrite_mask.
+ // That would help other threads to evict better slots,
+ // but it's unclear if it's worth it.
+ return false;
+}
+
+SHARED:
+ m128 thread_epochs = _mm_set1_epi32(0x7fffffff);
+ // Need to unwind this because _mm_extract_epi8/_mm_insert_epi32
+ // indexes must be constants.
+# define LOAD_EPOCH(idx) \
+ if (LIKELY(race_mask & (1 << (idx * 4)))) { \
+ u8 sid = _mm_extract_epi8(shadow, idx * 4 + 1); \
+ u16 epoch = static_cast<u16>(thr->clock.Get(static_cast<Sid>(sid))); \
+ thread_epochs = _mm_insert_epi32(thread_epochs, u32(epoch) << 16, idx); \
+ }
+ LOAD_EPOCH(0);
+ LOAD_EPOCH(1);
+ LOAD_EPOCH(2);
+ LOAD_EPOCH(3);
+# undef LOAD_EPOCH
+ const m128 mask_epoch = _mm_set1_epi32(0x3fff0000);
+ const m128 shadow_epochs = _mm_and_si128(shadow, mask_epoch);
+ const m128 concurrent = _mm_cmplt_epi32(thread_epochs, shadow_epochs);
+ const int concurrent_mask = _mm_movemask_epi8(concurrent);
+ if (LIKELY(concurrent_mask == 0))
+ goto STORE;
+
+ DoReportRaceV(thr, shadow_mem, cur, concurrent_mask, shadow, typ);
+ return true;
+}
+
+# define LOAD_CURRENT_SHADOW(cur, shadow_mem) \
+ const m128 access = _mm_set1_epi32(static_cast<u32>((cur).raw())); \
+ const m128 shadow = _mm_load_si128(reinterpret_cast<m128*>(shadow_mem))
+#endif
+
+char* DumpShadow(char* buf, RawShadow raw) {
+ if (raw == Shadow::kEmpty) {
+ internal_snprintf(buf, 64, "0");
+ return buf;
+ }
+ Shadow s(raw);
+ AccessType typ;
+ s.GetAccess(nullptr, nullptr, &typ);
+ internal_snprintf(buf, 64, "{tid=%u@%u access=0x%x typ=%x}",
+ static_cast<u32>(s.sid()), static_cast<u32>(s.epoch()),
+ s.access(), static_cast<u32>(typ));
+ return buf;
+}
+
+// TryTrace* and TraceRestart* functions allow to turn memory access and func
+// entry/exit callbacks into leaf functions with all associated performance
+// benefits. These hottest callbacks do only 2 slow path calls: report a race
+// and trace part switching. Race reporting is easy to turn into a tail call, we
+// just always return from the runtime after reporting a race. But trace part
+// switching is harder because it needs to be in the middle of callbacks. To
+// turn it into a tail call we immidiately return after TraceRestart* functions,
+// but TraceRestart* functions themselves recurse into the callback after
+// switching trace part. As the result the hottest callbacks contain only tail
+// calls, which effectively makes them leaf functions (can use all registers,
+// no frame setup, etc).
+NOINLINE void TraceRestartMemoryAccess(ThreadState* thr, uptr pc, uptr addr,
+ uptr size, AccessType typ) {
+ TraceSwitchPart(thr);
+ MemoryAccess(thr, pc, addr, size, typ);
+}
+
+ALWAYS_INLINE USED void MemoryAccess(ThreadState* thr, uptr pc, uptr addr,
+ uptr size, AccessType typ) {
+ RawShadow* shadow_mem = MemToShadow(addr);
+ UNUSED char memBuf[4][64];
+ DPrintf2("#%d: Access: %d@%d %p/%zd typ=0x%x {%s, %s, %s, %s}\n", thr->tid,
+ static_cast<int>(thr->fast_state.sid()),
+ static_cast<int>(thr->fast_state.epoch()), (void*)addr, size,
+ static_cast<int>(typ), DumpShadow(memBuf[0], shadow_mem[0]),
+ DumpShadow(memBuf[1], shadow_mem[1]),
+ DumpShadow(memBuf[2], shadow_mem[2]),
+ DumpShadow(memBuf[3], shadow_mem[3]));
+
+ FastState fast_state = thr->fast_state;
+ Shadow cur(fast_state, addr, size, typ);
+
+ LOAD_CURRENT_SHADOW(cur, shadow_mem);
+ if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))
+ return;
+ if (UNLIKELY(fast_state.GetIgnoreBit()))
+ return;
+ if (!TryTraceMemoryAccess(thr, pc, addr, size, typ))
+ return TraceRestartMemoryAccess(thr, pc, addr, size, typ);
+ CheckRaces(thr, shadow_mem, cur, shadow, access, typ);
+}
+
+void MemoryAccess16(ThreadState* thr, uptr pc, uptr addr, AccessType typ);
+
+NOINLINE
+void RestartMemoryAccess16(ThreadState* thr, uptr pc, uptr addr,
+ AccessType typ) {
+ TraceSwitchPart(thr);
+ MemoryAccess16(thr, pc, addr, typ);
+}
+
+ALWAYS_INLINE USED void MemoryAccess16(ThreadState* thr, uptr pc, uptr addr,
+ AccessType typ) {
+ const uptr size = 16;
+ FastState fast_state = thr->fast_state;
+ if (UNLIKELY(fast_state.GetIgnoreBit()))
+ return;
+ Shadow cur(fast_state, 0, 8, typ);
+ RawShadow* shadow_mem = MemToShadow(addr);
+ bool traced = false;
+ {
+ LOAD_CURRENT_SHADOW(cur, shadow_mem);
+ if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))
+ goto SECOND;
+ if (!TryTraceMemoryAccessRange(thr, pc, addr, size, typ))
+ return RestartMemoryAccess16(thr, pc, addr, typ);
+ traced = true;
+ if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, shadow, access, typ)))
+ return;
+ }
+SECOND:
+ shadow_mem += kShadowCnt;
+ LOAD_CURRENT_SHADOW(cur, shadow_mem);
+ if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))
+ return;
+ if (!traced && !TryTraceMemoryAccessRange(thr, pc, addr, size, typ))
+ return RestartMemoryAccess16(thr, pc, addr, typ);
+ CheckRaces(thr, shadow_mem, cur, shadow, access, typ);
+}
+
+NOINLINE
+void RestartUnalignedMemoryAccess(ThreadState* thr, uptr pc, uptr addr,
+ uptr size, AccessType typ) {
+ TraceSwitchPart(thr);
+ UnalignedMemoryAccess(thr, pc, addr, size, typ);
+}
+
+ALWAYS_INLINE USED void UnalignedMemoryAccess(ThreadState* thr, uptr pc,
+ uptr addr, uptr size,
+ AccessType typ) {
+ DCHECK_LE(size, 8);
+ FastState fast_state = thr->fast_state;
+ if (UNLIKELY(fast_state.GetIgnoreBit()))
+ return;
+ RawShadow* shadow_mem = MemToShadow(addr);
+ bool traced = false;
+ uptr size1 = Min<uptr>(size, RoundUp(addr + 1, kShadowCell) - addr);
+ {
+ Shadow cur(fast_state, addr, size1, typ);
+ LOAD_CURRENT_SHADOW(cur, shadow_mem);
+ if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))
+ goto SECOND;
+ if (!TryTraceMemoryAccessRange(thr, pc, addr, size, typ))
+ return RestartUnalignedMemoryAccess(thr, pc, addr, size, typ);
+ traced = true;
+ if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, shadow, access, typ)))
+ return;
+ }
+SECOND:
+ uptr size2 = size - size1;
+ if (LIKELY(size2 == 0))
+ return;
+ shadow_mem += kShadowCnt;
+ Shadow cur(fast_state, 0, size2, typ);
+ LOAD_CURRENT_SHADOW(cur, shadow_mem);
+ if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))
+ return;
+ if (!traced && !TryTraceMemoryAccessRange(thr, pc, addr, size, typ))
+ return RestartUnalignedMemoryAccess(thr, pc, addr, size, typ);
+ CheckRaces(thr, shadow_mem, cur, shadow, access, typ);
+}
+
+void ShadowSet(RawShadow* p, RawShadow* end, RawShadow v) {
+ DCHECK_LE(p, end);
+ DCHECK(IsShadowMem(p));
+ DCHECK(IsShadowMem(end));
+ UNUSED const uptr kAlign = kShadowCnt * kShadowSize;
+ DCHECK_EQ(reinterpret_cast<uptr>(p) % kAlign, 0);
+ DCHECK_EQ(reinterpret_cast<uptr>(end) % kAlign, 0);
+#if !TSAN_VECTORIZE
+ for (; p < end; p += kShadowCnt) {
+ p[0] = v;
+ for (uptr i = 1; i < kShadowCnt; i++) p[i] = Shadow::kEmpty;
+ }
+#else
+ m128 vv = _mm_setr_epi32(
+ static_cast<u32>(v), static_cast<u32>(Shadow::kEmpty),
+ static_cast<u32>(Shadow::kEmpty), static_cast<u32>(Shadow::kEmpty));
+ m128* vp = reinterpret_cast<m128*>(p);
+ m128* vend = reinterpret_cast<m128*>(end);
+ for (; vp < vend; vp++) _mm_store_si128(vp, vv);
+#endif
+}
+
+static void MemoryRangeSet(uptr addr, uptr size, RawShadow val) {
+ if (size == 0)
+ return;
+ DCHECK_EQ(addr % kShadowCell, 0);
+ DCHECK_EQ(size % kShadowCell, 0);
+ // If a user passes some insane arguments (memset(0)),
+ // let it just crash as usual.
+ if (!IsAppMem(addr) || !IsAppMem(addr + size - 1))
+ return;
+ RawShadow* begin = MemToShadow(addr);
+ RawShadow* end = begin + size / kShadowCell * kShadowCnt;
+ // Don't want to touch lots of shadow memory.
+ // If a program maps 10MB stack, there is no need reset the whole range.
+ // UnmapOrDie/MmapFixedNoReserve does not work on Windows.
+ if (SANITIZER_WINDOWS ||
+ size <= common_flags()->clear_shadow_mmap_threshold) {
+ ShadowSet(begin, end, val);
+ return;
+ }
+ // The region is big, reset only beginning and end.
+ const uptr kPageSize = GetPageSizeCached();
+ // Set at least first kPageSize/2 to page boundary.
+ RawShadow* mid1 =
+ Min(end, reinterpret_cast<RawShadow*>(RoundUp(
+ reinterpret_cast<uptr>(begin) + kPageSize / 2, kPageSize)));
+ ShadowSet(begin, mid1, val);
+ // Reset middle part.
+ RawShadow* mid2 = RoundDown(end, kPageSize);
+ if (mid2 > mid1) {
+ if (!MmapFixedSuperNoReserve((uptr)mid1, (uptr)mid2 - (uptr)mid1))
+ Die();
+ }
+ // Set the ending.
+ ShadowSet(mid2, end, val);
+}
+
+void MemoryResetRange(ThreadState* thr, uptr pc, uptr addr, uptr size) {
+ uptr addr1 = RoundDown(addr, kShadowCell);
+ uptr size1 = RoundUp(size + addr - addr1, kShadowCell);
+ MemoryRangeSet(addr1, size1, Shadow::kEmpty);
+}
+
+void MemoryRangeFreed(ThreadState* thr, uptr pc, uptr addr, uptr size) {
+ // Callers must lock the slot to ensure synchronization with the reset.
+ // The problem with "freed" memory is that it's not "monotonic"
+ // with respect to bug detection: freed memory is bad to access,
+ // but then if the heap block is reallocated later, it's good to access.
+ // As the result a garbage "freed" shadow can lead to a false positive
+ // if it happens to match a real free in the thread trace,
+ // but the heap block was reallocated before the current memory access,
+ // so it's still good to access. It's not the case with data races.
+ DCHECK(thr->slot_locked);
+ DCHECK_EQ(addr % kShadowCell, 0);
+ size = RoundUp(size, kShadowCell);
+ // Processing more than 1k (2k of shadow) is expensive,
+ // can cause excessive memory consumption (user does not necessary touch
+ // the whole range) and most likely unnecessary.
+ size = Min<uptr>(size, 1024);
+ const AccessType typ = kAccessWrite | kAccessFree | kAccessSlotLocked |
+ kAccessCheckOnly | kAccessNoRodata;
+ TraceMemoryAccessRange(thr, pc, addr, size, typ);
+ RawShadow* shadow_mem = MemToShadow(addr);
+ Shadow cur(thr->fast_state, 0, kShadowCell, typ);
+#if TSAN_VECTORIZE
+ const m128 access = _mm_set1_epi32(static_cast<u32>(cur.raw()));
+ const m128 freed = _mm_setr_epi32(
+ static_cast<u32>(Shadow::FreedMarker()),
+ static_cast<u32>(Shadow::FreedInfo(cur.sid(), cur.epoch())), 0, 0);
+ for (; size; size -= kShadowCell, shadow_mem += kShadowCnt) {
+ const m128 shadow = _mm_load_si128((m128*)shadow_mem);
+ if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, shadow, access, typ)))
+ return;
+ _mm_store_si128((m128*)shadow_mem, freed);
+ }
+#else
+ for (; size; size -= kShadowCell, shadow_mem += kShadowCnt) {
+ if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, 0, 0, typ)))
+ return;
+ StoreShadow(&shadow_mem[0], Shadow::FreedMarker());
+ StoreShadow(&shadow_mem[1], Shadow::FreedInfo(cur.sid(), cur.epoch()));
+ StoreShadow(&shadow_mem[2], Shadow::kEmpty);
+ StoreShadow(&shadow_mem[3], Shadow::kEmpty);
+ }
+#endif
+}
+
+void MemoryRangeImitateWrite(ThreadState* thr, uptr pc, uptr addr, uptr size) {
+ DCHECK_EQ(addr % kShadowCell, 0);
+ size = RoundUp(size, kShadowCell);
+ TraceMemoryAccessRange(thr, pc, addr, size, kAccessWrite);
+ Shadow cur(thr->fast_state, 0, 8, kAccessWrite);
+ MemoryRangeSet(addr, size, cur.raw());
+}
+
+void MemoryRangeImitateWriteOrResetRange(ThreadState* thr, uptr pc, uptr addr,
+ uptr size) {
+ if (thr->ignore_reads_and_writes == 0)
+ MemoryRangeImitateWrite(thr, pc, addr, size);
+ else
+ MemoryResetRange(thr, pc, addr, size);
+}
+
+ALWAYS_INLINE
+bool MemoryAccessRangeOne(ThreadState* thr, RawShadow* shadow_mem, Shadow cur,
+ AccessType typ) {
+ LOAD_CURRENT_SHADOW(cur, shadow_mem);
+ if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))
+ return false;
+ return CheckRaces(thr, shadow_mem, cur, shadow, access, typ);
+}
+
+template <bool is_read>
+NOINLINE void RestartMemoryAccessRange(ThreadState* thr, uptr pc, uptr addr,
+ uptr size) {
+ TraceSwitchPart(thr);
+ MemoryAccessRangeT<is_read>(thr, pc, addr, size);
+}
+
+template <bool is_read>
+void MemoryAccessRangeT(ThreadState* thr, uptr pc, uptr addr, uptr size) {
+ const AccessType typ =
+ (is_read ? kAccessRead : kAccessWrite) | kAccessNoRodata;
+ RawShadow* shadow_mem = MemToShadow(addr);
+ DPrintf2("#%d: MemoryAccessRange: @%p %p size=%d is_read=%d\n", thr->tid,
+ (void*)pc, (void*)addr, (int)size, is_read);
+
+#if SANITIZER_DEBUG
+ if (!IsAppMem(addr)) {
+ Printf("Access to non app mem %zx\n", addr);
+ DCHECK(IsAppMem(addr));
+ }
+ if (!IsAppMem(addr + size - 1)) {
+ Printf("Access to non app mem %zx\n", addr + size - 1);
+ DCHECK(IsAppMem(addr + size - 1));
+ }
+ if (!IsShadowMem(shadow_mem)) {
+ Printf("Bad shadow addr %p (%zx)\n", static_cast<void*>(shadow_mem), addr);
+ DCHECK(IsShadowMem(shadow_mem));
+ }
+ if (!IsShadowMem(shadow_mem + size * kShadowCnt - 1)) {
+ Printf("Bad shadow addr %p (%zx)\n",
+ static_cast<void*>(shadow_mem + size * kShadowCnt - 1),
+ addr + size - 1);
+ DCHECK(IsShadowMem(shadow_mem + size * kShadowCnt - 1));
+ }
+#endif
+
+ // Access to .rodata section, no races here.
+ // Measurements show that it can be 10-20% of all memory accesses.
+ // Check here once to not check for every access separately.
+ // Note: we could (and should) do this only for the is_read case
+ // (writes shouldn't go to .rodata). But it happens in Chromium tests:
+ // https://bugs.chromium.org/p/chromium/issues/detail?id=1275581#c19
+ // Details are unknown since it happens only on CI machines.
+ if (*shadow_mem == Shadow::kRodata)
+ return;
+
+ FastState fast_state = thr->fast_state;
+ if (UNLIKELY(fast_state.GetIgnoreBit()))
+ return;
+
+ if (!TryTraceMemoryAccessRange(thr, pc, addr, size, typ))
+ return RestartMemoryAccessRange<is_read>(thr, pc, addr, size);
+
+ if (UNLIKELY(addr % kShadowCell)) {
+ // Handle unaligned beginning, if any.
+ uptr size1 = Min(size, RoundUp(addr, kShadowCell) - addr);
+ size -= size1;
+ Shadow cur(fast_state, addr, size1, typ);
+ if (UNLIKELY(MemoryAccessRangeOne(thr, shadow_mem, cur, typ)))
+ return;
+ shadow_mem += kShadowCnt;
+ }
+ // Handle middle part, if any.
+ Shadow cur(fast_state, 0, kShadowCell, typ);
+ for (; size >= kShadowCell; size -= kShadowCell, shadow_mem += kShadowCnt) {
+ if (UNLIKELY(MemoryAccessRangeOne(thr, shadow_mem, cur, typ)))
+ return;
+ }
+ // Handle ending, if any.
+ if (UNLIKELY(size)) {
+ Shadow cur(fast_state, 0, size, typ);
+ if (UNLIKELY(MemoryAccessRangeOne(thr, shadow_mem, cur, typ)))
+ return;
+ }
+}
+
+template void MemoryAccessRangeT<true>(ThreadState* thr, uptr pc, uptr addr,
+ uptr size);
+template void MemoryAccessRangeT<false>(ThreadState* thr, uptr pc, uptr addr,
+ uptr size);
+
+} // namespace __tsan
+
+#if !SANITIZER_GO
+// Must be included in this file to make sure everything is inlined.
+# include "tsan_interface.inc"
+#endif
.section __TEXT,__text
#endif
-ASM_HIDDEN(__tsan_trace_switch)
-.globl ASM_SYMBOL(__tsan_trace_switch_thunk)
-ASM_SYMBOL(__tsan_trace_switch_thunk):
- CFI_STARTPROC
- # Save scratch registers.
- push %rax
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%rax, 0)
- push %rcx
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%rcx, 0)
- push %rdx
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%rdx, 0)
- push %rsi
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%rsi, 0)
- push %rdi
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%rdi, 0)
- push %r8
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%r8, 0)
- push %r9
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%r9, 0)
- push %r10
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%r10, 0)
- push %r11
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%r11, 0)
- # Align stack frame.
- push %rbx # non-scratch
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%rbx, 0)
- mov %rsp, %rbx # save current rsp
- CFI_DEF_CFA_REGISTER(%rbx)
- shr $4, %rsp # clear 4 lsb, align to 16
- shl $4, %rsp
-
- call ASM_SYMBOL(__tsan_trace_switch)
-
- # Unalign stack frame back.
- mov %rbx, %rsp # restore the original rsp
- CFI_DEF_CFA_REGISTER(%rsp)
- pop %rbx
- CFI_ADJUST_CFA_OFFSET(-8)
- # Restore scratch registers.
- pop %r11
- CFI_ADJUST_CFA_OFFSET(-8)
- pop %r10
- CFI_ADJUST_CFA_OFFSET(-8)
- pop %r9
- CFI_ADJUST_CFA_OFFSET(-8)
- pop %r8
- CFI_ADJUST_CFA_OFFSET(-8)
- pop %rdi
- CFI_ADJUST_CFA_OFFSET(-8)
- pop %rsi
- CFI_ADJUST_CFA_OFFSET(-8)
- pop %rdx
- CFI_ADJUST_CFA_OFFSET(-8)
- pop %rcx
- CFI_ADJUST_CFA_OFFSET(-8)
- pop %rax
- CFI_ADJUST_CFA_OFFSET(-8)
- CFI_RESTORE(%rax)
- CFI_RESTORE(%rbx)
- CFI_RESTORE(%rcx)
- CFI_RESTORE(%rdx)
- CFI_RESTORE(%rsi)
- CFI_RESTORE(%rdi)
- CFI_RESTORE(%r8)
- CFI_RESTORE(%r9)
- CFI_RESTORE(%r10)
- CFI_RESTORE(%r11)
- ret
- CFI_ENDPROC
-
-ASM_HIDDEN(__tsan_report_race)
-.globl ASM_SYMBOL(__tsan_report_race_thunk)
-ASM_SYMBOL(__tsan_report_race_thunk):
- CFI_STARTPROC
- # Save scratch registers.
- push %rax
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%rax, 0)
- push %rcx
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%rcx, 0)
- push %rdx
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%rdx, 0)
- push %rsi
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%rsi, 0)
- push %rdi
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%rdi, 0)
- push %r8
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%r8, 0)
- push %r9
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%r9, 0)
- push %r10
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%r10, 0)
- push %r11
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%r11, 0)
- # Align stack frame.
- push %rbx # non-scratch
- CFI_ADJUST_CFA_OFFSET(8)
- CFI_REL_OFFSET(%rbx, 0)
- mov %rsp, %rbx # save current rsp
- CFI_DEF_CFA_REGISTER(%rbx)
- shr $4, %rsp # clear 4 lsb, align to 16
- shl $4, %rsp
-
- call ASM_SYMBOL(__tsan_report_race)
-
- # Unalign stack frame back.
- mov %rbx, %rsp # restore the original rsp
- CFI_DEF_CFA_REGISTER(%rsp)
- pop %rbx
- CFI_ADJUST_CFA_OFFSET(-8)
- # Restore scratch registers.
- pop %r11
- CFI_ADJUST_CFA_OFFSET(-8)
- pop %r10
- CFI_ADJUST_CFA_OFFSET(-8)
- pop %r9
- CFI_ADJUST_CFA_OFFSET(-8)
- pop %r8
- CFI_ADJUST_CFA_OFFSET(-8)
- pop %rdi
- CFI_ADJUST_CFA_OFFSET(-8)
- pop %rsi
- CFI_ADJUST_CFA_OFFSET(-8)
- pop %rdx
- CFI_ADJUST_CFA_OFFSET(-8)
- pop %rcx
- CFI_ADJUST_CFA_OFFSET(-8)
- pop %rax
- CFI_ADJUST_CFA_OFFSET(-8)
- CFI_RESTORE(%rax)
- CFI_RESTORE(%rbx)
- CFI_RESTORE(%rcx)
- CFI_RESTORE(%rdx)
- CFI_RESTORE(%rsi)
- CFI_RESTORE(%rdi)
- CFI_RESTORE(%r8)
- CFI_RESTORE(%r9)
- CFI_RESTORE(%r10)
- CFI_RESTORE(%r11)
- ret
- CFI_ENDPROC
-
ASM_HIDDEN(__tsan_setjmp)
#if defined(__NetBSD__)
.comm _ZN14__interception15real___setjmp14E,8,8
ASM_SYMBOL_INTERCEPTOR(setjmp):
#endif
CFI_STARTPROC
+ _CET_ENDBR
// save env parameter
push %rdi
CFI_ADJUST_CFA_OFFSET(8)
ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_setjmp))
ASM_SYMBOL_INTERCEPTOR(_setjmp):
CFI_STARTPROC
+ _CET_ENDBR
// save env parameter
push %rdi
CFI_ADJUST_CFA_OFFSET(8)
ASM_SYMBOL_INTERCEPTOR(sigsetjmp):
#endif
CFI_STARTPROC
+ _CET_ENDBR
// save env parameter
push %rdi
CFI_ADJUST_CFA_OFFSET(8)
ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
ASM_SYMBOL_INTERCEPTOR(__sigsetjmp):
CFI_STARTPROC
+ _CET_ENDBR
// save env parameter
push %rdi
CFI_ADJUST_CFA_OFFSET(8)
--- /dev/null
+#include "sanitizer_common/sanitizer_asm.h"
+
+.section .text
+
+ASM_HIDDEN(__tsan_setjmp)
+.comm _ZN14__interception11real_setjmpE,8,8
+.globl ASM_SYMBOL_INTERCEPTOR(setjmp)
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(setjmp))
+ASM_SYMBOL_INTERCEPTOR(setjmp):
+ CFI_STARTPROC
+
+ // Save frame pointer and return address register
+ addi.d $sp, $sp, -32
+ st.d $ra, $sp, 24
+ st.d $fp, $sp, 16
+ CFI_DEF_CFA_OFFSET (32)
+ CFI_OFFSET (1, -8)
+ CFI_OFFSET (22, -16)
+
+ // Adjust the SP for previous frame
+ addi.d $fp, $sp, 32
+ CFI_DEF_CFA_REGISTER (22)
+
+ // Save env parameter
+ st.d $a0, $sp, 8
+ CFI_OFFSET (4, -24)
+
+ // Obtain SP, first argument to `void __tsan_setjmp(uptr sp)`
+ addi.d $a0, $fp, 0
+
+ // call tsan interceptor
+ bl ASM_SYMBOL(__tsan_setjmp)
+
+ // Restore env parameter
+ ld.d $a0, $sp, 8
+ CFI_RESTORE (4)
+
+ // Restore frame/link register
+ ld.d $fp, $sp, 16
+ ld.d $ra, $sp, 24
+ addi.d $sp, $sp, 32
+ CFI_RESTORE (22)
+ CFI_RESTORE (1)
+ CFI_DEF_CFA (3, 0)
+
+ // tail jump to libc setjmp
+ la.local $a1, _ZN14__interception11real_setjmpE
+ ld.d $a1, $a1, 0
+ jr $a1
+
+ CFI_ENDPROC
+ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(setjmp))
+
+.comm _ZN14__interception12real__setjmpE,8,8
+.globl ASM_SYMBOL_INTERCEPTOR(_setjmp)
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_setjmp))
+ASM_SYMBOL_INTERCEPTOR(_setjmp):
+ CFI_STARTPROC
+
+ // Save frame pointer and return address register
+ addi.d $sp, $sp, -32
+ st.d $ra, $sp, 24
+ st.d $fp, $sp, 16
+ CFI_DEF_CFA_OFFSET (32)
+ CFI_OFFSET (1, -8)
+ CFI_OFFSET (22, -16)
+
+ // Adjust the SP for previous frame
+ addi.d $fp, $sp, 32
+ CFI_DEF_CFA_REGISTER (22)
+
+ // Save env parameter
+ st.d $a0, $sp, 8
+ CFI_OFFSET (4, -24)
+
+ // Obtain SP, first argument to `void __tsan_setjmp(uptr sp)`
+ addi.d $a0, $fp, 0
+
+ // call tsan interceptor
+ bl ASM_SYMBOL(__tsan_setjmp)
+
+ // Restore env parameter
+ ld.d $a0, $sp, 8
+ CFI_RESTORE (4)
+
+ // Restore frame/link register
+ ld.d $fp, $sp, 16
+ ld.d $ra, $sp, 24
+ addi.d $sp, $sp, 32
+ CFI_RESTORE (22)
+ CFI_RESTORE (1)
+ CFI_DEF_CFA (3, 0)
+
+ // tail jump to libc setjmp
+ la.local $a1, _ZN14__interception12real__setjmpE
+ ld.d $a1, $a1, 0
+ jr $a1
+
+ CFI_ENDPROC
+ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(_setjmp))
+
+.comm _ZN14__interception14real_sigsetjmpE,8,8
+.globl ASM_SYMBOL_INTERCEPTOR(sigsetjmp)
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
+ASM_SYMBOL_INTERCEPTOR(sigsetjmp):
+ CFI_STARTPROC
+
+ // Save frame pointer and return address register
+ addi.d $sp, $sp, -32
+ st.d $ra, $sp, 24
+ st.d $fp, $sp, 16
+ CFI_DEF_CFA_OFFSET (32)
+ CFI_OFFSET (1, -8)
+ CFI_OFFSET (22, -16)
+
+ // Adjust the SP for previous frame
+ addi.d $fp, $sp, 32
+ CFI_DEF_CFA_REGISTER (22)
+
+ // Save env parameter
+ st.d $a0, $sp, 8
+ CFI_OFFSET (4, -24)
+
+ // Obtain SP, first argument to `void __tsan_setjmp(uptr sp)`
+ addi.d $a0, $fp, 0
+
+ // call tsan interceptor
+ bl ASM_SYMBOL(__tsan_setjmp)
+
+ // Restore env parameter
+ ld.d $a0, $sp, 8
+ CFI_RESTORE (4)
+
+ // Restore frame/link register
+ ld.d $fp, $sp, 16
+ ld.d $ra, $sp, 24
+ addi.d $sp, $sp, 32
+ CFI_RESTORE (22)
+ CFI_RESTORE (1)
+ CFI_DEF_CFA (3, 0)
+
+ // tail jump to libc setjmp
+ la.local $a1, _ZN14__interception14real_sigsetjmpE
+ ld.d $a1, $a1, 0
+ jr $a1
+
+ CFI_ENDPROC
+ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
+
+.comm _ZN14__interception16real___sigsetjmpE,8,8
+.globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
+ASM_SYMBOL_INTERCEPTOR(__sigsetjmp):
+ CFI_STARTPROC
+
+ // Save frame pointer and return address register
+ addi.d $sp, $sp, -32
+ st.d $ra, $sp, 24
+ st.d $fp, $sp, 16
+ CFI_DEF_CFA_OFFSET (32)
+ CFI_OFFSET (1, -8)
+ CFI_OFFSET (22, -16)
+
+ // Adjust the SP for previous frame
+ addi.d $fp, $sp, 32
+ CFI_DEF_CFA_REGISTER (22)
+
+ // Save env parameter
+ st.d $a0, $sp, 8
+ CFI_OFFSET (4, -24)
+
+ // Obtain SP, first argument to `void __tsan_setjmp(uptr sp)`
+ addi.d $a0, $fp, 0
+
+ // call tsan interceptor
+ bl ASM_SYMBOL(__tsan_setjmp)
+
+ // Restore env parameter
+ ld.d $a0, $sp, 8
+ CFI_RESTORE (4)
+
+ // Restore frame/link register
+ ld.d $fp, $sp, 16
+ ld.d $ra, $sp, 24
+ addi.d $sp, $sp, 32
+ CFI_RESTORE (22)
+ CFI_RESTORE (1)
+ CFI_DEF_CFA (3, 0)
+
+ // tail jump to libc setjmp
+ la.local $a1, _ZN14__interception16real___sigsetjmpE
+ ld.d $a1, $a1, 0
+ jr $a1
+
+ CFI_ENDPROC
+ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
namespace __tsan {
void ReportDeadlock(ThreadState *thr, uptr pc, DDReport *r);
+void ReportDestroyLocked(ThreadState *thr, uptr pc, uptr addr,
+ FastState last_lock, StackID creation_stack_id);
struct Callback final : public DDCallback {
ThreadState *thr;
DDCallback::lt = thr->dd_lt;
}
- u32 Unwind() override { return CurrentStackId(thr, pc); }
- int UniqueTid() override { return thr->unique_id; }
+ StackID Unwind() override { return CurrentStackId(thr, pc); }
+ int UniqueTid() override { return thr->tid; }
};
void DDMutexInit(ThreadState *thr, uptr pc, SyncVar *s) {
Callback cb(thr, pc);
ctx->dd->MutexInit(&cb, &s->dd);
- s->dd.ctx = s->GetId();
+ s->dd.ctx = s->addr;
}
static void ReportMutexMisuse(ThreadState *thr, uptr pc, ReportType typ,
- uptr addr, u64 mid) {
+ uptr addr, StackID creation_stack_id) {
// In Go, these misuses are either impossible, or detected by std lib,
// or false positives (e.g. unlock in a different thread).
if (SANITIZER_GO)
return;
if (!ShouldReport(thr, typ))
return;
- ThreadRegistryLock l(ctx->thread_registry);
+ ThreadRegistryLock l(&ctx->thread_registry);
ScopedReport rep(typ);
- rep.AddMutex(mid);
+ rep.AddMutex(addr, creation_stack_id);
VarSizeStackTrace trace;
ObtainCurrentStack(thr, pc, &trace);
rep.AddStack(trace, true);
OutputReport(thr, rep);
}
-void MutexCreate(ThreadState *thr, uptr pc, uptr addr, u32 flagz) NO_THREAD_SAFETY_ANALYSIS {
+static void RecordMutexLock(ThreadState *thr, uptr pc, uptr addr,
+ StackID stack_id, bool write) {
+ auto typ = write ? EventType::kLock : EventType::kRLock;
+ // Note: it's important to trace before modifying mutex set
+ // because tracing can switch trace part and we write the current
+ // mutex set in the beginning of each part.
+ // If we do it in the opposite order, we will write already reduced
+ // mutex set in the beginning of the part and then trace unlock again.
+ TraceMutexLock(thr, typ, pc, addr, stack_id);
+ thr->mset.AddAddr(addr, stack_id, write);
+}
+
+static void RecordMutexUnlock(ThreadState *thr, uptr addr) {
+ // See the comment in RecordMutexLock re order of operations.
+ TraceMutexUnlock(thr, addr);
+ thr->mset.DelAddr(addr);
+}
+
+void MutexCreate(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
DPrintf("#%d: MutexCreate %zx flagz=0x%x\n", thr->tid, addr, flagz);
- if (!(flagz & MutexFlagLinkerInit) && IsAppMem(addr)) {
- CHECK(!thr->is_freeing);
- thr->is_freeing = true;
- MemoryWrite(thr, pc, addr, kSizeLog1);
- thr->is_freeing = false;
- }
- SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);
+ if (!(flagz & MutexFlagLinkerInit) && pc && IsAppMem(addr))
+ MemoryAccess(thr, pc, addr, 1, kAccessWrite);
+ SlotLocker locker(thr);
+ auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
s->SetFlags(flagz & MutexCreationFlagMask);
- if (!SANITIZER_GO && s->creation_stack_id == 0)
+ // Save stack in the case the sync object was created before as atomic.
+ if (!SANITIZER_GO && s->creation_stack_id == kInvalidStackID)
s->creation_stack_id = CurrentStackId(thr, pc);
- s->mtx.Unlock();
}
-void MutexDestroy(ThreadState *thr, uptr pc, uptr addr, u32 flagz) NO_THREAD_SAFETY_ANALYSIS {
+void MutexDestroy(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
DPrintf("#%d: MutexDestroy %zx\n", thr->tid, addr);
- SyncVar *s = ctx->metamap.GetIfExistsAndLock(addr, true);
- if (s == 0)
- return;
- if ((flagz & MutexFlagLinkerInit)
- || s->IsFlagSet(MutexFlagLinkerInit)
- || ((flagz & MutexFlagNotStatic) && !s->IsFlagSet(MutexFlagNotStatic))) {
- // Destroy is no-op for linker-initialized mutexes.
- s->mtx.Unlock();
- return;
- }
- if (common_flags()->detect_deadlocks) {
- Callback cb(thr, pc);
- ctx->dd->MutexDestroy(&cb, &s->dd);
- ctx->dd->MutexInit(&cb, &s->dd);
- }
bool unlock_locked = false;
- if (flags()->report_destroy_locked && s->owner_tid != kInvalidTid &&
- !s->IsFlagSet(MutexFlagBroken)) {
- s->SetFlags(MutexFlagBroken);
- unlock_locked = true;
- }
- u64 mid = s->GetId();
- u64 last_lock = s->last_lock;
- if (!unlock_locked)
- s->Reset(thr->proc()); // must not reset it before the report is printed
- s->mtx.Unlock();
- if (unlock_locked && ShouldReport(thr, ReportTypeMutexDestroyLocked)) {
- ThreadRegistryLock l(ctx->thread_registry);
- ScopedReport rep(ReportTypeMutexDestroyLocked);
- rep.AddMutex(mid);
- VarSizeStackTrace trace;
- ObtainCurrentStack(thr, pc, &trace);
- rep.AddStack(trace, true);
- FastState last(last_lock);
- RestoreStack(last.tid(), last.epoch(), &trace, 0);
- rep.AddStack(trace, true);
- rep.AddLocation(addr, 1);
- OutputReport(thr, rep);
-
- SyncVar *s = ctx->metamap.GetIfExistsAndLock(addr, true);
- if (s != 0) {
- s->Reset(thr->proc());
- s->mtx.Unlock();
+ StackID creation_stack_id;
+ FastState last_lock;
+ {
+ auto s = ctx->metamap.GetSyncIfExists(addr);
+ if (!s)
+ return;
+ SlotLocker locker(thr);
+ {
+ Lock lock(&s->mtx);
+ creation_stack_id = s->creation_stack_id;
+ last_lock = s->last_lock;
+ if ((flagz & MutexFlagLinkerInit) || s->IsFlagSet(MutexFlagLinkerInit) ||
+ ((flagz & MutexFlagNotStatic) && !s->IsFlagSet(MutexFlagNotStatic))) {
+ // Destroy is no-op for linker-initialized mutexes.
+ return;
+ }
+ if (common_flags()->detect_deadlocks) {
+ Callback cb(thr, pc);
+ ctx->dd->MutexDestroy(&cb, &s->dd);
+ ctx->dd->MutexInit(&cb, &s->dd);
+ }
+ if (flags()->report_destroy_locked && s->owner_tid != kInvalidTid &&
+ !s->IsFlagSet(MutexFlagBroken)) {
+ s->SetFlags(MutexFlagBroken);
+ unlock_locked = true;
+ }
+ s->Reset();
}
+ // Imitate a memory write to catch unlock-destroy races.
+ if (pc && IsAppMem(addr))
+ MemoryAccess(thr, pc, addr, 1,
+ kAccessWrite | kAccessFree | kAccessSlotLocked);
}
- thr->mset.Remove(mid);
- // Imitate a memory write to catch unlock-destroy races.
- // Do this outside of sync mutex, because it can report a race which locks
- // sync mutexes.
- if (IsAppMem(addr)) {
- CHECK(!thr->is_freeing);
- thr->is_freeing = true;
- MemoryWrite(thr, pc, addr, kSizeLog1);
- thr->is_freeing = false;
- }
+ if (unlock_locked && ShouldReport(thr, ReportTypeMutexDestroyLocked))
+ ReportDestroyLocked(thr, pc, addr, last_lock, creation_stack_id);
+ thr->mset.DelAddr(addr, true);
// s will be destroyed and freed in MetaMap::FreeBlock.
}
-void MutexPreLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) NO_THREAD_SAFETY_ANALYSIS {
+void MutexPreLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
DPrintf("#%d: MutexPreLock %zx flagz=0x%x\n", thr->tid, addr, flagz);
- if (!(flagz & MutexFlagTryLock) && common_flags()->detect_deadlocks) {
- SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, false);
+ if (flagz & MutexFlagTryLock)
+ return;
+ if (!common_flags()->detect_deadlocks)
+ return;
+ Callback cb(thr, pc);
+ {
+ SlotLocker locker(thr);
+ auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
+ ReadLock lock(&s->mtx);
s->UpdateFlags(flagz);
- if (s->owner_tid != thr->tid) {
- Callback cb(thr, pc);
+ if (s->owner_tid != thr->tid)
ctx->dd->MutexBeforeLock(&cb, &s->dd, true);
- s->mtx.ReadUnlock();
- ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
- } else {
- s->mtx.ReadUnlock();
- }
}
+ ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
}
-void MutexPostLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz,
- int rec) NO_THREAD_SAFETY_ANALYSIS {
+void MutexPostLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz, int rec) {
DPrintf("#%d: MutexPostLock %zx flag=0x%x rec=%d\n",
thr->tid, addr, flagz, rec);
if (flagz & MutexFlagRecursiveLock)
CHECK_GT(rec, 0);
else
rec = 1;
- if (IsAppMem(addr))
- MemoryReadAtomic(thr, pc, addr, kSizeLog1);
- SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);
- s->UpdateFlags(flagz);
- thr->fast_state.IncrementEpoch();
- TraceAddEvent(thr, thr->fast_state, EventTypeLock, s->GetId());
+ if (pc && IsAppMem(addr))
+ MemoryAccess(thr, pc, addr, 1, kAccessRead | kAccessAtomic);
bool report_double_lock = false;
- if (s->owner_tid == kInvalidTid) {
- CHECK_EQ(s->recursion, 0);
- s->owner_tid = thr->tid;
- s->last_lock = thr->fast_state.raw();
- } else if (s->owner_tid == thr->tid) {
- CHECK_GT(s->recursion, 0);
- } else if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
- s->SetFlags(MutexFlagBroken);
- report_double_lock = true;
- }
- const bool first = s->recursion == 0;
- s->recursion += rec;
- if (first) {
- AcquireImpl(thr, pc, &s->clock);
- AcquireImpl(thr, pc, &s->read_clock);
- } else if (!s->IsFlagSet(MutexFlagWriteReentrant)) {
- }
- thr->mset.Add(s->GetId(), true, thr->fast_state.epoch());
bool pre_lock = false;
- if (first && common_flags()->detect_deadlocks) {
- pre_lock = (flagz & MutexFlagDoPreLockOnPostLock) &&
- !(flagz & MutexFlagTryLock);
- Callback cb(thr, pc);
- if (pre_lock)
- ctx->dd->MutexBeforeLock(&cb, &s->dd, true);
- ctx->dd->MutexAfterLock(&cb, &s->dd, true, flagz & MutexFlagTryLock);
+ bool first = false;
+ StackID creation_stack_id = kInvalidStackID;
+ {
+ SlotLocker locker(thr);
+ auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
+ creation_stack_id = s->creation_stack_id;
+ RecordMutexLock(thr, pc, addr, creation_stack_id, true);
+ {
+ Lock lock(&s->mtx);
+ first = s->recursion == 0;
+ s->UpdateFlags(flagz);
+ if (s->owner_tid == kInvalidTid) {
+ CHECK_EQ(s->recursion, 0);
+ s->owner_tid = thr->tid;
+ s->last_lock = thr->fast_state;
+ } else if (s->owner_tid == thr->tid) {
+ CHECK_GT(s->recursion, 0);
+ } else if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
+ s->SetFlags(MutexFlagBroken);
+ report_double_lock = true;
+ }
+ s->recursion += rec;
+ if (first) {
+ if (!thr->ignore_sync) {
+ thr->clock.Acquire(s->clock);
+ thr->clock.Acquire(s->read_clock);
+ }
+ }
+ if (first && common_flags()->detect_deadlocks) {
+ pre_lock = (flagz & MutexFlagDoPreLockOnPostLock) &&
+ !(flagz & MutexFlagTryLock);
+ Callback cb(thr, pc);
+ if (pre_lock)
+ ctx->dd->MutexBeforeLock(&cb, &s->dd, true);
+ ctx->dd->MutexAfterLock(&cb, &s->dd, true, flagz & MutexFlagTryLock);
+ }
+ }
}
- u64 mid = s->GetId();
- s->mtx.Unlock();
- // Can't touch s after this point.
- s = 0;
if (report_double_lock)
- ReportMutexMisuse(thr, pc, ReportTypeMutexDoubleLock, addr, mid);
+ ReportMutexMisuse(thr, pc, ReportTypeMutexDoubleLock, addr,
+ creation_stack_id);
if (first && pre_lock && common_flags()->detect_deadlocks) {
Callback cb(thr, pc);
ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
}
}
-int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) NO_THREAD_SAFETY_ANALYSIS {
+int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
DPrintf("#%d: MutexUnlock %zx flagz=0x%x\n", thr->tid, addr, flagz);
- if (IsAppMem(addr))
- MemoryReadAtomic(thr, pc, addr, kSizeLog1);
- SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);
- thr->fast_state.IncrementEpoch();
- TraceAddEvent(thr, thr->fast_state, EventTypeUnlock, s->GetId());
- int rec = 0;
+ if (pc && IsAppMem(addr))
+ MemoryAccess(thr, pc, addr, 1, kAccessRead | kAccessAtomic);
+ StackID creation_stack_id;
+ RecordMutexUnlock(thr, addr);
bool report_bad_unlock = false;
- if (!SANITIZER_GO && (s->recursion == 0 || s->owner_tid != thr->tid)) {
- if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
- s->SetFlags(MutexFlagBroken);
- report_bad_unlock = true;
- }
- } else {
- rec = (flagz & MutexFlagRecursiveUnlock) ? s->recursion : 1;
- s->recursion -= rec;
- if (s->recursion == 0) {
- s->owner_tid = kInvalidTid;
- ReleaseStoreImpl(thr, pc, &s->clock);
- } else {
+ int rec = 0;
+ {
+ SlotLocker locker(thr);
+ auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
+ bool released = false;
+ {
+ Lock lock(&s->mtx);
+ creation_stack_id = s->creation_stack_id;
+ if (!SANITIZER_GO && (s->recursion == 0 || s->owner_tid != thr->tid)) {
+ if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
+ s->SetFlags(MutexFlagBroken);
+ report_bad_unlock = true;
+ }
+ } else {
+ rec = (flagz & MutexFlagRecursiveUnlock) ? s->recursion : 1;
+ s->recursion -= rec;
+ if (s->recursion == 0) {
+ s->owner_tid = kInvalidTid;
+ if (!thr->ignore_sync) {
+ thr->clock.ReleaseStore(&s->clock);
+ released = true;
+ }
+ }
+ }
+ if (common_flags()->detect_deadlocks && s->recursion == 0 &&
+ !report_bad_unlock) {
+ Callback cb(thr, pc);
+ ctx->dd->MutexBeforeUnlock(&cb, &s->dd, true);
+ }
}
+ if (released)
+ IncrementEpoch(thr);
}
- thr->mset.Del(s->GetId(), true);
- if (common_flags()->detect_deadlocks && s->recursion == 0 &&
- !report_bad_unlock) {
- Callback cb(thr, pc);
- ctx->dd->MutexBeforeUnlock(&cb, &s->dd, true);
- }
- u64 mid = s->GetId();
- s->mtx.Unlock();
- // Can't touch s after this point.
if (report_bad_unlock)
- ReportMutexMisuse(thr, pc, ReportTypeMutexBadUnlock, addr, mid);
+ ReportMutexMisuse(thr, pc, ReportTypeMutexBadUnlock, addr,
+ creation_stack_id);
if (common_flags()->detect_deadlocks && !report_bad_unlock) {
Callback cb(thr, pc);
ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
return rec;
}
-void MutexPreReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) NO_THREAD_SAFETY_ANALYSIS {
+void MutexPreReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
DPrintf("#%d: MutexPreReadLock %zx flagz=0x%x\n", thr->tid, addr, flagz);
- if (!(flagz & MutexFlagTryLock) && common_flags()->detect_deadlocks) {
- SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, false);
+ if ((flagz & MutexFlagTryLock) || !common_flags()->detect_deadlocks)
+ return;
+ Callback cb(thr, pc);
+ {
+ SlotLocker locker(thr);
+ auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
+ ReadLock lock(&s->mtx);
s->UpdateFlags(flagz);
- Callback cb(thr, pc);
ctx->dd->MutexBeforeLock(&cb, &s->dd, false);
- s->mtx.ReadUnlock();
- ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
}
+ ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
}
-void MutexPostReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) NO_THREAD_SAFETY_ANALYSIS {
+void MutexPostReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
DPrintf("#%d: MutexPostReadLock %zx flagz=0x%x\n", thr->tid, addr, flagz);
- if (IsAppMem(addr))
- MemoryReadAtomic(thr, pc, addr, kSizeLog1);
- SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, false);
- s->UpdateFlags(flagz);
- thr->fast_state.IncrementEpoch();
- TraceAddEvent(thr, thr->fast_state, EventTypeRLock, s->GetId());
+ if (pc && IsAppMem(addr))
+ MemoryAccess(thr, pc, addr, 1, kAccessRead | kAccessAtomic);
bool report_bad_lock = false;
- if (s->owner_tid != kInvalidTid) {
- if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
- s->SetFlags(MutexFlagBroken);
- report_bad_lock = true;
- }
- }
- AcquireImpl(thr, pc, &s->clock);
- s->last_lock = thr->fast_state.raw();
- thr->mset.Add(s->GetId(), false, thr->fast_state.epoch());
bool pre_lock = false;
- if (common_flags()->detect_deadlocks) {
- pre_lock = (flagz & MutexFlagDoPreLockOnPostLock) &&
- !(flagz & MutexFlagTryLock);
- Callback cb(thr, pc);
- if (pre_lock)
- ctx->dd->MutexBeforeLock(&cb, &s->dd, false);
- ctx->dd->MutexAfterLock(&cb, &s->dd, false, flagz & MutexFlagTryLock);
+ StackID creation_stack_id = kInvalidStackID;
+ {
+ SlotLocker locker(thr);
+ auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
+ creation_stack_id = s->creation_stack_id;
+ RecordMutexLock(thr, pc, addr, creation_stack_id, false);
+ {
+ ReadLock lock(&s->mtx);
+ s->UpdateFlags(flagz);
+ if (s->owner_tid != kInvalidTid) {
+ if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
+ s->SetFlags(MutexFlagBroken);
+ report_bad_lock = true;
+ }
+ }
+ if (!thr->ignore_sync)
+ thr->clock.Acquire(s->clock);
+ s->last_lock = thr->fast_state;
+ if (common_flags()->detect_deadlocks) {
+ pre_lock = (flagz & MutexFlagDoPreLockOnPostLock) &&
+ !(flagz & MutexFlagTryLock);
+ Callback cb(thr, pc);
+ if (pre_lock)
+ ctx->dd->MutexBeforeLock(&cb, &s->dd, false);
+ ctx->dd->MutexAfterLock(&cb, &s->dd, false, flagz & MutexFlagTryLock);
+ }
+ }
}
- u64 mid = s->GetId();
- s->mtx.ReadUnlock();
- // Can't touch s after this point.
- s = 0;
if (report_bad_lock)
- ReportMutexMisuse(thr, pc, ReportTypeMutexBadReadLock, addr, mid);
+ ReportMutexMisuse(thr, pc, ReportTypeMutexBadReadLock, addr,
+ creation_stack_id);
if (pre_lock && common_flags()->detect_deadlocks) {
Callback cb(thr, pc);
ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
}
}
-void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr) NO_THREAD_SAFETY_ANALYSIS {
+void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr) {
DPrintf("#%d: MutexReadUnlock %zx\n", thr->tid, addr);
- if (IsAppMem(addr))
- MemoryReadAtomic(thr, pc, addr, kSizeLog1);
- SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);
- thr->fast_state.IncrementEpoch();
- TraceAddEvent(thr, thr->fast_state, EventTypeRUnlock, s->GetId());
+ if (pc && IsAppMem(addr))
+ MemoryAccess(thr, pc, addr, 1, kAccessRead | kAccessAtomic);
+ RecordMutexUnlock(thr, addr);
+ StackID creation_stack_id;
bool report_bad_unlock = false;
- if (s->owner_tid != kInvalidTid) {
- if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
- s->SetFlags(MutexFlagBroken);
- report_bad_unlock = true;
+ {
+ SlotLocker locker(thr);
+ auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
+ bool released = false;
+ {
+ Lock lock(&s->mtx);
+ creation_stack_id = s->creation_stack_id;
+ if (s->owner_tid != kInvalidTid) {
+ if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
+ s->SetFlags(MutexFlagBroken);
+ report_bad_unlock = true;
+ }
+ }
+ if (!thr->ignore_sync) {
+ thr->clock.Release(&s->read_clock);
+ released = true;
+ }
+ if (common_flags()->detect_deadlocks && s->recursion == 0) {
+ Callback cb(thr, pc);
+ ctx->dd->MutexBeforeUnlock(&cb, &s->dd, false);
+ }
}
+ if (released)
+ IncrementEpoch(thr);
}
- ReleaseImpl(thr, pc, &s->read_clock);
- if (common_flags()->detect_deadlocks && s->recursion == 0) {
- Callback cb(thr, pc);
- ctx->dd->MutexBeforeUnlock(&cb, &s->dd, false);
- }
- u64 mid = s->GetId();
- s->mtx.Unlock();
- // Can't touch s after this point.
- thr->mset.Del(mid, false);
if (report_bad_unlock)
- ReportMutexMisuse(thr, pc, ReportTypeMutexBadReadUnlock, addr, mid);
+ ReportMutexMisuse(thr, pc, ReportTypeMutexBadReadUnlock, addr,
+ creation_stack_id);
if (common_flags()->detect_deadlocks) {
Callback cb(thr, pc);
ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
}
}
-void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr) NO_THREAD_SAFETY_ANALYSIS {
+void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr) {
DPrintf("#%d: MutexReadOrWriteUnlock %zx\n", thr->tid, addr);
- if (IsAppMem(addr))
- MemoryReadAtomic(thr, pc, addr, kSizeLog1);
- SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);
- bool write = true;
+ if (pc && IsAppMem(addr))
+ MemoryAccess(thr, pc, addr, 1, kAccessRead | kAccessAtomic);
+ RecordMutexUnlock(thr, addr);
+ StackID creation_stack_id;
bool report_bad_unlock = false;
- if (s->owner_tid == kInvalidTid) {
- // Seems to be read unlock.
- write = false;
- thr->fast_state.IncrementEpoch();
- TraceAddEvent(thr, thr->fast_state, EventTypeRUnlock, s->GetId());
- ReleaseImpl(thr, pc, &s->read_clock);
- } else if (s->owner_tid == thr->tid) {
- // Seems to be write unlock.
- thr->fast_state.IncrementEpoch();
- TraceAddEvent(thr, thr->fast_state, EventTypeUnlock, s->GetId());
- CHECK_GT(s->recursion, 0);
- s->recursion--;
- if (s->recursion == 0) {
- s->owner_tid = kInvalidTid;
- ReleaseStoreImpl(thr, pc, &s->clock);
- } else {
+ bool write = true;
+ {
+ SlotLocker locker(thr);
+ auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
+ bool released = false;
+ {
+ Lock lock(&s->mtx);
+ creation_stack_id = s->creation_stack_id;
+ if (s->owner_tid == kInvalidTid) {
+ // Seems to be read unlock.
+ write = false;
+ if (!thr->ignore_sync) {
+ thr->clock.Release(&s->read_clock);
+ released = true;
+ }
+ } else if (s->owner_tid == thr->tid) {
+ // Seems to be write unlock.
+ CHECK_GT(s->recursion, 0);
+ s->recursion--;
+ if (s->recursion == 0) {
+ s->owner_tid = kInvalidTid;
+ if (!thr->ignore_sync) {
+ thr->clock.ReleaseStore(&s->clock);
+ released = true;
+ }
+ }
+ } else if (!s->IsFlagSet(MutexFlagBroken)) {
+ s->SetFlags(MutexFlagBroken);
+ report_bad_unlock = true;
+ }
+ if (common_flags()->detect_deadlocks && s->recursion == 0) {
+ Callback cb(thr, pc);
+ ctx->dd->MutexBeforeUnlock(&cb, &s->dd, write);
+ }
}
- } else if (!s->IsFlagSet(MutexFlagBroken)) {
- s->SetFlags(MutexFlagBroken);
- report_bad_unlock = true;
+ if (released)
+ IncrementEpoch(thr);
}
- thr->mset.Del(s->GetId(), write);
- if (common_flags()->detect_deadlocks && s->recursion == 0) {
- Callback cb(thr, pc);
- ctx->dd->MutexBeforeUnlock(&cb, &s->dd, write);
- }
- u64 mid = s->GetId();
- s->mtx.Unlock();
- // Can't touch s after this point.
if (report_bad_unlock)
- ReportMutexMisuse(thr, pc, ReportTypeMutexBadUnlock, addr, mid);
+ ReportMutexMisuse(thr, pc, ReportTypeMutexBadUnlock, addr,
+ creation_stack_id);
if (common_flags()->detect_deadlocks) {
Callback cb(thr, pc);
ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
}
}
-void MutexRepair(ThreadState *thr, uptr pc, uptr addr) NO_THREAD_SAFETY_ANALYSIS {
+void MutexRepair(ThreadState *thr, uptr pc, uptr addr) {
DPrintf("#%d: MutexRepair %zx\n", thr->tid, addr);
- SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);
+ SlotLocker locker(thr);
+ auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
+ Lock lock(&s->mtx);
s->owner_tid = kInvalidTid;
s->recursion = 0;
- s->mtx.Unlock();
}
-void MutexInvalidAccess(ThreadState *thr, uptr pc, uptr addr) NO_THREAD_SAFETY_ANALYSIS {
+void MutexInvalidAccess(ThreadState *thr, uptr pc, uptr addr) {
DPrintf("#%d: MutexInvalidAccess %zx\n", thr->tid, addr);
- SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);
- u64 mid = s->GetId();
- s->mtx.Unlock();
- ReportMutexMisuse(thr, pc, ReportTypeMutexInvalidAccess, addr, mid);
+ StackID creation_stack_id = kInvalidStackID;
+ {
+ SlotLocker locker(thr);
+ auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
+ if (s)
+ creation_stack_id = s->creation_stack_id;
+ }
+ ReportMutexMisuse(thr, pc, ReportTypeMutexInvalidAccess, addr,
+ creation_stack_id);
}
-void Acquire(ThreadState *thr, uptr pc, uptr addr) NO_THREAD_SAFETY_ANALYSIS {
+void Acquire(ThreadState *thr, uptr pc, uptr addr) {
DPrintf("#%d: Acquire %zx\n", thr->tid, addr);
if (thr->ignore_sync)
return;
- SyncVar *s = ctx->metamap.GetIfExistsAndLock(addr, false);
+ auto s = ctx->metamap.GetSyncIfExists(addr);
if (!s)
return;
- AcquireImpl(thr, pc, &s->clock);
- s->mtx.ReadUnlock();
-}
-
-static void UpdateClockCallback(ThreadContextBase *tctx_base, void *arg) {
- ThreadState *thr = reinterpret_cast<ThreadState*>(arg);
- ThreadContext *tctx = static_cast<ThreadContext*>(tctx_base);
- u64 epoch = tctx->epoch1;
- if (tctx->status == ThreadStatusRunning) {
- epoch = tctx->thr->fast_state.epoch();
- tctx->thr->clock.NoteGlobalAcquire(epoch);
- }
- thr->clock.set(&thr->proc()->clock_cache, tctx->tid, epoch);
+ SlotLocker locker(thr);
+ if (!s->clock)
+ return;
+ ReadLock lock(&s->mtx);
+ thr->clock.Acquire(s->clock);
}
-void AcquireGlobal(ThreadState *thr, uptr pc) {
+void AcquireGlobal(ThreadState *thr) {
DPrintf("#%d: AcquireGlobal\n", thr->tid);
if (thr->ignore_sync)
return;
- ThreadRegistryLock l(ctx->thread_registry);
- ctx->thread_registry->RunCallbackForEachThreadLocked(
- UpdateClockCallback, thr);
+ SlotLocker locker(thr);
+ for (auto &slot : ctx->slots) thr->clock.Set(slot.sid, slot.epoch());
}
-void ReleaseStoreAcquire(ThreadState *thr, uptr pc, uptr addr) NO_THREAD_SAFETY_ANALYSIS {
- DPrintf("#%d: ReleaseStoreAcquire %zx\n", thr->tid, addr);
+void Release(ThreadState *thr, uptr pc, uptr addr) {
+ DPrintf("#%d: Release %zx\n", thr->tid, addr);
if (thr->ignore_sync)
return;
- SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);
- thr->fast_state.IncrementEpoch();
- // Can't increment epoch w/o writing to the trace as well.
- TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
- ReleaseStoreAcquireImpl(thr, pc, &s->clock);
- s->mtx.Unlock();
+ SlotLocker locker(thr);
+ {
+ auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, false);
+ Lock lock(&s->mtx);
+ thr->clock.Release(&s->clock);
+ }
+ IncrementEpoch(thr);
}
-void Release(ThreadState *thr, uptr pc, uptr addr) NO_THREAD_SAFETY_ANALYSIS {
- DPrintf("#%d: Release %zx\n", thr->tid, addr);
+void ReleaseStore(ThreadState *thr, uptr pc, uptr addr) {
+ DPrintf("#%d: ReleaseStore %zx\n", thr->tid, addr);
if (thr->ignore_sync)
return;
- SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);
- thr->fast_state.IncrementEpoch();
- // Can't increment epoch w/o writing to the trace as well.
- TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
- ReleaseImpl(thr, pc, &s->clock);
- s->mtx.Unlock();
+ SlotLocker locker(thr);
+ {
+ auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, false);
+ Lock lock(&s->mtx);
+ thr->clock.ReleaseStore(&s->clock);
+ }
+ IncrementEpoch(thr);
}
-void ReleaseStore(ThreadState *thr, uptr pc, uptr addr) NO_THREAD_SAFETY_ANALYSIS {
- DPrintf("#%d: ReleaseStore %zx\n", thr->tid, addr);
+void ReleaseStoreAcquire(ThreadState *thr, uptr pc, uptr addr) {
+ DPrintf("#%d: ReleaseStoreAcquire %zx\n", thr->tid, addr);
if (thr->ignore_sync)
return;
- SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);
- thr->fast_state.IncrementEpoch();
- // Can't increment epoch w/o writing to the trace as well.
- TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
- ReleaseStoreImpl(thr, pc, &s->clock);
- s->mtx.Unlock();
+ SlotLocker locker(thr);
+ {
+ auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, false);
+ Lock lock(&s->mtx);
+ thr->clock.ReleaseStoreAcquire(&s->clock);
+ }
+ IncrementEpoch(thr);
}
-#if !SANITIZER_GO
-static void UpdateSleepClockCallback(ThreadContextBase *tctx_base, void *arg) {
- ThreadState *thr = reinterpret_cast<ThreadState*>(arg);
- ThreadContext *tctx = static_cast<ThreadContext*>(tctx_base);
- u64 epoch = tctx->epoch1;
- if (tctx->status == ThreadStatusRunning)
- epoch = tctx->thr->fast_state.epoch();
- thr->last_sleep_clock.set(&thr->proc()->clock_cache, tctx->tid, epoch);
+void IncrementEpoch(ThreadState *thr) {
+ DCHECK(!thr->ignore_sync);
+ DCHECK(thr->slot_locked);
+ Epoch epoch = EpochInc(thr->fast_state.epoch());
+ if (!EpochOverflow(epoch)) {
+ Sid sid = thr->fast_state.sid();
+ thr->clock.Set(sid, epoch);
+ thr->fast_state.SetEpoch(epoch);
+ thr->slot->SetEpoch(epoch);
+ TraceTime(thr);
+ }
}
+#if !SANITIZER_GO
void AfterSleep(ThreadState *thr, uptr pc) {
- DPrintf("#%d: AfterSleep %zx\n", thr->tid);
+ DPrintf("#%d: AfterSleep\n", thr->tid);
if (thr->ignore_sync)
return;
thr->last_sleep_stack_id = CurrentStackId(thr, pc);
- ThreadRegistryLock l(ctx->thread_registry);
- ctx->thread_registry->RunCallbackForEachThreadLocked(
- UpdateSleepClockCallback, thr);
+ thr->last_sleep_clock.Reset();
+ SlotLocker locker(thr);
+ for (auto &slot : ctx->slots)
+ thr->last_sleep_clock.Set(slot.sid, slot.epoch());
}
#endif
-void AcquireImpl(ThreadState *thr, uptr pc, SyncClock *c) {
- if (thr->ignore_sync)
- return;
- thr->clock.set(thr->fast_state.epoch());
- thr->clock.acquire(&thr->proc()->clock_cache, c);
-}
-
-void ReleaseStoreAcquireImpl(ThreadState *thr, uptr pc, SyncClock *c) {
- if (thr->ignore_sync)
- return;
- thr->clock.set(thr->fast_state.epoch());
- thr->fast_synch_epoch = thr->fast_state.epoch();
- thr->clock.releaseStoreAcquire(&thr->proc()->clock_cache, c);
-}
-
-void ReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c) {
- if (thr->ignore_sync)
- return;
- thr->clock.set(thr->fast_state.epoch());
- thr->fast_synch_epoch = thr->fast_state.epoch();
- thr->clock.release(&thr->proc()->clock_cache, c);
-}
-
-void ReleaseStoreImpl(ThreadState *thr, uptr pc, SyncClock *c) {
- if (thr->ignore_sync)
- return;
- thr->clock.set(thr->fast_state.epoch());
- thr->fast_synch_epoch = thr->fast_state.epoch();
- thr->clock.ReleaseStore(&thr->proc()->clock_cache, c);
-}
-
-void AcquireReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c) {
- if (thr->ignore_sync)
- return;
- thr->clock.set(thr->fast_state.epoch());
- thr->fast_synch_epoch = thr->fast_state.epoch();
- thr->clock.acq_rel(&thr->proc()->clock_cache, c);
-}
-
void ReportDeadlock(ThreadState *thr, uptr pc, DDReport *r) {
if (r == 0 || !ShouldReport(thr, ReportTypeDeadlock))
return;
- ThreadRegistryLock l(ctx->thread_registry);
+ ThreadRegistryLock l(&ctx->thread_registry);
ScopedReport rep(ReportTypeDeadlock);
for (int i = 0; i < r->n; i++) {
- rep.AddMutex(r->loop[i].mtx_ctx0);
+ rep.AddMutex(r->loop[i].mtx_ctx0, r->loop[i].stk[0]);
rep.AddUniqueTid((int)r->loop[i].thr_ctx);
rep.AddThread((int)r->loop[i].thr_ctx);
}
for (int i = 0; i < r->n; i++) {
for (int j = 0; j < (flags()->second_deadlock_stack ? 2 : 1); j++) {
u32 stk = r->loop[i].stk[j];
- if (stk && stk != 0xffffffff) {
+ if (stk && stk != kInvalidStackID) {
rep.AddStack(StackDepotGet(stk), true);
} else {
// Sometimes we fail to extract the stack trace (FIXME: investigate),
OutputReport(thr, rep);
}
+void ReportDestroyLocked(ThreadState *thr, uptr pc, uptr addr,
+ FastState last_lock, StackID creation_stack_id) {
+ // We need to lock the slot during RestoreStack because it protects
+ // the slot journal.
+ Lock slot_lock(&ctx->slots[static_cast<uptr>(last_lock.sid())].mtx);
+ ThreadRegistryLock l0(&ctx->thread_registry);
+ Lock slots_lock(&ctx->slot_mtx);
+ ScopedReport rep(ReportTypeMutexDestroyLocked);
+ rep.AddMutex(addr, creation_stack_id);
+ VarSizeStackTrace trace;
+ ObtainCurrentStack(thr, pc, &trace);
+ rep.AddStack(trace, true);
+
+ Tid tid;
+ DynamicMutexSet mset;
+ uptr tag;
+ if (!RestoreStack(EventType::kLock, last_lock.sid(), last_lock.epoch(), addr,
+ 0, kAccessWrite, &tid, &trace, mset, &tag))
+ return;
+ rep.AddStack(trace, true);
+ rep.AddLocation(addr, 1);
+ OutputReport(thr, rep);
+}
+
} // namespace __tsan
#if !SANITIZER_GO
AllocatorProcFinish(proc);
#endif
- ctx->clock_alloc.FlushCache(&proc->clock_cache);
ctx->metamap.OnProcIdle(proc);
if (common_flags()->detect_deadlocks)
ctx->dd->DestroyPhysicalThread(proc->dd_pt);
} else if (last && 0 == internal_strcmp(last, "__tsan_thread_start_func")) {
last_frame->ClearAll();
last_frame2->next = nullptr;
- // Strip global ctors init.
- } else if (last && 0 == internal_strcmp(last, "__do_global_ctors_aux")) {
+ // Strip global ctors init, .preinit_array and main caller.
+ } else if (last && (0 == internal_strcmp(last, "__do_global_ctors_aux") ||
+ 0 == internal_strcmp(last, "__libc_csu_init") ||
+ 0 == internal_strcmp(last, "__libc_start_main"))) {
last_frame->ClearAll();
last_frame2->next = nullptr;
// If both are 0, then we probably just failed to symbolize.
}
StackStripMain(top);
- ReportStack *stack = ReportStack::New();
+ auto *stack = New<ReportStack>();
stack->frames = top;
return stack;
}
CheckedMutex::CheckNoLocks();
// For the same reason check we didn't lock thread_registry yet.
if (SANITIZER_DEBUG)
- ThreadRegistryLock l(ctx->thread_registry);
+ ThreadRegistryLock l(&ctx->thread_registry);
if (!flags()->report_bugs || thr->suppress_reports)
return false;
switch (typ) {
}
ScopedReportBase::ScopedReportBase(ReportType typ, uptr tag) {
- ctx->thread_registry->CheckLocked();
- void *mem = internal_alloc(MBlockReport, sizeof(ReportDesc));
- rep_ = new(mem) ReportDesc;
+ ctx->thread_registry.CheckLocked();
+ rep_ = New<ReportDesc>();
rep_->typ = typ;
rep_->tag = tag;
ctx->report_mtx.Lock();
ScopedReportBase::~ScopedReportBase() {
ctx->report_mtx.Unlock();
DestroyAndFree(rep_);
- rep_ = nullptr;
}
void ScopedReportBase::AddStack(StackTrace stack, bool suppressable) {
}
void ScopedReportBase::AddMemoryAccess(uptr addr, uptr external_tag, Shadow s,
- StackTrace stack, const MutexSet *mset) {
- void *mem = internal_alloc(MBlockReportMop, sizeof(ReportMop));
- ReportMop *mop = new(mem) ReportMop;
+ Tid tid, StackTrace stack,
+ const MutexSet *mset) {
+ uptr addr0, size;
+ AccessType typ;
+ s.GetAccess(&addr0, &size, &typ);
+ auto *mop = New<ReportMop>();
rep_->mops.PushBack(mop);
- mop->tid = s.tid();
- mop->addr = addr + s.addr0();
- mop->size = s.size();
- mop->write = s.IsWrite();
- mop->atomic = s.IsAtomic();
+ mop->tid = tid;
+ mop->addr = addr + addr0;
+ mop->size = size;
+ mop->write = !(typ & kAccessRead);
+ mop->atomic = typ & kAccessAtomic;
mop->stack = SymbolizeStack(stack);
mop->external_tag = external_tag;
if (mop->stack)
mop->stack->suppressable = true;
for (uptr i = 0; i < mset->Size(); i++) {
MutexSet::Desc d = mset->Get(i);
- u64 mid = this->AddMutex(d.id);
- ReportMopMutex mtx = {mid, d.write};
+ int id = this->AddMutex(d.addr, d.stack_id);
+ ReportMopMutex mtx = {id, d.write};
mop->mset.PushBack(mtx);
}
}
-void ScopedReportBase::AddUniqueTid(int unique_tid) {
+void ScopedReportBase::AddUniqueTid(Tid unique_tid) {
rep_->unique_tids.PushBack(unique_tid);
}
if ((u32)rep_->threads[i]->id == tctx->tid)
return;
}
- void *mem = internal_alloc(MBlockReportThread, sizeof(ReportThread));
- ReportThread *rt = new(mem) ReportThread;
+ auto *rt = New<ReportThread>();
rep_->threads.PushBack(rt);
rt->id = tctx->tid;
rt->os_id = tctx->os_id;
}
#if !SANITIZER_GO
-static bool FindThreadByUidLockedCallback(ThreadContextBase *tctx, void *arg) {
- int unique_id = *(int *)arg;
- return tctx->unique_id == (u32)unique_id;
-}
-
-static ThreadContext *FindThreadByUidLocked(int unique_id) {
- ctx->thread_registry->CheckLocked();
+static ThreadContext *FindThreadByTidLocked(Tid tid) {
+ ctx->thread_registry.CheckLocked();
return static_cast<ThreadContext *>(
- ctx->thread_registry->FindThreadContextLocked(
- FindThreadByUidLockedCallback, &unique_id));
-}
-
-static ThreadContext *FindThreadByTidLocked(int tid) {
- ctx->thread_registry->CheckLocked();
- return static_cast<ThreadContext*>(
- ctx->thread_registry->GetThreadLocked(tid));
+ ctx->thread_registry.GetThreadLocked(tid));
}
static bool IsInStackOrTls(ThreadContextBase *tctx_base, void *arg) {
}
ThreadContext *IsThreadStackOrTls(uptr addr, bool *is_stack) {
- ctx->thread_registry->CheckLocked();
- ThreadContext *tctx = static_cast<ThreadContext*>(
- ctx->thread_registry->FindThreadContextLocked(IsInStackOrTls,
- (void*)addr));
+ ctx->thread_registry.CheckLocked();
+ ThreadContext *tctx =
+ static_cast<ThreadContext *>(ctx->thread_registry.FindThreadContextLocked(
+ IsInStackOrTls, (void *)addr));
if (!tctx)
return 0;
ThreadState *thr = tctx->thr;
}
#endif
-void ScopedReportBase::AddThread(int unique_tid, bool suppressable) {
+void ScopedReportBase::AddThread(Tid tid, bool suppressable) {
#if !SANITIZER_GO
- if (const ThreadContext *tctx = FindThreadByUidLocked(unique_tid))
+ if (const ThreadContext *tctx = FindThreadByTidLocked(tid))
AddThread(tctx, suppressable);
#endif
}
-void ScopedReportBase::AddMutex(const SyncVar *s) {
+int ScopedReportBase::AddMutex(uptr addr, StackID creation_stack_id) {
for (uptr i = 0; i < rep_->mutexes.Size(); i++) {
- if (rep_->mutexes[i]->id == s->uid)
- return;
+ if (rep_->mutexes[i]->addr == addr)
+ return rep_->mutexes[i]->id;
}
- void *mem = internal_alloc(MBlockReportMutex, sizeof(ReportMutex));
- ReportMutex *rm = new(mem) ReportMutex;
+ auto *rm = New<ReportMutex>();
rep_->mutexes.PushBack(rm);
- rm->id = s->uid;
- rm->addr = s->addr;
- rm->destroyed = false;
- rm->stack = SymbolizeStackId(s->creation_stack_id);
-}
-
-u64 ScopedReportBase::AddMutex(u64 id) NO_THREAD_SAFETY_ANALYSIS {
- u64 uid = 0;
- u64 mid = id;
- uptr addr = SyncVar::SplitId(id, &uid);
- SyncVar *s = ctx->metamap.GetIfExistsAndLock(addr, true);
- // Check that the mutex is still alive.
- // Another mutex can be created at the same address,
- // so check uid as well.
- if (s && s->CheckId(uid)) {
- mid = s->uid;
- AddMutex(s);
- } else {
- AddDeadMutex(id);
- }
- if (s)
- s->mtx.Unlock();
- return mid;
-}
-
-void ScopedReportBase::AddDeadMutex(u64 id) {
- for (uptr i = 0; i < rep_->mutexes.Size(); i++) {
- if (rep_->mutexes[i]->id == id)
- return;
- }
- void *mem = internal_alloc(MBlockReportMutex, sizeof(ReportMutex));
- ReportMutex *rm = new(mem) ReportMutex;
- rep_->mutexes.PushBack(rm);
- rm->id = id;
- rm->addr = 0;
- rm->destroyed = true;
- rm->stack = 0;
+ rm->id = rep_->mutexes.Size() - 1;
+ rm->addr = addr;
+ rm->stack = SymbolizeStackId(creation_stack_id);
+ return rm->id;
}
void ScopedReportBase::AddLocation(uptr addr, uptr size) {
return;
#if !SANITIZER_GO
int fd = -1;
- int creat_tid = kInvalidTid;
- u32 creat_stack = 0;
- if (FdLocation(addr, &fd, &creat_tid, &creat_stack)) {
- ReportLocation *loc = ReportLocation::New(ReportLocationFD);
+ Tid creat_tid = kInvalidTid;
+ StackID creat_stack = 0;
+ bool closed = false;
+ if (FdLocation(addr, &fd, &creat_tid, &creat_stack, &closed)) {
+ auto *loc = New<ReportLocation>();
+ loc->type = ReportLocationFD;
+ loc->fd_closed = closed;
loc->fd = fd;
loc->tid = creat_tid;
loc->stack = SymbolizeStackId(creat_stack);
rep_->locs.PushBack(loc);
- ThreadContext *tctx = FindThreadByUidLocked(creat_tid);
- if (tctx)
- AddThread(tctx);
+ AddThread(creat_tid);
return;
}
MBlock *b = 0;
+ uptr block_begin = 0;
Allocator *a = allocator();
if (a->PointerIsMine((void*)addr)) {
- void *block_begin = a->GetBlockBegin((void*)addr);
+ block_begin = (uptr)a->GetBlockBegin((void *)addr);
if (block_begin)
- b = ctx->metamap.GetBlock((uptr)block_begin);
+ b = ctx->metamap.GetBlock(block_begin);
}
+ if (!b)
+ b = JavaHeapBlock(addr, &block_begin);
if (b != 0) {
- ThreadContext *tctx = FindThreadByTidLocked(b->tid);
- ReportLocation *loc = ReportLocation::New(ReportLocationHeap);
- loc->heap_chunk_start = (uptr)allocator()->GetBlockBegin((void *)addr);
+ auto *loc = New<ReportLocation>();
+ loc->type = ReportLocationHeap;
+ loc->heap_chunk_start = block_begin;
loc->heap_chunk_size = b->siz;
loc->external_tag = b->tag;
- loc->tid = tctx ? tctx->tid : b->tid;
+ loc->tid = b->tid;
loc->stack = SymbolizeStackId(b->stk);
rep_->locs.PushBack(loc);
- if (tctx)
- AddThread(tctx);
+ AddThread(b->tid);
return;
}
bool is_stack = false;
if (ThreadContext *tctx = IsThreadStackOrTls(addr, &is_stack)) {
- ReportLocation *loc =
- ReportLocation::New(is_stack ? ReportLocationStack : ReportLocationTLS);
+ auto *loc = New<ReportLocation>();
+ loc->type = is_stack ? ReportLocationStack : ReportLocationTLS;
loc->tid = tctx->tid;
rep_->locs.PushBack(loc);
AddThread(tctx);
}
#if !SANITIZER_GO
-void ScopedReportBase::AddSleep(u32 stack_id) {
+void ScopedReportBase::AddSleep(StackID stack_id) {
rep_->sleep = SymbolizeStackId(stack_id);
}
#endif
void ScopedReportBase::SetCount(int count) { rep_->count = count; }
+void ScopedReportBase::SetSigNum(int sig) { rep_->signum = sig; }
+
const ReportDesc *ScopedReportBase::GetReport() const { return rep_; }
ScopedReport::ScopedReport(ReportType typ, uptr tag)
ScopedReport::~ScopedReport() {}
-void RestoreStack(int tid, const u64 epoch, VarSizeStackTrace *stk,
- MutexSet *mset, uptr *tag) {
+// Replays the trace up to last_pos position in the last part
+// or up to the provided epoch/sid (whichever is earlier)
+// and calls the provided function f for each event.
+template <typename Func>
+void TraceReplay(Trace *trace, TracePart *last, Event *last_pos, Sid sid,
+ Epoch epoch, Func f) {
+ TracePart *part = trace->parts.Front();
+ Sid ev_sid = kFreeSid;
+ Epoch ev_epoch = kEpochOver;
+ for (;;) {
+ DCHECK_EQ(part->trace, trace);
+ // Note: an event can't start in the last element.
+ // Since an event can take up to 2 elements,
+ // we ensure we have at least 2 before adding an event.
+ Event *end = &part->events[TracePart::kSize - 1];
+ if (part == last)
+ end = last_pos;
+ f(kFreeSid, kEpochOver, nullptr); // notify about part start
+ for (Event *evp = &part->events[0]; evp < end; evp++) {
+ Event *evp0 = evp;
+ if (!evp->is_access && !evp->is_func) {
+ switch (evp->type) {
+ case EventType::kTime: {
+ auto *ev = reinterpret_cast<EventTime *>(evp);
+ ev_sid = static_cast<Sid>(ev->sid);
+ ev_epoch = static_cast<Epoch>(ev->epoch);
+ if (ev_sid == sid && ev_epoch > epoch)
+ return;
+ break;
+ }
+ case EventType::kAccessExt:
+ FALLTHROUGH;
+ case EventType::kAccessRange:
+ FALLTHROUGH;
+ case EventType::kLock:
+ FALLTHROUGH;
+ case EventType::kRLock:
+ // These take 2 Event elements.
+ evp++;
+ break;
+ case EventType::kUnlock:
+ // This takes 1 Event element.
+ break;
+ }
+ }
+ CHECK_NE(ev_sid, kFreeSid);
+ CHECK_NE(ev_epoch, kEpochOver);
+ f(ev_sid, ev_epoch, evp0);
+ }
+ if (part == last)
+ return;
+ part = trace->parts.Next(part);
+ CHECK(part);
+ }
+ CHECK(0);
+}
+
+static void RestoreStackMatch(VarSizeStackTrace *pstk, MutexSet *pmset,
+ Vector<uptr> *stack, MutexSet *mset, uptr pc,
+ bool *found) {
+ DPrintf2(" MATCHED\n");
+ *pmset = *mset;
+ stack->PushBack(pc);
+ pstk->Init(&(*stack)[0], stack->Size());
+ stack->PopBack();
+ *found = true;
+}
+
+// Checks if addr1|size1 is fully contained in addr2|size2.
+// We check for fully contained instread of just overlapping
+// because a memory access is always traced once, but can be
+// split into multiple accesses in the shadow.
+static constexpr bool IsWithinAccess(uptr addr1, uptr size1, uptr addr2,
+ uptr size2) {
+ return addr1 >= addr2 && addr1 + size1 <= addr2 + size2;
+}
+
+// Replays the trace of slot sid up to the target event identified
+// by epoch/addr/size/typ and restores and returns tid, stack, mutex set
+// and tag for that event. If there are multiple such events, it returns
+// the last one. Returns false if the event is not present in the trace.
+bool RestoreStack(EventType type, Sid sid, Epoch epoch, uptr addr, uptr size,
+ AccessType typ, Tid *ptid, VarSizeStackTrace *pstk,
+ MutexSet *pmset, uptr *ptag) {
// This function restores stack trace and mutex set for the thread/epoch.
// It does so by getting stack trace and mutex set at the beginning of
// trace part, and then replaying the trace till the given epoch.
- Trace* trace = ThreadTrace(tid);
- ReadLock l(&trace->mtx);
- const int partidx = (epoch / kTracePartSize) % TraceParts();
- TraceHeader* hdr = &trace->headers[partidx];
- if (epoch < hdr->epoch0 || epoch >= hdr->epoch0 + kTracePartSize)
- return;
- CHECK_EQ(RoundDown(epoch, kTracePartSize), hdr->epoch0);
- const u64 epoch0 = RoundDown(epoch, TraceSize());
- const u64 eend = epoch % TraceSize();
- const u64 ebegin = RoundDown(eend, kTracePartSize);
- DPrintf("#%d: RestoreStack epoch=%zu ebegin=%zu eend=%zu partidx=%d\n",
- tid, (uptr)epoch, (uptr)ebegin, (uptr)eend, partidx);
- Vector<uptr> stack;
- stack.Resize(hdr->stack0.size + 64);
- for (uptr i = 0; i < hdr->stack0.size; i++) {
- stack[i] = hdr->stack0.trace[i];
- DPrintf2(" #%02zu: pc=%zx\n", i, stack[i]);
- }
- if (mset)
- *mset = hdr->mset0;
- uptr pos = hdr->stack0.size;
- Event *events = (Event*)GetThreadTrace(tid);
- for (uptr i = ebegin; i <= eend; i++) {
- Event ev = events[i];
- EventType typ = (EventType)(ev >> kEventPCBits);
- uptr pc = (uptr)(ev & ((1ull << kEventPCBits) - 1));
- DPrintf2(" %zu typ=%d pc=%zx\n", i, typ, pc);
- if (typ == EventTypeMop) {
- stack[pos] = pc;
- } else if (typ == EventTypeFuncEnter) {
- if (stack.Size() < pos + 2)
- stack.Resize(pos + 2);
- stack[pos++] = pc;
- } else if (typ == EventTypeFuncExit) {
- if (pos > 0)
- pos--;
+ DPrintf2("RestoreStack: sid=%u@%u addr=0x%zx/%zu typ=%x\n",
+ static_cast<int>(sid), static_cast<int>(epoch), addr, size,
+ static_cast<int>(typ));
+ ctx->slot_mtx.CheckLocked(); // needed to prevent trace part recycling
+ ctx->thread_registry.CheckLocked();
+ TidSlot *slot = &ctx->slots[static_cast<uptr>(sid)];
+ Tid tid = kInvalidTid;
+ // Need to lock the slot mutex as it protects slot->journal.
+ slot->mtx.CheckLocked();
+ for (uptr i = 0; i < slot->journal.Size(); i++) {
+ DPrintf2(" journal: epoch=%d tid=%d\n",
+ static_cast<int>(slot->journal[i].epoch), slot->journal[i].tid);
+ if (i == slot->journal.Size() - 1 || slot->journal[i + 1].epoch > epoch) {
+ tid = slot->journal[i].tid;
+ break;
}
- if (mset) {
- if (typ == EventTypeLock) {
- mset->Add(pc, true, epoch0 + i);
- } else if (typ == EventTypeUnlock) {
- mset->Del(pc, true);
- } else if (typ == EventTypeRLock) {
- mset->Add(pc, false, epoch0 + i);
- } else if (typ == EventTypeRUnlock) {
- mset->Del(pc, false);
- }
+ }
+ if (tid == kInvalidTid)
+ return false;
+ *ptid = tid;
+ ThreadContext *tctx =
+ static_cast<ThreadContext *>(ctx->thread_registry.GetThreadLocked(tid));
+ Trace *trace = &tctx->trace;
+ // Snapshot first/last parts and the current position in the last part.
+ TracePart *first_part;
+ TracePart *last_part;
+ Event *last_pos;
+ {
+ Lock lock(&trace->mtx);
+ first_part = trace->parts.Front();
+ if (!first_part) {
+ DPrintf2("RestoreStack: tid=%d trace=%p no trace parts\n", tid, trace);
+ return false;
}
- for (uptr j = 0; j <= pos; j++)
- DPrintf2(" #%zu: %zx\n", j, stack[j]);
+ last_part = trace->parts.Back();
+ last_pos = trace->final_pos;
+ if (tctx->thr)
+ last_pos = (Event *)atomic_load_relaxed(&tctx->thr->trace_pos);
}
- if (pos == 0 && stack[0] == 0)
- return;
- pos++;
- stk->Init(&stack[0], pos);
- ExtractTagFromStack(stk, tag);
+ DynamicMutexSet mset;
+ Vector<uptr> stack;
+ uptr prev_pc = 0;
+ bool found = false;
+ bool is_read = typ & kAccessRead;
+ bool is_atomic = typ & kAccessAtomic;
+ bool is_free = typ & kAccessFree;
+ DPrintf2("RestoreStack: tid=%d parts=[%p-%p] last_pos=%p\n", tid,
+ trace->parts.Front(), last_part, last_pos);
+ TraceReplay(
+ trace, last_part, last_pos, sid, epoch,
+ [&](Sid ev_sid, Epoch ev_epoch, Event *evp) {
+ if (evp == nullptr) {
+ // Each trace part is self-consistent, so we reset state.
+ stack.Resize(0);
+ mset->Reset();
+ prev_pc = 0;
+ return;
+ }
+ bool match = ev_sid == sid && ev_epoch == epoch;
+ if (evp->is_access) {
+ if (evp->is_func == 0 && evp->type == EventType::kAccessExt &&
+ evp->_ == 0) // NopEvent
+ return;
+ auto *ev = reinterpret_cast<EventAccess *>(evp);
+ uptr ev_addr = RestoreAddr(ev->addr);
+ uptr ev_size = 1 << ev->size_log;
+ uptr ev_pc =
+ prev_pc + ev->pc_delta - (1 << (EventAccess::kPCBits - 1));
+ prev_pc = ev_pc;
+ DPrintf2(" Access: pc=0x%zx addr=0x%zx/%zu type=%u/%u\n", ev_pc,
+ ev_addr, ev_size, ev->is_read, ev->is_atomic);
+ if (match && type == EventType::kAccessExt &&
+ IsWithinAccess(addr, size, ev_addr, ev_size) &&
+ is_read == ev->is_read && is_atomic == ev->is_atomic && !is_free)
+ RestoreStackMatch(pstk, pmset, &stack, mset, ev_pc, &found);
+ return;
+ }
+ if (evp->is_func) {
+ auto *ev = reinterpret_cast<EventFunc *>(evp);
+ if (ev->pc) {
+ DPrintf2(" FuncEnter: pc=0x%llx\n", ev->pc);
+ stack.PushBack(ev->pc);
+ } else {
+ DPrintf2(" FuncExit\n");
+ // We don't log pathologically large stacks in each part,
+ // if the stack was truncated we can have more func exits than
+ // entries.
+ if (stack.Size())
+ stack.PopBack();
+ }
+ return;
+ }
+ switch (evp->type) {
+ case EventType::kAccessExt: {
+ auto *ev = reinterpret_cast<EventAccessExt *>(evp);
+ uptr ev_addr = RestoreAddr(ev->addr);
+ uptr ev_size = 1 << ev->size_log;
+ prev_pc = ev->pc;
+ DPrintf2(" AccessExt: pc=0x%llx addr=0x%zx/%zu type=%u/%u\n",
+ ev->pc, ev_addr, ev_size, ev->is_read, ev->is_atomic);
+ if (match && type == EventType::kAccessExt &&
+ IsWithinAccess(addr, size, ev_addr, ev_size) &&
+ is_read == ev->is_read && is_atomic == ev->is_atomic &&
+ !is_free)
+ RestoreStackMatch(pstk, pmset, &stack, mset, ev->pc, &found);
+ break;
+ }
+ case EventType::kAccessRange: {
+ auto *ev = reinterpret_cast<EventAccessRange *>(evp);
+ uptr ev_addr = RestoreAddr(ev->addr);
+ uptr ev_size =
+ (ev->size_hi << EventAccessRange::kSizeLoBits) + ev->size_lo;
+ uptr ev_pc = RestoreAddr(ev->pc);
+ prev_pc = ev_pc;
+ DPrintf2(" Range: pc=0x%zx addr=0x%zx/%zu type=%u/%u\n", ev_pc,
+ ev_addr, ev_size, ev->is_read, ev->is_free);
+ if (match && type == EventType::kAccessExt &&
+ IsWithinAccess(addr, size, ev_addr, ev_size) &&
+ is_read == ev->is_read && !is_atomic && is_free == ev->is_free)
+ RestoreStackMatch(pstk, pmset, &stack, mset, ev_pc, &found);
+ break;
+ }
+ case EventType::kLock:
+ FALLTHROUGH;
+ case EventType::kRLock: {
+ auto *ev = reinterpret_cast<EventLock *>(evp);
+ bool is_write = ev->type == EventType::kLock;
+ uptr ev_addr = RestoreAddr(ev->addr);
+ uptr ev_pc = RestoreAddr(ev->pc);
+ StackID stack_id =
+ (ev->stack_hi << EventLock::kStackIDLoBits) + ev->stack_lo;
+ DPrintf2(" Lock: pc=0x%zx addr=0x%zx stack=%u write=%d\n", ev_pc,
+ ev_addr, stack_id, is_write);
+ mset->AddAddr(ev_addr, stack_id, is_write);
+ // Events with ev_pc == 0 are written to the beginning of trace
+ // part as initial mutex set (are not real).
+ if (match && type == EventType::kLock && addr == ev_addr && ev_pc)
+ RestoreStackMatch(pstk, pmset, &stack, mset, ev_pc, &found);
+ break;
+ }
+ case EventType::kUnlock: {
+ auto *ev = reinterpret_cast<EventUnlock *>(evp);
+ uptr ev_addr = RestoreAddr(ev->addr);
+ DPrintf2(" Unlock: addr=0x%zx\n", ev_addr);
+ mset->DelAddr(ev_addr);
+ break;
+ }
+ case EventType::kTime:
+ // TraceReplay already extracted sid/epoch from it,
+ // nothing else to do here.
+ break;
+ }
+ });
+ ExtractTagFromStack(pstk, ptag);
+ return found;
+}
+
+bool RacyStacks::operator==(const RacyStacks &other) const {
+ if (hash[0] == other.hash[0] && hash[1] == other.hash[1])
+ return true;
+ if (hash[0] == other.hash[1] && hash[1] == other.hash[0])
+ return true;
+ return false;
}
static bool FindRacyStacks(const RacyStacks &hash) {
return false;
}
-static bool FindRacyAddress(const RacyAddress &ra0) {
- for (uptr i = 0; i < ctx->racy_addresses.Size(); i++) {
- RacyAddress ra2 = ctx->racy_addresses[i];
- uptr maxbeg = max(ra0.addr_min, ra2.addr_min);
- uptr minend = min(ra0.addr_max, ra2.addr_max);
- if (maxbeg < minend) {
- VPrintf(2, "ThreadSanitizer: suppressing report as doubled (addr)\n");
- return true;
- }
- }
- return false;
-}
-
-static bool HandleRacyAddress(ThreadState *thr, uptr addr_min, uptr addr_max) {
- if (!flags()->suppress_equal_addresses)
- return false;
- RacyAddress ra0 = {addr_min, addr_max};
- {
- ReadLock lock(&ctx->racy_mtx);
- if (FindRacyAddress(ra0))
- return true;
- }
- Lock lock(&ctx->racy_mtx);
- if (FindRacyAddress(ra0))
- return true;
- ctx->racy_addresses.PushBack(ra0);
- return false;
-}
-
bool OutputReport(ThreadState *thr, const ScopedReport &srep) {
// These should have been checked in ShouldReport.
// It's too late to check them here, we have already taken locks.
ctx->fired_suppressions.push_back(s);
}
{
- bool old_is_freeing = thr->is_freeing;
- thr->is_freeing = false;
bool suppressed = OnReport(rep, pc_or_addr != 0);
- thr->is_freeing = old_is_freeing;
if (suppressed) {
thr->current_report = nullptr;
return false;
return false;
}
-static bool RaceBetweenAtomicAndFree(ThreadState *thr) {
- Shadow s0(thr->racy_state[0]);
- Shadow s1(thr->racy_state[1]);
- CHECK(!(s0.IsAtomic() && s1.IsAtomic()));
- if (!s0.IsAtomic() && !s1.IsAtomic())
- return true;
- if (s0.IsAtomic() && s1.IsFreed())
- return true;
- if (s1.IsAtomic() && thr->is_freeing)
- return true;
- return false;
+static bool SpuriousRace(Shadow old) {
+ Shadow last(LoadShadow(&ctx->last_spurious_race));
+ return last.sid() == old.sid() && last.epoch() == old.epoch();
}
-void ReportRace(ThreadState *thr) {
+void ReportRace(ThreadState *thr, RawShadow *shadow_mem, Shadow cur, Shadow old,
+ AccessType typ0) {
CheckedMutex::CheckNoLocks();
// Symbolizer makes lots of intercepted calls. If we try to process them,
// at best it will cause deadlocks on internal mutexes.
ScopedIgnoreInterceptors ignore;
+ uptr addr = ShadowToMem(shadow_mem);
+ DPrintf("#%d: ReportRace %p\n", thr->tid, (void *)addr);
if (!ShouldReport(thr, ReportTypeRace))
return;
- if (!flags()->report_atomic_races && !RaceBetweenAtomicAndFree(thr))
+ uptr addr_off0, size0;
+ cur.GetAccess(&addr_off0, &size0, nullptr);
+ uptr addr_off1, size1, typ1;
+ old.GetAccess(&addr_off1, &size1, &typ1);
+ if (!flags()->report_atomic_races &&
+ ((typ0 & kAccessAtomic) || (typ1 & kAccessAtomic)) &&
+ !(typ0 & kAccessFree) && !(typ1 & kAccessFree))
+ return;
+ if (SpuriousRace(old))
return;
- bool freed = false;
- {
- Shadow s(thr->racy_state[1]);
- freed = s.GetFreedAndReset();
- thr->racy_state[1] = s.raw();
- }
-
- uptr addr = ShadowToMem((uptr)thr->racy_shadow_addr);
- uptr addr_min = 0;
- uptr addr_max = 0;
- {
- uptr a0 = addr + Shadow(thr->racy_state[0]).addr0();
- uptr a1 = addr + Shadow(thr->racy_state[1]).addr0();
- uptr e0 = a0 + Shadow(thr->racy_state[0]).size();
- uptr e1 = a1 + Shadow(thr->racy_state[1]).size();
- addr_min = min(a0, a1);
- addr_max = max(e0, e1);
- if (IsExpectedReport(addr_min, addr_max - addr_min))
- return;
- }
- if (HandleRacyAddress(thr, addr_min, addr_max))
+ const uptr kMop = 2;
+ Shadow s[kMop] = {cur, old};
+ uptr addr0 = addr + addr_off0;
+ uptr addr1 = addr + addr_off1;
+ uptr end0 = addr0 + size0;
+ uptr end1 = addr1 + size1;
+ uptr addr_min = min(addr0, addr1);
+ uptr addr_max = max(end0, end1);
+ if (IsExpectedReport(addr_min, addr_max - addr_min))
return;
- ReportType typ = ReportTypeRace;
- if (thr->is_vptr_access && freed)
- typ = ReportTypeVptrUseAfterFree;
- else if (thr->is_vptr_access)
- typ = ReportTypeVptrRace;
- else if (freed)
- typ = ReportTypeUseAfterFree;
+ ReportType rep_typ = ReportTypeRace;
+ if ((typ0 & kAccessVptr) && (typ1 & kAccessFree))
+ rep_typ = ReportTypeVptrUseAfterFree;
+ else if (typ0 & kAccessVptr)
+ rep_typ = ReportTypeVptrRace;
+ else if (typ1 & kAccessFree)
+ rep_typ = ReportTypeUseAfterFree;
- if (IsFiredSuppression(ctx, typ, addr))
+ if (IsFiredSuppression(ctx, rep_typ, addr))
return;
- const uptr kMop = 2;
VarSizeStackTrace traces[kMop];
- uptr tags[kMop] = {kExternalTagNone};
- uptr toppc = TraceTopPC(thr);
- if (toppc >> kEventPCBits) {
- // This is a work-around for a known issue.
- // The scenario where this happens is rather elaborate and requires
- // an instrumented __sanitizer_report_error_summary callback and
- // a __tsan_symbolize_external callback and a race during a range memory
- // access larger than 8 bytes. MemoryAccessRange adds the current PC to
- // the trace and starts processing memory accesses. A first memory access
- // triggers a race, we report it and call the instrumented
- // __sanitizer_report_error_summary, which adds more stuff to the trace
- // since it is intrumented. Then a second memory access in MemoryAccessRange
- // also triggers a race and we get here and call TraceTopPC to get the
- // current PC, however now it contains some unrelated events from the
- // callback. Most likely, TraceTopPC will now return a EventTypeFuncExit
- // event. Later we subtract -1 from it (in GetPreviousInstructionPc)
- // and the resulting PC has kExternalPCBit set, so we pass it to
- // __tsan_symbolize_external_ex. __tsan_symbolize_external_ex is within its
- // rights to crash since the PC is completely bogus.
- // test/tsan/double_race.cpp contains a test case for this.
- toppc = 0;
- }
- ObtainCurrentStack(thr, toppc, &traces[0], &tags[0]);
- if (IsFiredSuppression(ctx, typ, traces[0]))
+ Tid tids[kMop] = {thr->tid, kInvalidTid};
+ uptr tags[kMop] = {kExternalTagNone, kExternalTagNone};
+
+ ObtainCurrentStack(thr, thr->trace_prev_pc, &traces[0], &tags[0]);
+ if (IsFiredSuppression(ctx, rep_typ, traces[0]))
return;
- // MutexSet is too large to live on stack.
- Vector<u64> mset_buffer;
- mset_buffer.Resize(sizeof(MutexSet) / sizeof(u64) + 1);
- MutexSet *mset2 = new(&mset_buffer[0]) MutexSet();
+ DynamicMutexSet mset1;
+ MutexSet *mset[kMop] = {&thr->mset, mset1};
+
+ // We need to lock the slot during RestoreStack because it protects
+ // the slot journal.
+ Lock slot_lock(&ctx->slots[static_cast<uptr>(s[1].sid())].mtx);
+ ThreadRegistryLock l0(&ctx->thread_registry);
+ Lock slots_lock(&ctx->slot_mtx);
+ if (SpuriousRace(old))
+ return;
+ if (!RestoreStack(EventType::kAccessExt, s[1].sid(), s[1].epoch(), addr1,
+ size1, typ1, &tids[1], &traces[1], mset[1], &tags[1])) {
+ StoreShadow(&ctx->last_spurious_race, old.raw());
+ return;
+ }
- Shadow s2(thr->racy_state[1]);
- RestoreStack(s2.tid(), s2.epoch(), &traces[1], mset2, &tags[1]);
- if (IsFiredSuppression(ctx, typ, traces[1]))
+ if (IsFiredSuppression(ctx, rep_typ, traces[1]))
return;
if (HandleRacyStacks(thr, traces))
uptr tag = kExternalTagNone;
for (uptr i = 0; i < kMop; i++) {
if (tags[i] != kExternalTagNone) {
- typ = ReportTypeExternalRace;
+ rep_typ = ReportTypeExternalRace;
tag = tags[i];
break;
}
}
- ThreadRegistryLock l0(ctx->thread_registry);
- ScopedReport rep(typ, tag);
- for (uptr i = 0; i < kMop; i++) {
- Shadow s(thr->racy_state[i]);
- rep.AddMemoryAccess(addr, tags[i], s, traces[i],
- i == 0 ? &thr->mset : mset2);
- }
+ ScopedReport rep(rep_typ, tag);
+ for (uptr i = 0; i < kMop; i++)
+ rep.AddMemoryAccess(addr, tags[i], s[i], tids[i], traces[i], mset[i]);
for (uptr i = 0; i < kMop; i++) {
- FastState s(thr->racy_state[i]);
- ThreadContext *tctx = static_cast<ThreadContext*>(
- ctx->thread_registry->GetThreadLocked(s.tid()));
- if (s.epoch() < tctx->epoch0 || s.epoch() > tctx->epoch1)
- continue;
+ ThreadContext *tctx = static_cast<ThreadContext *>(
+ ctx->thread_registry.GetThreadLocked(tids[i]));
rep.AddThread(tctx);
}
rep.AddLocation(addr_min, addr_max - addr_min);
-#if !SANITIZER_GO
- {
- Shadow s(thr->racy_state[1]);
- if (s.epoch() <= thr->last_sleep_clock.get(s.tid()))
- rep.AddSleep(thr->last_sleep_stack_id);
+ if (flags()->print_full_thread_history) {
+ const ReportDesc *rep_desc = rep.GetReport();
+ for (uptr i = 0; i < rep_desc->threads.Size(); i++) {
+ Tid parent_tid = rep_desc->threads[i]->parent_tid;
+ if (parent_tid == kMainTid || parent_tid == kInvalidTid)
+ continue;
+ ThreadContext *parent_tctx = static_cast<ThreadContext *>(
+ ctx->thread_registry.GetThreadLocked(parent_tid));
+ rep.AddThread(parent_tctx);
+ }
}
-#endif
+#if !SANITIZER_GO
+ if (!((typ0 | typ1) & kAccessFree) &&
+ s[1].epoch() <= thr->last_sleep_clock.Get(s[1].sid()))
+ rep.AddSleep(thr->last_sleep_stack_id);
+#endif
OutputReport(thr, rep);
}
ALWAYS_INLINE USED void PrintCurrentStackSlow(uptr pc) {
#if !SANITIZER_GO
uptr bp = GET_CURRENT_FRAME();
- BufferedStackTrace *ptrace =
- new(internal_alloc(MBlockStackTrace, sizeof(BufferedStackTrace)))
- BufferedStackTrace();
+ auto *ptrace = New<BufferedStackTrace>();
ptrace->Unwind(pc, bp, nullptr, false);
for (uptr i = 0; i < ptrace->size / 2; i++) {
intercept _setjmp, _ZN14__interception12real__setjmpE
intercept sigsetjmp, _ZN14__interception14real_sigsetjmpE
intercept __sigsetjmp, _ZN14__interception16real___sigsetjmpE
+
+NO_EXEC_STACK_DIRECTIVE
// ThreadContext implementation.
-ThreadContext::ThreadContext(int tid)
- : ThreadContextBase(tid)
- , thr()
- , sync()
- , epoch0()
- , epoch1() {
-}
+ThreadContext::ThreadContext(Tid tid) : ThreadContextBase(tid), thr(), sync() {}
#if !SANITIZER_GO
ThreadContext::~ThreadContext() {
}
#endif
-void ThreadContext::OnDead() {
- CHECK_EQ(sync.size(), 0);
-}
-
-void ThreadContext::OnJoined(void *arg) {
- ThreadState *caller_thr = static_cast<ThreadState *>(arg);
- AcquireImpl(caller_thr, 0, &sync);
- sync.Reset(&caller_thr->proc()->clock_cache);
-}
-
-struct OnCreatedArgs {
- ThreadState *thr;
- uptr pc;
-};
-
-void ThreadContext::OnCreated(void *arg) {
- thr = 0;
- if (tid == kMainTid)
- return;
- OnCreatedArgs *args = static_cast<OnCreatedArgs *>(arg);
- if (!args->thr) // GCD workers don't have a parent thread.
- return;
- args->thr->fast_state.IncrementEpoch();
- // Can't increment epoch w/o writing to the trace as well.
- TraceAddEvent(args->thr, args->thr->fast_state, EventTypeMop, 0);
- ReleaseImpl(args->thr, 0, &sync);
- creation_stack_id = CurrentStackId(args->thr, args->pc);
-}
-
-void ThreadContext::OnReset() {
- CHECK_EQ(sync.size(), 0);
- uptr trace_p = GetThreadTrace(tid);
- ReleaseMemoryPagesToOS(trace_p, trace_p + TraceSize() * sizeof(Event));
- //!!! ReleaseMemoryToOS(GetThreadTraceHeader(tid), sizeof(Trace));
-}
-
-void ThreadContext::OnDetached(void *arg) {
- ThreadState *thr1 = static_cast<ThreadState*>(arg);
- sync.Reset(&thr1->proc()->clock_cache);
-}
-
-struct OnStartedArgs {
- ThreadState *thr;
- uptr stk_addr;
- uptr stk_size;
- uptr tls_addr;
- uptr tls_size;
-};
-
-void ThreadContext::OnStarted(void *arg) {
- OnStartedArgs *args = static_cast<OnStartedArgs*>(arg);
- thr = args->thr;
- // RoundUp so that one trace part does not contain events
- // from different threads.
- epoch0 = RoundUp(epoch1 + 1, kTracePartSize);
- epoch1 = (u64)-1;
- new(thr) ThreadState(ctx, tid, unique_id, epoch0, reuse_count,
- args->stk_addr, args->stk_size, args->tls_addr, args->tls_size);
-#if !SANITIZER_GO
- thr->shadow_stack = &ThreadTrace(thr->tid)->shadow_stack[0];
- thr->shadow_stack_pos = thr->shadow_stack;
- thr->shadow_stack_end = thr->shadow_stack + kShadowStackSize;
-#else
- // Setup dynamic shadow stack.
- const int kInitStackSize = 8;
- thr->shadow_stack = (uptr*)internal_alloc(MBlockShadowStack,
- kInitStackSize * sizeof(uptr));
- thr->shadow_stack_pos = thr->shadow_stack;
- thr->shadow_stack_end = thr->shadow_stack + kInitStackSize;
-#endif
- if (common_flags()->detect_deadlocks)
- thr->dd_lt = ctx->dd->CreateLogicalThread(unique_id);
- thr->fast_state.SetHistorySize(flags()->history_size);
- // Commit switch to the new part of the trace.
- // TraceAddEvent will reset stack0/mset0 in the new part for us.
- TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
-
- thr->fast_synch_epoch = epoch0;
- AcquireImpl(thr, 0, &sync);
- sync.Reset(&thr->proc()->clock_cache);
- thr->is_inited = true;
- DPrintf("#%d: ThreadStart epoch=%zu stk_addr=%zx stk_size=%zx "
- "tls_addr=%zx tls_size=%zx\n",
- tid, (uptr)epoch0, args->stk_addr, args->stk_size,
- args->tls_addr, args->tls_size);
-}
-
-void ThreadContext::OnFinished() {
-#if SANITIZER_GO
- internal_free(thr->shadow_stack);
- thr->shadow_stack = nullptr;
- thr->shadow_stack_pos = nullptr;
- thr->shadow_stack_end = nullptr;
-#endif
- if (!detached) {
- thr->fast_state.IncrementEpoch();
- // Can't increment epoch w/o writing to the trace as well.
- TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
- ReleaseImpl(thr, 0, &sync);
- }
- epoch1 = thr->fast_state.epoch();
-
- if (common_flags()->detect_deadlocks)
- ctx->dd->DestroyLogicalThread(thr->dd_lt);
- thr->clock.ResetCached(&thr->proc()->clock_cache);
-#if !SANITIZER_GO
- thr->last_sleep_clock.ResetCached(&thr->proc()->clock_cache);
-#endif
-#if !SANITIZER_GO
- PlatformCleanUpThreadState(thr);
-#endif
- thr->~ThreadState();
- thr = 0;
-}
+void ThreadContext::OnReset() { CHECK(!sync); }
#if !SANITIZER_GO
struct ThreadLeak {
int count;
};
-static void MaybeReportThreadLeak(ThreadContextBase *tctx_base, void *arg) {
- Vector<ThreadLeak> &leaks = *(Vector<ThreadLeak>*)arg;
- ThreadContext *tctx = static_cast<ThreadContext*>(tctx_base);
+static void CollectThreadLeaks(ThreadContextBase *tctx_base, void *arg) {
+ auto &leaks = *static_cast<Vector<ThreadLeak> *>(arg);
+ auto *tctx = static_cast<ThreadContext *>(tctx_base);
if (tctx->detached || tctx->status != ThreadStatusFinished)
return;
for (uptr i = 0; i < leaks.Size(); i++) {
return;
}
}
- ThreadLeak leak = {tctx, 1};
- leaks.PushBack(leak);
+ leaks.PushBack({tctx, 1});
}
#endif
-#if !SANITIZER_GO
+// Disabled on Mac because lldb test TestTsanBasic fails:
+// https://reviews.llvm.org/D112603#3163158
+#if !SANITIZER_GO && !SANITIZER_APPLE
static void ReportIgnoresEnabled(ThreadContext *tctx, IgnoreSet *set) {
if (tctx->tid == kMainTid) {
Printf("ThreadSanitizer: main thread finished with ignores enabled\n");
#if !SANITIZER_GO
if (!ShouldReport(thr, ReportTypeThreadLeak))
return;
- ThreadRegistryLock l(ctx->thread_registry);
+ ThreadRegistryLock l(&ctx->thread_registry);
Vector<ThreadLeak> leaks;
- ctx->thread_registry->RunCallbackForEachThreadLocked(
- MaybeReportThreadLeak, &leaks);
+ ctx->thread_registry.RunCallbackForEachThreadLocked(CollectThreadLeaks,
+ &leaks);
for (uptr i = 0; i < leaks.Size(); i++) {
ScopedReport rep(ReportTypeThreadLeak);
rep.AddThread(leaks[i].tctx, true);
int ThreadCount(ThreadState *thr) {
uptr result;
- ctx->thread_registry->GetNumberOfThreads(0, 0, &result);
+ ctx->thread_registry.GetNumberOfThreads(0, 0, &result);
return (int)result;
}
-int ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached) {
- OnCreatedArgs args = { thr, pc };
- u32 parent_tid = thr ? thr->tid : kInvalidTid; // No parent for GCD workers.
- int tid =
- ctx->thread_registry->CreateThread(uid, detached, parent_tid, &args);
- DPrintf("#%d: ThreadCreate tid=%d uid=%zu\n", parent_tid, tid, uid);
+struct OnCreatedArgs {
+ VectorClock *sync;
+ uptr sync_epoch;
+ StackID stack;
+};
+
+Tid ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached) {
+ // The main thread and GCD workers don't have a parent thread.
+ Tid parent = kInvalidTid;
+ OnCreatedArgs arg = {nullptr, 0, kInvalidStackID};
+ if (thr) {
+ parent = thr->tid;
+ arg.stack = CurrentStackId(thr, pc);
+ if (!thr->ignore_sync) {
+ SlotLocker locker(thr);
+ thr->clock.ReleaseStore(&arg.sync);
+ arg.sync_epoch = ctx->global_epoch;
+ IncrementEpoch(thr);
+ }
+ }
+ Tid tid = ctx->thread_registry.CreateThread(uid, detached, parent, &arg);
+ DPrintf("#%d: ThreadCreate tid=%d uid=%zu\n", parent, tid, uid);
return tid;
}
-void ThreadStart(ThreadState *thr, int tid, tid_t os_id,
+void ThreadContext::OnCreated(void *arg) {
+ OnCreatedArgs *args = static_cast<OnCreatedArgs *>(arg);
+ sync = args->sync;
+ sync_epoch = args->sync_epoch;
+ creation_stack_id = args->stack;
+}
+
+extern "C" void __tsan_stack_initialization() {}
+
+struct OnStartedArgs {
+ ThreadState *thr;
+ uptr stk_addr;
+ uptr stk_size;
+ uptr tls_addr;
+ uptr tls_size;
+};
+
+void ThreadStart(ThreadState *thr, Tid tid, tid_t os_id,
ThreadType thread_type) {
+ ctx->thread_registry.StartThread(tid, os_id, thread_type, thr);
+ if (!thr->ignore_sync) {
+ SlotAttachAndLock(thr);
+ if (thr->tctx->sync_epoch == ctx->global_epoch)
+ thr->clock.Acquire(thr->tctx->sync);
+ SlotUnlock(thr);
+ }
+ Free(thr->tctx->sync);
+
uptr stk_addr = 0;
uptr stk_size = 0;
uptr tls_addr = 0;
if (thread_type != ThreadType::Fiber)
GetThreadStackAndTls(tid == kMainTid, &stk_addr, &stk_size, &tls_addr,
&tls_size);
-
- if (tid != kMainTid) {
- if (stk_addr && stk_size)
- MemoryRangeImitateWrite(thr, /*pc=*/ 1, stk_addr, stk_size);
-
- if (tls_addr && tls_size) ImitateTlsWrite(thr, tls_addr, tls_size);
- }
#endif
-
- ThreadRegistry *tr = ctx->thread_registry;
- OnStartedArgs args = { thr, stk_addr, stk_size, tls_addr, tls_size };
- tr->StartThread(tid, os_id, thread_type, &args);
-
- tr->Lock();
- thr->tctx = (ThreadContext*)tr->GetThreadLocked(tid);
- tr->Unlock();
+ thr->stk_addr = stk_addr;
+ thr->stk_size = stk_size;
+ thr->tls_addr = tls_addr;
+ thr->tls_size = tls_size;
#if !SANITIZER_GO
if (ctx->after_multithreaded_fork) {
ThreadIgnoreSyncBegin(thr, 0);
}
#endif
+
+#if !SANITIZER_GO
+ // Don't imitate stack/TLS writes for the main thread,
+ // because its initialization is synchronized with all
+ // subsequent threads anyway.
+ if (tid != kMainTid) {
+ if (stk_addr && stk_size) {
+ const uptr pc = StackTrace::GetNextInstructionPc(
+ reinterpret_cast<uptr>(__tsan_stack_initialization));
+ MemoryRangeImitateWrite(thr, pc, stk_addr, stk_size);
+ }
+
+ if (tls_addr && tls_size)
+ ImitateTlsWrite(thr, tls_addr, tls_size);
+ }
+#endif
+}
+
+void ThreadContext::OnStarted(void *arg) {
+ thr = static_cast<ThreadState *>(arg);
+ DPrintf("#%d: ThreadStart\n", tid);
+ new (thr) ThreadState(tid);
+ if (common_flags()->detect_deadlocks)
+ thr->dd_lt = ctx->dd->CreateLogicalThread(tid);
+ thr->tctx = this;
+#if !SANITIZER_GO
+ thr->is_inited = true;
+#endif
}
void ThreadFinish(ThreadState *thr) {
+ DPrintf("#%d: ThreadFinish\n", thr->tid);
ThreadCheckIgnore(thr);
if (thr->stk_addr && thr->stk_size)
DontNeedShadowFor(thr->stk_addr, thr->stk_size);
if (thr->tls_addr && thr->tls_size)
DontNeedShadowFor(thr->tls_addr, thr->tls_size);
thr->is_dead = true;
- ctx->thread_registry->FinishThread(thr->tid);
+#if !SANITIZER_GO
+ thr->is_inited = false;
+ thr->ignore_interceptors++;
+ PlatformCleanUpThreadState(thr);
+#endif
+ if (!thr->ignore_sync) {
+ SlotLocker locker(thr);
+ ThreadRegistryLock lock(&ctx->thread_registry);
+ // Note: detached is protected by the thread registry mutex,
+ // the thread may be detaching concurrently in another thread.
+ if (!thr->tctx->detached) {
+ thr->clock.ReleaseStore(&thr->tctx->sync);
+ thr->tctx->sync_epoch = ctx->global_epoch;
+ IncrementEpoch(thr);
+ }
+ }
+#if !SANITIZER_GO
+ UnmapOrDie(thr->shadow_stack, kShadowStackSize * sizeof(uptr));
+#else
+ Free(thr->shadow_stack);
+#endif
+ thr->shadow_stack = nullptr;
+ thr->shadow_stack_pos = nullptr;
+ thr->shadow_stack_end = nullptr;
+ if (common_flags()->detect_deadlocks)
+ ctx->dd->DestroyLogicalThread(thr->dd_lt);
+ SlotDetach(thr);
+ ctx->thread_registry.FinishThread(thr->tid);
+ thr->~ThreadState();
+}
+
+void ThreadContext::OnFinished() {
+ Lock lock(&ctx->slot_mtx);
+ Lock lock1(&trace.mtx);
+ // Queue all trace parts into the global recycle queue.
+ auto parts = &trace.parts;
+ while (trace.local_head) {
+ CHECK(parts->Queued(trace.local_head));
+ ctx->trace_part_recycle.PushBack(trace.local_head);
+ trace.local_head = parts->Next(trace.local_head);
+ }
+ ctx->trace_part_recycle_finished += parts->Size();
+ if (ctx->trace_part_recycle_finished > Trace::kFinishedThreadHi) {
+ ctx->trace_part_finished_excess += parts->Size();
+ trace.parts_allocated = 0;
+ } else if (ctx->trace_part_recycle_finished > Trace::kFinishedThreadLo &&
+ parts->Size() > 1) {
+ ctx->trace_part_finished_excess += parts->Size() - 1;
+ trace.parts_allocated = 1;
+ }
+ // From now on replay will use trace->final_pos.
+ trace.final_pos = (Event *)atomic_load_relaxed(&thr->trace_pos);
+ atomic_store_relaxed(&thr->trace_pos, 0);
+ thr->tctx = nullptr;
+ thr = nullptr;
}
struct ConsumeThreadContext {
ThreadContextBase *tctx;
};
-static bool ConsumeThreadByUid(ThreadContextBase *tctx, void *arg) {
- ConsumeThreadContext *findCtx = (ConsumeThreadContext *)arg;
- if (tctx->user_id == findCtx->uid && tctx->status != ThreadStatusInvalid) {
- if (findCtx->tctx) {
- // Ensure that user_id is unique. If it's not the case we are screwed.
- // Something went wrong before, but now there is no way to recover.
- // Returning a wrong thread is not an option, it may lead to very hard
- // to debug false positives (e.g. if we join a wrong thread).
- Report("ThreadSanitizer: dup thread with used id 0x%zx\n", findCtx->uid);
- Die();
- }
- findCtx->tctx = tctx;
- tctx->user_id = 0;
- }
- return false;
+Tid ThreadConsumeTid(ThreadState *thr, uptr pc, uptr uid) {
+ return ctx->thread_registry.ConsumeThreadUserId(uid);
}
-int ThreadConsumeTid(ThreadState *thr, uptr pc, uptr uid) {
- ConsumeThreadContext findCtx = {uid, nullptr};
- ctx->thread_registry->FindThread(ConsumeThreadByUid, &findCtx);
- int tid = findCtx.tctx ? findCtx.tctx->tid : kInvalidTid;
- DPrintf("#%d: ThreadTid uid=%zu tid=%d\n", thr->tid, uid, tid);
- return tid;
-}
+struct JoinArg {
+ VectorClock *sync;
+ uptr sync_epoch;
+};
-void ThreadJoin(ThreadState *thr, uptr pc, int tid) {
+void ThreadJoin(ThreadState *thr, uptr pc, Tid tid) {
CHECK_GT(tid, 0);
- CHECK_LT(tid, kMaxTid);
DPrintf("#%d: ThreadJoin tid=%d\n", thr->tid, tid);
- ctx->thread_registry->JoinThread(tid, thr);
+ JoinArg arg = {};
+ ctx->thread_registry.JoinThread(tid, &arg);
+ if (!thr->ignore_sync) {
+ SlotLocker locker(thr);
+ if (arg.sync_epoch == ctx->global_epoch)
+ thr->clock.Acquire(arg.sync);
+ }
+ Free(arg.sync);
}
-void ThreadDetach(ThreadState *thr, uptr pc, int tid) {
- CHECK_GT(tid, 0);
- CHECK_LT(tid, kMaxTid);
- ctx->thread_registry->DetachThread(tid, thr);
+void ThreadContext::OnJoined(void *ptr) {
+ auto arg = static_cast<JoinArg *>(ptr);
+ arg->sync = sync;
+ arg->sync_epoch = sync_epoch;
+ sync = nullptr;
+ sync_epoch = 0;
}
-void ThreadNotJoined(ThreadState *thr, uptr pc, int tid, uptr uid) {
- CHECK_GT(tid, 0);
- CHECK_LT(tid, kMaxTid);
- ctx->thread_registry->SetThreadUserId(tid, uid);
-}
+void ThreadContext::OnDead() { CHECK_EQ(sync, nullptr); }
-void ThreadSetName(ThreadState *thr, const char *name) {
- ctx->thread_registry->SetThreadName(thr->tid, name);
+void ThreadDetach(ThreadState *thr, uptr pc, Tid tid) {
+ CHECK_GT(tid, 0);
+ ctx->thread_registry.DetachThread(tid, thr);
}
-void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr,
- uptr size, bool is_write) {
- if (size == 0)
- return;
-
- u64 *shadow_mem = (u64*)MemToShadow(addr);
- DPrintf2("#%d: MemoryAccessRange: @%p %p size=%d is_write=%d\n",
- thr->tid, (void*)pc, (void*)addr,
- (int)size, is_write);
+void ThreadContext::OnDetached(void *arg) { Free(sync); }
-#if SANITIZER_DEBUG
- if (!IsAppMem(addr)) {
- Printf("Access to non app mem %zx\n", addr);
- DCHECK(IsAppMem(addr));
- }
- if (!IsAppMem(addr + size - 1)) {
- Printf("Access to non app mem %zx\n", addr + size - 1);
- DCHECK(IsAppMem(addr + size - 1));
- }
- if (!IsShadowMem((uptr)shadow_mem)) {
- Printf("Bad shadow addr %p (%zx)\n", shadow_mem, addr);
- DCHECK(IsShadowMem((uptr)shadow_mem));
- }
- if (!IsShadowMem((uptr)(shadow_mem + size * kShadowCnt / 8 - 1))) {
- Printf("Bad shadow addr %p (%zx)\n",
- shadow_mem + size * kShadowCnt / 8 - 1, addr + size - 1);
- DCHECK(IsShadowMem((uptr)(shadow_mem + size * kShadowCnt / 8 - 1)));
- }
-#endif
-
- if (*shadow_mem == kShadowRodata) {
- DCHECK(!is_write);
- // Access to .rodata section, no races here.
- // Measurements show that it can be 10-20% of all memory accesses.
- return;
- }
-
- FastState fast_state = thr->fast_state;
- if (fast_state.GetIgnoreBit())
- return;
-
- fast_state.IncrementEpoch();
- thr->fast_state = fast_state;
- TraceAddEvent(thr, fast_state, EventTypeMop, pc);
-
- bool unaligned = (addr % kShadowCell) != 0;
+void ThreadNotJoined(ThreadState *thr, uptr pc, Tid tid, uptr uid) {
+ CHECK_GT(tid, 0);
+ ctx->thread_registry.SetThreadUserId(tid, uid);
+}
- // Handle unaligned beginning, if any.
- for (; addr % kShadowCell && size; addr++, size--) {
- int const kAccessSizeLog = 0;
- Shadow cur(fast_state);
- cur.SetWrite(is_write);
- cur.SetAddr0AndSizeLog(addr & (kShadowCell - 1), kAccessSizeLog);
- MemoryAccessImpl(thr, addr, kAccessSizeLog, is_write, false,
- shadow_mem, cur);
- }
- if (unaligned)
- shadow_mem += kShadowCnt;
- // Handle middle part, if any.
- for (; size >= kShadowCell; addr += kShadowCell, size -= kShadowCell) {
- int const kAccessSizeLog = 3;
- Shadow cur(fast_state);
- cur.SetWrite(is_write);
- cur.SetAddr0AndSizeLog(0, kAccessSizeLog);
- MemoryAccessImpl(thr, addr, kAccessSizeLog, is_write, false,
- shadow_mem, cur);
- shadow_mem += kShadowCnt;
- }
- // Handle ending, if any.
- for (; size; addr++, size--) {
- int const kAccessSizeLog = 0;
- Shadow cur(fast_state);
- cur.SetWrite(is_write);
- cur.SetAddr0AndSizeLog(addr & (kShadowCell - 1), kAccessSizeLog);
- MemoryAccessImpl(thr, addr, kAccessSizeLog, is_write, false,
- shadow_mem, cur);
- }
+void ThreadSetName(ThreadState *thr, const char *name) {
+ ctx->thread_registry.SetThreadName(thr->tid, name);
}
#if !SANITIZER_GO
}
ThreadState *FiberCreate(ThreadState *thr, uptr pc, unsigned flags) {
- void *mem = internal_alloc(MBlockThreadContex, sizeof(ThreadState));
+ void *mem = Alloc(sizeof(ThreadState));
ThreadState *fiber = static_cast<ThreadState *>(mem);
internal_memset(fiber, 0, sizeof(*fiber));
- int tid = ThreadCreate(thr, pc, 0, true);
+ Tid tid = ThreadCreate(thr, pc, 0, true);
FiberSwitchImpl(thr, fiber);
ThreadStart(fiber, tid, 0, ThreadType::Fiber);
FiberSwitchImpl(fiber, thr);
FiberSwitchImpl(thr, fiber);
ThreadFinish(fiber);
FiberSwitchImpl(fiber, thr);
- internal_free(fiber);
+ Free(fiber);
}
void FiberSwitch(ThreadState *thr, uptr pc,
--- /dev/null
+//===-- tsan_shadow.h -------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef TSAN_SHADOW_H
+#define TSAN_SHADOW_H
+
+#include "tsan_defs.h"
+
+namespace __tsan {
+
+class FastState {
+ public:
+ FastState() { Reset(); }
+
+ void Reset() {
+ part_.unused0_ = 0;
+ part_.sid_ = static_cast<u8>(kFreeSid);
+ part_.epoch_ = static_cast<u16>(kEpochLast);
+ part_.unused1_ = 0;
+ part_.ignore_accesses_ = false;
+ }
+
+ void SetSid(Sid sid) { part_.sid_ = static_cast<u8>(sid); }
+
+ Sid sid() const { return static_cast<Sid>(part_.sid_); }
+
+ Epoch epoch() const { return static_cast<Epoch>(part_.epoch_); }
+
+ void SetEpoch(Epoch epoch) { part_.epoch_ = static_cast<u16>(epoch); }
+
+ void SetIgnoreBit() { part_.ignore_accesses_ = 1; }
+ void ClearIgnoreBit() { part_.ignore_accesses_ = 0; }
+ bool GetIgnoreBit() const { return part_.ignore_accesses_; }
+
+ private:
+ friend class Shadow;
+ struct Parts {
+ u32 unused0_ : 8;
+ u32 sid_ : 8;
+ u32 epoch_ : kEpochBits;
+ u32 unused1_ : 1;
+ u32 ignore_accesses_ : 1;
+ };
+ union {
+ Parts part_;
+ u32 raw_;
+ };
+};
+
+static_assert(sizeof(FastState) == kShadowSize, "bad FastState size");
+
+class Shadow {
+ public:
+ static constexpr RawShadow kEmpty = static_cast<RawShadow>(0);
+
+ Shadow(FastState state, u32 addr, u32 size, AccessType typ) {
+ raw_ = state.raw_;
+ DCHECK_GT(size, 0);
+ DCHECK_LE(size, 8);
+ UNUSED Sid sid0 = part_.sid_;
+ UNUSED u16 epoch0 = part_.epoch_;
+ raw_ |= (!!(typ & kAccessAtomic) << kIsAtomicShift) |
+ (!!(typ & kAccessRead) << kIsReadShift) |
+ (((((1u << size) - 1) << (addr & 0x7)) & 0xff) << kAccessShift);
+ // Note: we don't check kAccessAtomic because it overlaps with
+ // FastState::ignore_accesses_ and it may be set spuriously.
+ DCHECK_EQ(part_.is_read_, !!(typ & kAccessRead));
+ DCHECK_EQ(sid(), sid0);
+ DCHECK_EQ(epoch(), epoch0);
+ }
+
+ explicit Shadow(RawShadow x = Shadow::kEmpty) { raw_ = static_cast<u32>(x); }
+
+ RawShadow raw() const { return static_cast<RawShadow>(raw_); }
+ Sid sid() const { return part_.sid_; }
+ Epoch epoch() const { return static_cast<Epoch>(part_.epoch_); }
+ u8 access() const { return part_.access_; }
+
+ void GetAccess(uptr *addr, uptr *size, AccessType *typ) const {
+ DCHECK(part_.access_ != 0 || raw_ == static_cast<u32>(Shadow::kRodata));
+ if (addr)
+ *addr = part_.access_ ? __builtin_ffs(part_.access_) - 1 : 0;
+ if (size)
+ *size = part_.access_ == kFreeAccess ? kShadowCell
+ : __builtin_popcount(part_.access_);
+ if (typ) {
+ *typ = part_.is_read_ ? kAccessRead : kAccessWrite;
+ if (part_.is_atomic_)
+ *typ |= kAccessAtomic;
+ if (part_.access_ == kFreeAccess)
+ *typ |= kAccessFree;
+ }
+ }
+
+ ALWAYS_INLINE
+ bool IsBothReadsOrAtomic(AccessType typ) const {
+ u32 is_read = !!(typ & kAccessRead);
+ u32 is_atomic = !!(typ & kAccessAtomic);
+ bool res =
+ raw_ & ((is_atomic << kIsAtomicShift) | (is_read << kIsReadShift));
+ DCHECK_EQ(res,
+ (part_.is_read_ && is_read) || (part_.is_atomic_ && is_atomic));
+ return res;
+ }
+
+ ALWAYS_INLINE
+ bool IsRWWeakerOrEqual(AccessType typ) const {
+ u32 is_read = !!(typ & kAccessRead);
+ u32 is_atomic = !!(typ & kAccessAtomic);
+ UNUSED u32 res0 =
+ (part_.is_atomic_ > is_atomic) ||
+ (part_.is_atomic_ == is_atomic && part_.is_read_ >= is_read);
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ const u32 kAtomicReadMask = (1 << kIsAtomicShift) | (1 << kIsReadShift);
+ bool res = (raw_ & kAtomicReadMask) >=
+ ((is_atomic << kIsAtomicShift) | (is_read << kIsReadShift));
+
+ DCHECK_EQ(res, res0);
+ return res;
+#else
+ return res0;
+#endif
+ }
+
+ // The FreedMarker must not pass "the same access check" so that we don't
+ // return from the race detection algorithm early.
+ static RawShadow FreedMarker() {
+ FastState fs;
+ fs.SetSid(kFreeSid);
+ fs.SetEpoch(kEpochLast);
+ Shadow s(fs, 0, 8, kAccessWrite);
+ return s.raw();
+ }
+
+ static RawShadow FreedInfo(Sid sid, Epoch epoch) {
+ Shadow s;
+ s.part_.sid_ = sid;
+ s.part_.epoch_ = static_cast<u16>(epoch);
+ s.part_.access_ = kFreeAccess;
+ return s.raw();
+ }
+
+ private:
+ struct Parts {
+ u8 access_;
+ Sid sid_;
+ u16 epoch_ : kEpochBits;
+ u16 is_read_ : 1;
+ u16 is_atomic_ : 1;
+ };
+ union {
+ Parts part_;
+ u32 raw_;
+ };
+
+ static constexpr u8 kFreeAccess = 0x81;
+
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ static constexpr uptr kAccessShift = 0;
+ static constexpr uptr kIsReadShift = 30;
+ static constexpr uptr kIsAtomicShift = 31;
+#else
+ static constexpr uptr kAccessShift = 24;
+ static constexpr uptr kIsReadShift = 1;
+ static constexpr uptr kIsAtomicShift = 0;
+#endif
+
+ public:
+ // .rodata shadow marker, see MapRodata and ContainsSameAccessFast.
+ static constexpr RawShadow kRodata =
+ static_cast<RawShadow>(1 << kIsReadShift);
+};
+
+static_assert(sizeof(Shadow) == kShadowSize, "bad Shadow size");
+
+ALWAYS_INLINE RawShadow LoadShadow(RawShadow *p) {
+ return static_cast<RawShadow>(
+ atomic_load((atomic_uint32_t *)p, memory_order_relaxed));
+}
+
+ALWAYS_INLINE void StoreShadow(RawShadow *sp, RawShadow s) {
+ atomic_store((atomic_uint32_t *)sp, static_cast<u32>(s),
+ memory_order_relaxed);
+}
+
+} // namespace __tsan
+
+#endif
}
void VarSizeStackTrace::ResizeBuffer(uptr new_size) {
- if (trace_buffer) {
- internal_free(trace_buffer);
- }
- trace_buffer =
- (new_size > 0)
- ? (uptr *)internal_alloc(MBlockStackTrace,
- new_size * sizeof(trace_buffer[0]))
- : nullptr;
+ Free(trace_buffer);
+ trace_buffer = (new_size > 0)
+ ? (uptr *)Alloc(new_size * sizeof(trace_buffer[0]))
+ : nullptr;
trace = trace_buffer;
size = new_size;
}
DataInfo info;
if (!Symbolizer::GetOrInit()->SymbolizeData(addr, &info))
return 0;
- ReportLocation *ent = ReportLocation::New(ReportLocationGlobal);
+ auto *ent = New<ReportLocation>();
+ ent->type = ReportLocationGlobal;
internal_memcpy(&ent->global, &info, sizeof(info));
return ent;
}
void DDMutexInit(ThreadState *thr, uptr pc, SyncVar *s);
-SyncVar::SyncVar() : mtx(MutexTypeSyncVar) { Reset(0); }
+SyncVar::SyncVar() : mtx(MutexTypeSyncVar) { Reset(); }
-void SyncVar::Init(ThreadState *thr, uptr pc, uptr addr, u64 uid) {
+void SyncVar::Init(ThreadState *thr, uptr pc, uptr addr, bool save_stack) {
+ Reset();
this->addr = addr;
- this->uid = uid;
- this->next = 0;
-
- creation_stack_id = 0;
- if (!SANITIZER_GO) // Go does not use them
+ next = 0;
+ if (save_stack && !SANITIZER_GO) // Go does not use them
creation_stack_id = CurrentStackId(thr, pc);
if (common_flags()->detect_deadlocks)
DDMutexInit(thr, pc, this);
}
-void SyncVar::Reset(Processor *proc) {
- uid = 0;
- creation_stack_id = 0;
+void SyncVar::Reset() {
+ CHECK(!ctx->resetting);
+ creation_stack_id = kInvalidStackID;
owner_tid = kInvalidTid;
- last_lock = 0;
+ last_lock.Reset();
recursion = 0;
atomic_store_relaxed(&flags, 0);
-
- if (proc == 0) {
- CHECK_EQ(clock.size(), 0);
- CHECK_EQ(read_clock.size(), 0);
- } else {
- clock.Reset(&proc->clock_cache);
- read_clock.Reset(&proc->clock_cache);
- }
+ Free(clock);
+ Free(read_clock);
}
MetaMap::MetaMap()
- : block_alloc_(LINKER_INITIALIZED, "heap block allocator"),
- sync_alloc_(LINKER_INITIALIZED, "sync allocator") {
- atomic_store(&uid_gen_, 0, memory_order_relaxed);
-}
+ : block_alloc_("heap block allocator"), sync_alloc_("sync allocator") {}
void MetaMap::AllocBlock(ThreadState *thr, uptr pc, uptr p, uptr sz) {
u32 idx = block_alloc_.Alloc(&thr->proc()->block_cache);
*meta = idx | kFlagBlock;
}
-uptr MetaMap::FreeBlock(Processor *proc, uptr p) {
+uptr MetaMap::FreeBlock(Processor *proc, uptr p, bool reset) {
MBlock* b = GetBlock(p);
if (b == 0)
return 0;
uptr sz = RoundUpTo(b->siz, kMetaShadowCell);
- FreeRange(proc, p, sz);
+ FreeRange(proc, p, sz, reset);
return sz;
}
-bool MetaMap::FreeRange(Processor *proc, uptr p, uptr sz) {
+bool MetaMap::FreeRange(Processor *proc, uptr p, uptr sz, bool reset) {
bool has_something = false;
u32 *meta = MemToMeta(p);
u32 *end = MemToMeta(p + sz);
DCHECK(idx & kFlagSync);
SyncVar *s = sync_alloc_.Map(idx & ~kFlagMask);
u32 next = s->next;
- s->Reset(proc);
+ if (reset)
+ s->Reset();
sync_alloc_.Free(&proc->sync_cache, idx & ~kFlagMask);
idx = next;
} else {
// which can be huge. The function probes pages one-by-one until it finds a page
// without meta objects, at this point it stops freeing meta objects. Because
// thread stacks grow top-down, we do the same starting from end as well.
-void MetaMap::ResetRange(Processor *proc, uptr p, uptr sz) {
+void MetaMap::ResetRange(Processor *proc, uptr p, uptr sz, bool reset) {
if (SANITIZER_GO) {
// UnmapOrDie/MmapFixedNoReserve does not work on Windows,
// so we do the optimization only for C/C++.
- FreeRange(proc, p, sz);
+ FreeRange(proc, p, sz, reset);
return;
}
const uptr kMetaRatio = kMetaShadowCell / kMetaShadowSize;
const uptr kPageSize = GetPageSizeCached() * kMetaRatio;
if (sz <= 4 * kPageSize) {
// If the range is small, just do the normal free procedure.
- FreeRange(proc, p, sz);
+ FreeRange(proc, p, sz, reset);
return;
}
// First, round both ends of the range to page size.
uptr diff = RoundUp(p, kPageSize) - p;
if (diff != 0) {
- FreeRange(proc, p, diff);
+ FreeRange(proc, p, diff, reset);
p += diff;
sz -= diff;
}
diff = p + sz - RoundDown(p + sz, kPageSize);
if (diff != 0) {
- FreeRange(proc, p + sz - diff, diff);
+ FreeRange(proc, p + sz - diff, diff, reset);
sz -= diff;
}
// Now we must have a non-empty page-aligned range.
const uptr sz0 = sz;
// Probe start of the range.
for (uptr checked = 0; sz > 0; checked += kPageSize) {
- bool has_something = FreeRange(proc, p, kPageSize);
+ bool has_something = FreeRange(proc, p, kPageSize, reset);
p += kPageSize;
sz -= kPageSize;
if (!has_something && checked > (128 << 10))
}
// Probe end of the range.
for (uptr checked = 0; sz > 0; checked += kPageSize) {
- bool has_something = FreeRange(proc, p + sz - kPageSize, kPageSize);
+ bool has_something = FreeRange(proc, p + sz - kPageSize, kPageSize, reset);
sz -= kPageSize;
// Stacks grow down, so sync object are most likely at the end of the region
// (if it is a stack). The very end of the stack is TLS and tsan increases
Die();
}
+void MetaMap::ResetClocks() {
+ // This can be called from the background thread
+ // which does not have proc/cache.
+ // The cache is too large for stack.
+ static InternalAllocatorCache cache;
+ internal_memset(&cache, 0, sizeof(cache));
+ internal_allocator()->InitCache(&cache);
+ sync_alloc_.ForEach([&](SyncVar *s) {
+ if (s->clock) {
+ InternalFree(s->clock, &cache);
+ s->clock = nullptr;
+ }
+ if (s->read_clock) {
+ InternalFree(s->read_clock, &cache);
+ s->read_clock = nullptr;
+ }
+ s->last_lock.Reset();
+ });
+ internal_allocator()->DestroyCache(&cache);
+}
+
MBlock* MetaMap::GetBlock(uptr p) {
u32 *meta = MemToMeta(p);
u32 idx = *meta;
}
}
-SyncVar* MetaMap::GetOrCreateAndLock(ThreadState *thr, uptr pc,
- uptr addr, bool write_lock) {
- return GetAndLock(thr, pc, addr, write_lock, true);
-}
-
-SyncVar* MetaMap::GetIfExistsAndLock(uptr addr, bool write_lock) {
- return GetAndLock(0, 0, addr, write_lock, false);
-}
-
-SyncVar *MetaMap::GetAndLock(ThreadState *thr, uptr pc, uptr addr, bool write_lock,
- bool create) NO_THREAD_SAFETY_ANALYSIS {
+SyncVar *MetaMap::GetSync(ThreadState *thr, uptr pc, uptr addr, bool create,
+ bool save_stack) {
+ DCHECK(!create || thr->slot_locked);
u32 *meta = MemToMeta(addr);
u32 idx0 = *meta;
u32 myidx = 0;
- SyncVar *mys = 0;
+ SyncVar *mys = nullptr;
for (;;) {
- u32 idx = idx0;
- for (;;) {
- if (idx == 0)
- break;
- if (idx & kFlagBlock)
- break;
+ for (u32 idx = idx0; idx && !(idx & kFlagBlock);) {
DCHECK(idx & kFlagSync);
SyncVar * s = sync_alloc_.Map(idx & ~kFlagMask);
- if (s->addr == addr) {
- if (myidx != 0) {
- mys->Reset(thr->proc());
+ if (LIKELY(s->addr == addr)) {
+ if (UNLIKELY(myidx != 0)) {
+ mys->Reset();
sync_alloc_.Free(&thr->proc()->sync_cache, myidx);
}
- if (write_lock)
- s->mtx.Lock();
- else
- s->mtx.ReadLock();
return s;
}
idx = s->next;
}
if (!create)
- return 0;
- if (*meta != idx0) {
+ return nullptr;
+ if (UNLIKELY(*meta != idx0)) {
idx0 = *meta;
continue;
}
- if (myidx == 0) {
- const u64 uid = atomic_fetch_add(&uid_gen_, 1, memory_order_relaxed);
+ if (LIKELY(myidx == 0)) {
myidx = sync_alloc_.Alloc(&thr->proc()->sync_cache);
mys = sync_alloc_.Map(myidx);
- mys->Init(thr, pc, addr, uid);
+ mys->Init(thr, pc, addr, save_stack);
}
mys->next = idx0;
if (atomic_compare_exchange_strong((atomic_uint32_t*)meta, &idx0,
myidx | kFlagSync, memory_order_release)) {
- if (write_lock)
- mys->mtx.Lock();
- else
- mys->mtx.ReadLock();
return mys;
}
}
sync_alloc_.FlushCache(&proc->sync_cache);
}
+MetaMap::MemoryStats MetaMap::GetMemoryStats() const {
+ MemoryStats stats;
+ stats.mem_block = block_alloc_.AllocatedMemory();
+ stats.sync_obj = sync_alloc_.AllocatedMemory();
+ return stats;
+}
+
} // namespace __tsan
#include "sanitizer_common/sanitizer_common.h"
#include "sanitizer_common/sanitizer_deadlock_detector_interface.h"
#include "tsan_defs.h"
-#include "tsan_clock.h"
#include "tsan_dense_alloc.h"
+#include "tsan_shadow.h"
+#include "tsan_vector_clock.h"
namespace __tsan {
MutexFlagNotStatic,
};
+// SyncVar is a descriptor of a user synchronization object
+// (mutex or an atomic variable).
struct SyncVar {
SyncVar();
uptr addr; // overwritten by DenseSlabAlloc freelist
Mutex mtx;
- u64 uid; // Globally unique id.
- u32 creation_stack_id;
- u32 owner_tid; // Set only by exclusive owners.
- u64 last_lock;
+ StackID creation_stack_id;
+ Tid owner_tid; // Set only by exclusive owners.
+ FastState last_lock;
int recursion;
atomic_uint32_t flags;
u32 next; // in MetaMap
DDMutex dd;
- SyncClock read_clock; // Used for rw mutexes only.
- // The clock is placed last, so that it is situated on a different cache line
- // with the mtx. This reduces contention for hot sync objects.
- SyncClock clock;
+ VectorClock *read_clock; // Used for rw mutexes only.
+ VectorClock *clock;
- void Init(ThreadState *thr, uptr pc, uptr addr, u64 uid);
- void Reset(Processor *proc);
-
- u64 GetId() const {
- // 48 lsb is addr, then 14 bits is low part of uid, then 2 zero bits.
- return GetLsb((u64)addr | (uid << 48), 60);
- }
- bool CheckId(u64 uid) const {
- CHECK_EQ(uid, GetLsb(uid, 14));
- return GetLsb(this->uid, 14) == uid;
- }
- static uptr SplitId(u64 id, u64 *uid) {
- *uid = id >> 48;
- return (uptr)GetLsb(id, 48);
- }
+ void Init(ThreadState *thr, uptr pc, uptr addr, bool save_stack);
+ void Reset();
bool IsFlagSet(u32 f) const {
return atomic_load_relaxed(&flags) & f;
}
};
-/* MetaMap allows to map arbitrary user pointers onto various descriptors.
- Currently it maps pointers to heap block descriptors and sync var descs.
- It uses 1/2 direct shadow, see tsan_platform.h.
-*/
+// MetaMap maps app addresses to heap block (MBlock) and sync var (SyncVar)
+// descriptors. It uses 1/2 direct shadow, see tsan_platform.h for the mapping.
class MetaMap {
public:
MetaMap();
void AllocBlock(ThreadState *thr, uptr pc, uptr p, uptr sz);
- uptr FreeBlock(Processor *proc, uptr p);
- bool FreeRange(Processor *proc, uptr p, uptr sz);
- void ResetRange(Processor *proc, uptr p, uptr sz);
+
+ // FreeBlock resets all sync objects in the range if reset=true and must not
+ // run concurrently with ResetClocks which resets all sync objects
+ // w/o any synchronization (as part of DoReset).
+ // If we don't have a thread slot (very early/late in thread lifetime or
+ // Go/Java callbacks) or the slot is not locked, then reset must be set to
+ // false. In such case sync object clocks will be reset later (when it's
+ // reused or during the next ResetClocks).
+ uptr FreeBlock(Processor *proc, uptr p, bool reset);
+ bool FreeRange(Processor *proc, uptr p, uptr sz, bool reset);
+ void ResetRange(Processor *proc, uptr p, uptr sz, bool reset);
+ // Reset vector clocks of all sync objects.
+ // Must be called when no other threads access sync objects.
+ void ResetClocks();
MBlock* GetBlock(uptr p);
- SyncVar* GetOrCreateAndLock(ThreadState *thr, uptr pc,
- uptr addr, bool write_lock);
- SyncVar* GetIfExistsAndLock(uptr addr, bool write_lock);
+ SyncVar *GetSyncOrCreate(ThreadState *thr, uptr pc, uptr addr,
+ bool save_stack) {
+ return GetSync(thr, pc, addr, true, save_stack);
+ }
+ SyncVar *GetSyncIfExists(uptr addr) {
+ return GetSync(nullptr, 0, addr, false, false);
+ }
void MoveMemory(uptr src, uptr dst, uptr sz);
void OnProcIdle(Processor *proc);
+ struct MemoryStats {
+ uptr mem_block;
+ uptr sync_obj;
+ };
+
+ MemoryStats GetMemoryStats() const;
+
private:
static const u32 kFlagMask = 3u << 30;
static const u32 kFlagBlock = 1u << 30;
typedef DenseSlabAlloc<SyncVar, 1 << 20, 1 << 10, kFlagMask> SyncAlloc;
BlockAlloc block_alloc_;
SyncAlloc sync_alloc_;
- atomic_uint64_t uid_gen_;
- SyncVar* GetAndLock(ThreadState *thr, uptr pc, uptr addr, bool write_lock,
- bool create);
+ SyncVar *GetSync(ThreadState *thr, uptr pc, uptr addr, bool create,
+ bool save_stack);
};
} // namespace __tsan
#define TSAN_TRACE_H
#include "tsan_defs.h"
-#include "tsan_stack_trace.h"
+#include "tsan_ilist.h"
#include "tsan_mutexset.h"
+#include "tsan_stack_trace.h"
namespace __tsan {
-const int kTracePartSizeBits = 13;
-const int kTracePartSize = 1 << kTracePartSizeBits;
-const int kTraceParts = 2 * 1024 * 1024 / kTracePartSize;
-const int kTraceSize = kTracePartSize * kTraceParts;
-
-// Must fit into 3 bits.
-enum EventType {
- EventTypeMop,
- EventTypeFuncEnter,
- EventTypeFuncExit,
- EventTypeLock,
- EventTypeUnlock,
- EventTypeRLock,
- EventTypeRUnlock
+enum class EventType : u64 {
+ kAccessExt,
+ kAccessRange,
+ kLock,
+ kRLock,
+ kUnlock,
+ kTime,
+};
+
+// "Base" type for all events for type dispatch.
+struct Event {
+ // We use variable-length type encoding to give more bits to some event
+ // types that need them. If is_access is set, this is EventAccess.
+ // Otherwise, if is_func is set, this is EventFunc.
+ // Otherwise type denotes the type.
+ u64 is_access : 1;
+ u64 is_func : 1;
+ EventType type : 3;
+ u64 _ : 59;
+};
+static_assert(sizeof(Event) == 8, "bad Event size");
+
+// Nop event used as padding and does not affect state during replay.
+static constexpr Event NopEvent = {1, 0, EventType::kAccessExt, 0};
+
+// Compressed memory access can represent only some events with PCs
+// close enough to each other. Otherwise we fall back to EventAccessExt.
+struct EventAccess {
+ static constexpr uptr kPCBits = 15;
+ static_assert(kPCBits + kCompressedAddrBits + 5 == 64,
+ "unused bits in EventAccess");
+
+ u64 is_access : 1; // = 1
+ u64 is_read : 1;
+ u64 is_atomic : 1;
+ u64 size_log : 2;
+ u64 pc_delta : kPCBits; // signed delta from the previous memory access PC
+ u64 addr : kCompressedAddrBits;
};
+static_assert(sizeof(EventAccess) == 8, "bad EventAccess size");
-// Represents a thread event (from most significant bit):
-// u64 typ : 3; // EventType.
-// u64 addr : 61; // Associated pc.
-typedef u64 Event;
+// Function entry (pc != 0) or exit (pc == 0).
+struct EventFunc {
+ u64 is_access : 1; // = 0
+ u64 is_func : 1; // = 1
+ u64 pc : 62;
+};
+static_assert(sizeof(EventFunc) == 8, "bad EventFunc size");
+
+// Extended memory access with full PC.
+struct EventAccessExt {
+ // Note: precisely specifying the unused parts of the bitfield is critical for
+ // performance. If we don't specify them, compiler will generate code to load
+ // the old value and shuffle it to extract the unused bits to apply to the new
+ // value. If we specify the unused part and store 0 in there, all that
+ // unnecessary code goes away (store of the 0 const is combined with other
+ // constant parts).
+ static constexpr uptr kUnusedBits = 11;
+ static_assert(kCompressedAddrBits + kUnusedBits + 9 == 64,
+ "unused bits in EventAccessExt");
+
+ u64 is_access : 1; // = 0
+ u64 is_func : 1; // = 0
+ EventType type : 3; // = EventType::kAccessExt
+ u64 is_read : 1;
+ u64 is_atomic : 1;
+ u64 size_log : 2;
+ u64 _ : kUnusedBits;
+ u64 addr : kCompressedAddrBits;
+ u64 pc;
+};
+static_assert(sizeof(EventAccessExt) == 16, "bad EventAccessExt size");
+
+// Access to a memory range.
+struct EventAccessRange {
+ static constexpr uptr kSizeLoBits = 13;
+ static_assert(kCompressedAddrBits + kSizeLoBits + 7 == 64,
+ "unused bits in EventAccessRange");
+
+ u64 is_access : 1; // = 0
+ u64 is_func : 1; // = 0
+ EventType type : 3; // = EventType::kAccessRange
+ u64 is_read : 1;
+ u64 is_free : 1;
+ u64 size_lo : kSizeLoBits;
+ u64 pc : kCompressedAddrBits;
+ u64 addr : kCompressedAddrBits;
+ u64 size_hi : 64 - kCompressedAddrBits;
+};
+static_assert(sizeof(EventAccessRange) == 16, "bad EventAccessRange size");
-const uptr kEventPCBits = 61;
+// Mutex lock.
+struct EventLock {
+ static constexpr uptr kStackIDLoBits = 15;
+ static constexpr uptr kStackIDHiBits =
+ sizeof(StackID) * kByteBits - kStackIDLoBits;
+ static constexpr uptr kUnusedBits = 3;
+ static_assert(kCompressedAddrBits + kStackIDLoBits + 5 == 64,
+ "unused bits in EventLock");
+ static_assert(kCompressedAddrBits + kStackIDHiBits + kUnusedBits == 64,
+ "unused bits in EventLock");
+
+ u64 is_access : 1; // = 0
+ u64 is_func : 1; // = 0
+ EventType type : 3; // = EventType::kLock or EventType::kRLock
+ u64 pc : kCompressedAddrBits;
+ u64 stack_lo : kStackIDLoBits;
+ u64 stack_hi : sizeof(StackID) * kByteBits - kStackIDLoBits;
+ u64 _ : kUnusedBits;
+ u64 addr : kCompressedAddrBits;
+};
+static_assert(sizeof(EventLock) == 16, "bad EventLock size");
+
+// Mutex unlock.
+struct EventUnlock {
+ static constexpr uptr kUnusedBits = 15;
+ static_assert(kCompressedAddrBits + kUnusedBits + 5 == 64,
+ "unused bits in EventUnlock");
+
+ u64 is_access : 1; // = 0
+ u64 is_func : 1; // = 0
+ EventType type : 3; // = EventType::kUnlock
+ u64 _ : kUnusedBits;
+ u64 addr : kCompressedAddrBits;
+};
+static_assert(sizeof(EventUnlock) == 8, "bad EventUnlock size");
+
+// Time change event.
+struct EventTime {
+ static constexpr uptr kUnusedBits = 37;
+ static_assert(kUnusedBits + sizeof(Sid) * kByteBits + kEpochBits + 5 == 64,
+ "unused bits in EventTime");
+
+ u64 is_access : 1; // = 0
+ u64 is_func : 1; // = 0
+ EventType type : 3; // = EventType::kTime
+ u64 sid : sizeof(Sid) * kByteBits;
+ u64 epoch : kEpochBits;
+ u64 _ : kUnusedBits;
+};
+static_assert(sizeof(EventTime) == 8, "bad EventTime size");
+
+struct Trace;
struct TraceHeader {
-#if !SANITIZER_GO
- BufferedStackTrace stack0; // Start stack for the trace.
-#else
- VarSizeStackTrace stack0;
-#endif
- u64 epoch0; // Start epoch for the trace.
- MutexSet mset0;
-
- TraceHeader() : stack0(), epoch0() {}
+ Trace* trace = nullptr; // back-pointer to Trace containing this part
+ INode trace_parts; // in Trace::parts
+ INode global; // in Contex::trace_part_recycle
};
+struct TracePart : TraceHeader {
+ // There are a lot of goroutines in Go, so we use smaller parts.
+ static constexpr uptr kByteSize = (SANITIZER_GO ? 128 : 256) << 10;
+ static constexpr uptr kSize =
+ (kByteSize - sizeof(TraceHeader)) / sizeof(Event);
+ // TraceAcquire does a fast event pointer overflow check by comparing
+ // pointer into TracePart::events with kAlignment mask. Since TracePart's
+ // are allocated page-aligned, this check detects end of the array
+ // (it also have false positives in the middle that are filtered separately).
+ // This also requires events to be the last field.
+ static constexpr uptr kAlignment = 0xff0;
+ Event events[kSize];
+
+ TracePart() {}
+};
+static_assert(sizeof(TracePart) == TracePart::kByteSize, "bad TracePart size");
+
struct Trace {
Mutex mtx;
-#if !SANITIZER_GO
- // Must be last to catch overflow as paging fault.
- // Go shadow stack is dynamically allocated.
- uptr shadow_stack[kShadowStackSize];
-#endif
- // Must be the last field, because we unmap the unused part in
- // CreateThreadContext.
- TraceHeader headers[kTraceParts];
+ IList<TraceHeader, &TraceHeader::trace_parts, TracePart> parts;
+ // First node non-queued into ctx->trace_part_recycle.
+ TracePart* local_head;
+ // Final position in the last part for finished threads.
+ Event* final_pos = nullptr;
+ // Number of trace parts allocated on behalf of this trace specifically.
+ // Total number of parts in this trace can be larger if we retake some
+ // parts from other traces.
+ uptr parts_allocated = 0;
Trace() : mtx(MutexTypeTrace) {}
+
+ // We need at least 3 parts per thread, because we want to keep at last
+ // 2 parts per thread that are not queued into ctx->trace_part_recycle
+ // (the current one being filled and one full part that ensures that
+ // we always have at least one part worth of previous memory accesses).
+ static constexpr uptr kMinParts = 3;
+
+ static constexpr uptr kFinishedThreadLo = 16;
+ static constexpr uptr kFinishedThreadHi = 64;
};
} // namespace __tsan
--- /dev/null
+//===-- tsan_vector_clock.cpp ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#include "tsan_vector_clock.h"
+
+#include "sanitizer_common/sanitizer_placement_new.h"
+#include "tsan_mman.h"
+
+namespace __tsan {
+
+#if TSAN_VECTORIZE
+const uptr kVectorClockSize = kThreadSlotCount * sizeof(Epoch) / sizeof(m128);
+#endif
+
+VectorClock::VectorClock() { Reset(); }
+
+void VectorClock::Reset() {
+#if !TSAN_VECTORIZE
+ for (uptr i = 0; i < kThreadSlotCount; i++)
+ clk_[i] = kEpochZero;
+#else
+ m128 z = _mm_setzero_si128();
+ m128* vclk = reinterpret_cast<m128*>(clk_);
+ for (uptr i = 0; i < kVectorClockSize; i++) _mm_store_si128(&vclk[i], z);
+#endif
+}
+
+void VectorClock::Acquire(const VectorClock* src) {
+ if (!src)
+ return;
+#if !TSAN_VECTORIZE
+ for (uptr i = 0; i < kThreadSlotCount; i++)
+ clk_[i] = max(clk_[i], src->clk_[i]);
+#else
+ m128* __restrict vdst = reinterpret_cast<m128*>(clk_);
+ m128 const* __restrict vsrc = reinterpret_cast<m128 const*>(src->clk_);
+ for (uptr i = 0; i < kVectorClockSize; i++) {
+ m128 s = _mm_load_si128(&vsrc[i]);
+ m128 d = _mm_load_si128(&vdst[i]);
+ m128 m = _mm_max_epu16(s, d);
+ _mm_store_si128(&vdst[i], m);
+ }
+#endif
+}
+
+static VectorClock* AllocClock(VectorClock** dstp) {
+ if (UNLIKELY(!*dstp))
+ *dstp = New<VectorClock>();
+ return *dstp;
+}
+
+void VectorClock::Release(VectorClock** dstp) const {
+ VectorClock* dst = AllocClock(dstp);
+ dst->Acquire(this);
+}
+
+void VectorClock::ReleaseStore(VectorClock** dstp) const {
+ VectorClock* dst = AllocClock(dstp);
+ *dst = *this;
+}
+
+VectorClock& VectorClock::operator=(const VectorClock& other) {
+#if !TSAN_VECTORIZE
+ for (uptr i = 0; i < kThreadSlotCount; i++)
+ clk_[i] = other.clk_[i];
+#else
+ m128* __restrict vdst = reinterpret_cast<m128*>(clk_);
+ m128 const* __restrict vsrc = reinterpret_cast<m128 const*>(other.clk_);
+ for (uptr i = 0; i < kVectorClockSize; i++) {
+ m128 s = _mm_load_si128(&vsrc[i]);
+ _mm_store_si128(&vdst[i], s);
+ }
+#endif
+ return *this;
+}
+
+void VectorClock::ReleaseStoreAcquire(VectorClock** dstp) {
+ VectorClock* dst = AllocClock(dstp);
+#if !TSAN_VECTORIZE
+ for (uptr i = 0; i < kThreadSlotCount; i++) {
+ Epoch tmp = dst->clk_[i];
+ dst->clk_[i] = clk_[i];
+ clk_[i] = max(clk_[i], tmp);
+ }
+#else
+ m128* __restrict vdst = reinterpret_cast<m128*>(dst->clk_);
+ m128* __restrict vclk = reinterpret_cast<m128*>(clk_);
+ for (uptr i = 0; i < kVectorClockSize; i++) {
+ m128 t = _mm_load_si128(&vdst[i]);
+ m128 c = _mm_load_si128(&vclk[i]);
+ m128 m = _mm_max_epu16(c, t);
+ _mm_store_si128(&vdst[i], c);
+ _mm_store_si128(&vclk[i], m);
+ }
+#endif
+}
+
+void VectorClock::ReleaseAcquire(VectorClock** dstp) {
+ VectorClock* dst = AllocClock(dstp);
+#if !TSAN_VECTORIZE
+ for (uptr i = 0; i < kThreadSlotCount; i++) {
+ dst->clk_[i] = max(dst->clk_[i], clk_[i]);
+ clk_[i] = dst->clk_[i];
+ }
+#else
+ m128* __restrict vdst = reinterpret_cast<m128*>(dst->clk_);
+ m128* __restrict vclk = reinterpret_cast<m128*>(clk_);
+ for (uptr i = 0; i < kVectorClockSize; i++) {
+ m128 c = _mm_load_si128(&vclk[i]);
+ m128 d = _mm_load_si128(&vdst[i]);
+ m128 m = _mm_max_epu16(c, d);
+ _mm_store_si128(&vdst[i], m);
+ _mm_store_si128(&vclk[i], m);
+ }
+#endif
+}
+
+} // namespace __tsan
--- /dev/null
+//===-- tsan_vector_clock.h -------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#ifndef TSAN_VECTOR_CLOCK_H
+#define TSAN_VECTOR_CLOCK_H
+
+#include "tsan_defs.h"
+
+namespace __tsan {
+
+// Fixed-size vector clock, used both for threads and sync objects.
+class VectorClock {
+ public:
+ VectorClock();
+
+ Epoch Get(Sid sid) const;
+ void Set(Sid sid, Epoch v);
+
+ void Reset();
+ void Acquire(const VectorClock* src);
+ void Release(VectorClock** dstp) const;
+ void ReleaseStore(VectorClock** dstp) const;
+ void ReleaseStoreAcquire(VectorClock** dstp);
+ void ReleaseAcquire(VectorClock** dstp);
+
+ VectorClock& operator=(const VectorClock& other);
+
+ private:
+ Epoch clk_[kThreadSlotCount] VECTOR_ALIGNED;
+};
+
+ALWAYS_INLINE Epoch VectorClock::Get(Sid sid) const {
+ return clk_[static_cast<u8>(sid)];
+}
+
+ALWAYS_INLINE void VectorClock::Set(Sid sid, Epoch v) {
+ DCHECK_GE(v, clk_[static_cast<u8>(sid)]);
+ clk_[static_cast<u8>(sid)] = v;
+}
+
+} // namespace __tsan
+
+#endif // TSAN_VECTOR_CLOCK_H
set(TSAN_UNITTEST_CFLAGS
${COMPILER_RT_UNITTEST_CFLAGS}
${COMPILER_RT_GTEST_CFLAGS}
+ ${SANITIZER_TEST_CXX_CFLAGS}
-I${COMPILER_RT_SOURCE_DIR}/include
-I${COMPILER_RT_SOURCE_DIR}/lib
-I${COMPILER_RT_SOURCE_DIR}/lib/tsan/rtl
-DTSAN_DEBUG_OUTPUT=2)
endif()
+append_list_if(COMPILER_RT_HAS_MSSE4_2_FLAG -msse4.2 TSAN_UNITTEST_CFLAGS)
+
set(TSAN_TEST_ARCH ${TSAN_SUPPORTED_ARCH})
-set(LINK_FLAGS ${COMPILER_RT_UNITTEST_LINK_FLAGS})
-foreach(lib ${SANITIZER_TEST_CXX_LIBRARIES})
- list(APPEND LINK_FLAGS -l${lib})
-endforeach()
+set(TSAN_UNITTEST_LINK_FLAGS
+ ${COMPILER_RT_UNITTEST_LINK_FLAGS}
+ ${COMPILER_RT_UNWINDER_LINK_LIBS}
+ ${SANITIZER_TEST_CXX_LIBRARIES})
if(APPLE)
darwin_filter_host_archs(TSAN_SUPPORTED_ARCH TSAN_TEST_ARCH)
list(APPEND TSAN_UNITTEST_CFLAGS ${DARWIN_osx_CFLAGS})
- list(APPEND LINK_FLAGS ${DARWIN_osx_LINK_FLAGS})
- add_weak_symbols("ubsan" LINK_FLAGS)
- add_weak_symbols("sanitizer_common" LINK_FLAGS)
+ list(APPEND TSAN_UNITTEST_LINK_FLAGS ${DARWIN_osx_LINK_FLAGS})
+ add_weak_symbols("ubsan" TSAN_UNITTEST_LINK_FLAGS)
+ add_weak_symbols("sanitizer_common" TSAN_UNITTEST_LINK_FLAGS)
else()
- list(APPEND LINK_FLAGS -fsanitize=thread)
- list(APPEND LINK_FLAGS -lm)
- list(APPEND LINK_FLAGS ${COMPILER_RT_TEST_LIBDISPATCH_CFLAGS})
+ list(APPEND TSAN_UNITTEST_LINK_FLAGS -fsanitize=thread)
+ list(APPEND TSAN_UNITTEST_LINK_FLAGS -lm)
+ list(APPEND TSAN_UNITTEST_LINK_FLAGS ${COMPILER_RT_TEST_LIBDISPATCH_CFLAGS})
endif()
set(TSAN_RTL_HEADERS)
list(APPEND TSAN_RTL_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/../${header})
endforeach()
-set(TSAN_DEPS gtest tsan)
+set(TSAN_DEPS llvm_gtest tsan)
# TSan uses C++ standard library headers.
if (TARGET cxx-headers OR HAVE_LIBCXX)
set(TSAN_DEPS cxx-headers)
COMPILE_DEPS ${TEST_HEADERS} ${TSAN_RTL_HEADERS}
DEPS ${TSAN_DEPS}
CFLAGS ${TSAN_UNITTEST_CFLAGS}
- LINK_FLAGS ${LINK_FLAGS})
+ LINK_FLAGS ${TSAN_UNITTEST_LINK_FLAGS})
endforeach()
endif()
endmacro()
#include <stddef.h>
#include <stdint.h>
-TEST(ThreadSanitizer, SimpleWrite) {
+TEST_F(ThreadSanitizer, SimpleWrite) {
ScopedThread t;
MemLoc l;
t.Write1(l);
}
-TEST(ThreadSanitizer, SimpleWriteWrite) {
+TEST_F(ThreadSanitizer, SimpleWriteWrite) {
ScopedThread t1, t2;
MemLoc l1, l2;
t1.Write1(l1);
t2.Write1(l2);
}
-TEST(ThreadSanitizer, WriteWriteRace) {
+TEST_F(ThreadSanitizer, WriteWriteRace) {
ScopedThread t1, t2;
MemLoc l;
t1.Write1(l);
t2.Write1(l, true);
}
-TEST(ThreadSanitizer, ReadWriteRace) {
+TEST_F(ThreadSanitizer, ReadWriteRace) {
ScopedThread t1, t2;
MemLoc l;
t1.Read1(l);
t2.Write1(l, true);
}
-TEST(ThreadSanitizer, WriteReadRace) {
+TEST_F(ThreadSanitizer, WriteReadRace) {
ScopedThread t1, t2;
MemLoc l;
t1.Write1(l);
t2.Read1(l, true);
}
-TEST(ThreadSanitizer, ReadReadNoRace) {
+TEST_F(ThreadSanitizer, ReadReadNoRace) {
ScopedThread t1, t2;
MemLoc l;
t1.Read1(l);
t2.Read1(l);
}
-TEST(ThreadSanitizer, WriteThenRead) {
+TEST_F(ThreadSanitizer, WriteThenRead) {
MemLoc l;
ScopedThread t1, t2;
t1.Write1(l);
t2.Read1(l, true);
}
-TEST(ThreadSanitizer, WriteThenLockedRead) {
+TEST_F(ThreadSanitizer, WriteThenLockedRead) {
UserMutex m(UserMutex::RW);
MainThread t0;
t0.Create(m);
t0.Destroy(m);
}
-TEST(ThreadSanitizer, LockedWriteThenRead) {
+TEST_F(ThreadSanitizer, LockedWriteThenRead) {
UserMutex m(UserMutex::RW);
MainThread t0;
t0.Create(m);
}
-TEST(ThreadSanitizer, RaceWithOffset) {
+TEST_F(ThreadSanitizer, RaceWithOffset) {
ScopedThread t1, t2;
{
MemLoc l;
}
}
-TEST(ThreadSanitizer, RaceWithOffset2) {
+TEST_F(ThreadSanitizer, RaceWithOffset2) {
ScopedThread t1, t2;
{
MemLoc l;
}
}
-TEST(ThreadSanitizer, NoRaceWithOffset) {
+TEST_F(ThreadSanitizer, NoRaceWithOffset) {
ScopedThread t1, t2;
{
MemLoc l;
}
}
-TEST(ThreadSanitizer, RaceWithDeadThread) {
+TEST_F(ThreadSanitizer, RaceWithDeadThread) {
MemLoc l;
ScopedThread t;
ScopedThread().Write1(l);
t.Write1(l, true);
}
-TEST(ThreadSanitizer, BenignRaceOnVptr) {
+TEST_F(ThreadSanitizer, BenignRaceOnVptr) {
void *vptr_storage;
MemLoc vptr(&vptr_storage), val;
vptr_storage = val.loc();
t2.Read8(vptr);
}
-TEST(ThreadSanitizer, HarmfulRaceOnVptr) {
+TEST_F(ThreadSanitizer, HarmfulRaceOnVptr) {
void *vptr_storage;
MemLoc vptr(&vptr_storage), val1, val2;
vptr_storage = val1.loc();
(void)x2;
}
-TEST(ThreadSanitizer, ReportDeadThread) {
+TEST_F(ThreadSanitizer, ReportDeadThread) {
MemLoc l;
ScopedThread t1;
{
static void foobarbaz() {}
-TEST(ThreadSanitizer, ReportRace) {
+TEST_F(ThreadSanitizer, ReportRace) {
ScopedThread t1;
MainThread().Access(&ClassWithStatic::Data, true, 4, false);
t1.Call(&foobarbaz);
namespace __tsan {
-TEST(ThreadSanitizer, BasicMutex) {
+TEST_F(ThreadSanitizer, BasicMutex) {
ScopedThread t;
UserMutex m;
t.Create(m);
t.Destroy(m);
}
-TEST(ThreadSanitizer, BasicSpinMutex) {
+TEST_F(ThreadSanitizer, BasicSpinMutex) {
ScopedThread t;
UserMutex m(UserMutex::Spin);
t.Create(m);
t.Destroy(m);
}
-TEST(ThreadSanitizer, BasicRwMutex) {
+TEST_F(ThreadSanitizer, BasicRwMutex) {
ScopedThread t;
UserMutex m(UserMutex::RW);
t.Create(m);
t.Destroy(m);
}
-TEST(ThreadSanitizer, Mutex) {
+TEST_F(ThreadSanitizer, Mutex) {
UserMutex m;
MainThread t0;
t0.Create(m);
t2.Destroy(m);
}
-TEST(ThreadSanitizer, SpinMutex) {
+TEST_F(ThreadSanitizer, SpinMutex) {
UserMutex m(UserMutex::Spin);
MainThread t0;
t0.Create(m);
t2.Destroy(m);
}
-TEST(ThreadSanitizer, RwMutex) {
+TEST_F(ThreadSanitizer, RwMutex) {
UserMutex m(UserMutex::RW);
MainThread t0;
t0.Create(m);
t2.Destroy(m);
}
-TEST(ThreadSanitizer, StaticMutex) {
+TEST_F(ThreadSanitizer, StaticMutex) {
// Emulates statically initialized mutex.
UserMutex m;
m.StaticInit();
namespace __tsan {
-TEST(ThreadSanitizer, Memcpy) {
+TEST_F(ThreadSanitizer, Memcpy) {
char data0[7] = {1, 2, 3, 4, 5, 6, 7};
char data[7] = {42, 42, 42, 42, 42, 42, 42};
MainThread().Memcpy(data+1, data0+1, 5);
EXPECT_EQ(data[6], 42);
}
-TEST(ThreadSanitizer, MemcpyRace1) {
+TEST_F(ThreadSanitizer, MemcpyRace1) {
char *data = new char[10];
char *data1 = new char[10];
char *data2 = new char[10];
t2.Memcpy(data, data2, 10, true);
}
-TEST(ThreadSanitizer, MemcpyRace2) {
+TEST_F(ThreadSanitizer, MemcpyRace2) {
char *data = new char[10];
char *data1 = new char[10];
char *data2 = new char[10];
t2.Memcpy(data+3, data2, 4, true);
}
-TEST(ThreadSanitizer, MemcpyRace3) {
+TEST_F(ThreadSanitizer, MemcpyRace3) {
char *data = new char[10];
char *data1 = new char[10];
char *data2 = new char[10];
t2.Memcpy(data1, data2, 10, true);
}
-TEST(ThreadSanitizer, MemcpyStack) {
+TEST_F(ThreadSanitizer, MemcpyStack) {
char *data = new char[10];
char *data1 = new char[10];
ScopedThread t1, t2;
t2.Memcpy(data, data1, 10, true);
}
-TEST(ThreadSanitizer, MemsetRace1) {
+TEST_F(ThreadSanitizer, MemsetRace1) {
char *data = new char[10];
ScopedThread t1, t2;
t1.Memset(data, 1, 10);
static void foo() {}
static void bar() {}
-TEST(ThreadSanitizer, FuncCall) {
+TEST_F(ThreadSanitizer, FuncCall) {
ScopedThread t1, t2;
MemLoc l;
t1.Write1(l);
}
#endif
-namespace __sanitizer {
-bool ReexecDisabled() {
- return true;
-}
-}
-
int main(int argc, char **argv) {
argv0 = argv[0];
return run_tests(argc, argv);
#ifndef TSAN_TEST_UTIL_H
#define TSAN_TEST_UTIL_H
+#include "gtest/gtest.h"
+
+class ThreadSanitizer : public ::testing::Test {
+ protected:
+ void TearDown() override;
+};
+
void TestMutexBeforeInit();
// A location of memory on which a race may be detected.
#include "sanitizer_common/sanitizer_atomic.h"
#include "tsan_interface.h"
#include "tsan_posix_util.h"
+#include "tsan_rtl.h"
#include "tsan_test_util.h"
#include "tsan_report.h"
-#include "gtest/gtest.h"
-
#include <assert.h>
#include <pthread.h>
#include <stdio.h>
#define CALLERPC (__builtin_return_address(0))
-using namespace __tsan;
-
static __thread bool expect_report;
static __thread bool expect_report_reported;
-static __thread ReportType expect_report_type;
+static __thread __tsan::ReportType expect_report_type;
+
+void ThreadSanitizer::TearDown() {
+ __tsan::ctx->racy_stacks.Reset();
+}
static void *BeforeInitThread(void *param) {
(void)param;
static void* allocate_addr(int size, int offset_from_aligned = 0) {
static uintptr_t foo;
- static atomic_uintptr_t uniq = {(uintptr_t)&foo}; // Some real address.
+ static __tsan::atomic_uintptr_t uniq = {(uintptr_t)&foo}; // Some real address.
const int kAlign = 16;
CHECK(offset_from_aligned < kAlign);
size = (size + 2 * kAlign) & ~(kAlign - 1);
- uintptr_t addr = atomic_fetch_add(&uniq, size, memory_order_relaxed);
+ uintptr_t addr = atomic_fetch_add(&uniq, size, __tsan::memory_order_relaxed);
return (void*)(addr + offset_from_aligned);
}
uptr arg2;
bool res;
bool expect_report;
- ReportType report_type;
+ __tsan::ReportType report_type;
explicit Event(Type type, const void *ptr = 0, uptr arg = 0, uptr arg2 = 0)
: type(type),
expect_report(),
report_type() {}
- void ExpectReport(ReportType type) {
+ void ExpectReport(__tsan::ReportType type) {
expect_report = true;
report_type = type;
}
pthread_t thread;
bool main;
bool detached;
- atomic_uintptr_t event; // Event*
+ __tsan::atomic_uintptr_t event; // Event*
static void *ScopedThreadCallback(void *arg);
void send(Event *ev);
__tsan_func_entry(CALLERPC);
Impl *impl = (Impl*)arg;
for (;;) {
- Event* ev = (Event*)atomic_load(&impl->event, memory_order_acquire);
+ Event *ev =
+ (Event *)atomic_load(&impl->event, __tsan::memory_order_acquire);
if (ev == 0) {
sched_yield();
continue;
}
if (ev->type == Event::SHUTDOWN) {
- atomic_store(&impl->event, 0, memory_order_release);
+ atomic_store(&impl->event, 0, __tsan::memory_order_release);
break;
}
impl->HandleEvent(ev);
- atomic_store(&impl->event, 0, memory_order_release);
+ atomic_store(&impl->event, 0, __tsan::memory_order_release);
}
__tsan_func_exit();
return 0;
if (main) {
HandleEvent(e);
} else {
- CHECK_EQ(atomic_load(&event, memory_order_relaxed), 0);
- atomic_store(&event, (uintptr_t)e, memory_order_release);
- while (atomic_load(&event, memory_order_acquire) != 0)
+ CHECK_EQ(atomic_load(&event, __tsan::memory_order_relaxed), 0);
+ atomic_store(&event, (uintptr_t)e, __tsan::memory_order_release);
+ while (atomic_load(&event, __tsan::memory_order_acquire) != 0)
sched_yield();
}
}
impl_ = new Impl;
impl_->main = main;
impl_->detached = detached;
- atomic_store(&impl_->event, 0, memory_order_relaxed);
+ atomic_store(&impl_->event, 0, __tsan::memory_order_relaxed);
if (!main) {
pthread_attr_t attr;
pthread_attr_init(&attr);
Event event(is_write ? Event::WRITE : Event::READ, addr, size,
(uptr)CALLERPC);
if (expect_race)
- event.ExpectReport(ReportTypeRace);
+ event.ExpectReport(__tsan::ReportTypeRace);
impl_->send(&event);
}
bool expect_race) {
Event event(Event::VPTR_UPDATE, vptr.loc(), (uptr)new_val.loc());
if (expect_race)
- event.ExpectReport(ReportTypeRace);
+ event.ExpectReport(__tsan::ReportTypeRace);
impl_->send(&event);
}
bool expect_race) {
Event event(Event::MEMCPY, dst, (uptr)src, size);
if (expect_race)
- event.ExpectReport(ReportTypeRace);
+ event.ExpectReport(__tsan::ReportTypeRace);
impl_->send(&event);
}
bool expect_race) {
Event event(Event::MEMSET, dst, val, size);
if (expect_race)
- event.ExpectReport(ReportTypeRace);
+ event.ExpectReport(__tsan::ReportTypeRace);
impl_->send(&event);
}
#include "tsan_test_util.h"
#include "gtest/gtest.h"
-TEST(ThreadSanitizer, ThreadSync) {
+TEST_F(ThreadSanitizer, ThreadSync) {
MainThread t0;
MemLoc l;
t0.Write1(l);
t0.Write1(l);
}
-TEST(ThreadSanitizer, ThreadDetach1) {
+TEST_F(ThreadSanitizer, ThreadDetach1) {
ScopedThread t1(true);
MemLoc l;
t1.Write1(l);
}
-TEST(ThreadSanitizer, ThreadDetach2) {
+TEST_F(ThreadSanitizer, ThreadDetach2) {
ScopedThread t1;
MemLoc l;
t1.Write1(l);
set(TSAN_UNIT_TEST_SOURCES
- tsan_clock_test.cpp
tsan_dense_alloc_test.cpp
tsan_flags_test.cpp
+ tsan_ilist_test.cpp
tsan_mman_test.cpp
tsan_shadow_test.cpp
tsan_stack_test.cpp
tsan_sync_test.cpp
+ tsan_trace_test.cpp
tsan_unit_test_main.cpp
+ tsan_vector_clock_test.cpp
)
add_tsan_unittest(TsanUnitTest
static const char *options1 =
" enable_annotations=0"
" suppress_equal_stacks=0"
- " suppress_equal_addresses=0"
" report_bugs=0"
" report_thread_leaks=0"
" report_destroy_locked=0"
" report_signal_unsafe=0"
" report_atomic_races=0"
" force_seq_cst_atomics=0"
- " print_benign=0"
" halt_on_error=0"
" atexit_sleep_ms=222"
" profile_memory=qqq"
static const char *options2 =
" enable_annotations=true"
" suppress_equal_stacks=true"
- " suppress_equal_addresses=true"
" report_bugs=true"
" report_thread_leaks=true"
" report_destroy_locked=true"
" report_signal_unsafe=true"
" report_atomic_races=true"
" force_seq_cst_atomics=true"
- " print_benign=true"
" halt_on_error=true"
" atexit_sleep_ms=123"
" profile_memory=bbbbb"
void VerifyOptions1(Flags *f) {
EXPECT_EQ(f->enable_annotations, 0);
EXPECT_EQ(f->suppress_equal_stacks, 0);
- EXPECT_EQ(f->suppress_equal_addresses, 0);
EXPECT_EQ(f->report_bugs, 0);
EXPECT_EQ(f->report_thread_leaks, 0);
EXPECT_EQ(f->report_destroy_locked, 0);
EXPECT_EQ(f->report_signal_unsafe, 0);
EXPECT_EQ(f->report_atomic_races, 0);
EXPECT_EQ(f->force_seq_cst_atomics, 0);
- EXPECT_EQ(f->print_benign, 0);
EXPECT_EQ(f->halt_on_error, 0);
EXPECT_EQ(f->atexit_sleep_ms, 222);
EXPECT_EQ(f->profile_memory, std::string("qqq"));
EXPECT_EQ(f->memory_limit_mb, 666);
EXPECT_EQ(f->stop_on_start, 0);
EXPECT_EQ(f->running_on_valgrind, 0);
- EXPECT_EQ(f->history_size, 5);
+ EXPECT_EQ(f->history_size, (uptr)5);
EXPECT_EQ(f->io_sync, 1);
EXPECT_EQ(f->die_after_fork, true);
}
void VerifyOptions2(Flags *f) {
EXPECT_EQ(f->enable_annotations, true);
EXPECT_EQ(f->suppress_equal_stacks, true);
- EXPECT_EQ(f->suppress_equal_addresses, true);
EXPECT_EQ(f->report_bugs, true);
EXPECT_EQ(f->report_thread_leaks, true);
EXPECT_EQ(f->report_destroy_locked, true);
EXPECT_EQ(f->report_signal_unsafe, true);
EXPECT_EQ(f->report_atomic_races, true);
EXPECT_EQ(f->force_seq_cst_atomics, true);
- EXPECT_EQ(f->print_benign, true);
EXPECT_EQ(f->halt_on_error, true);
EXPECT_EQ(f->atexit_sleep_ms, 123);
EXPECT_EQ(f->profile_memory, std::string("bbbbb"));
EXPECT_EQ(f->memory_limit_mb, 456);
EXPECT_EQ(f->stop_on_start, true);
EXPECT_EQ(f->running_on_valgrind, true);
- EXPECT_EQ(f->history_size, 6);
+ EXPECT_EQ(f->history_size, 6ul);
EXPECT_EQ(f->io_sync, 2);
EXPECT_EQ(f->die_after_fork, false);
}
--- /dev/null
+//===-- tsan_ilist_test.cpp -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#include "tsan_ilist.h"
+
+#include "gtest/gtest.h"
+
+namespace __tsan {
+
+struct Node {
+ INode node1;
+ INode node2;
+};
+
+struct Parent : Node {};
+
+TEST(IList, Empty) {
+ IList<Node, &Node::node1> list;
+ Node node;
+
+ EXPECT_TRUE(list.Empty());
+ EXPECT_EQ(list.Size(), (size_t)0);
+ EXPECT_EQ(list.Back(), nullptr);
+ EXPECT_EQ(list.Front(), nullptr);
+ EXPECT_EQ(list.PopBack(), nullptr);
+ EXPECT_EQ(list.PopFront(), nullptr);
+ EXPECT_FALSE(list.Queued(&node));
+}
+
+TEST(IList, OneNode) {
+ IList<Node, &Node::node1> list;
+ Node node;
+
+ list.PushBack(&node);
+ EXPECT_FALSE(list.Empty());
+ EXPECT_EQ(list.Size(), (size_t)1);
+ EXPECT_EQ(list.Back(), &node);
+ EXPECT_EQ(list.Front(), &node);
+ EXPECT_TRUE(list.Queued(&node));
+ EXPECT_EQ(list.Prev(&node), nullptr);
+ EXPECT_EQ(list.Next(&node), nullptr);
+
+ EXPECT_EQ(list.PopFront(), &node);
+ EXPECT_TRUE(list.Empty());
+ EXPECT_EQ(list.Size(), (size_t)0);
+ EXPECT_FALSE(list.Queued(&node));
+}
+
+TEST(IList, MultipleNodes) {
+ IList<Node, &Node::node1> list;
+ Node nodes[3];
+
+ list.PushBack(&nodes[1]);
+ list.PushBack(&nodes[0]);
+ list.PushFront(&nodes[2]);
+
+ EXPECT_EQ(list.Size(), (size_t)3);
+ EXPECT_EQ(list.Back(), &nodes[0]);
+ EXPECT_EQ(list.Front(), &nodes[2]);
+
+ EXPECT_EQ(list.Next(&nodes[0]), nullptr);
+ EXPECT_EQ(list.Prev(&nodes[0]), &nodes[1]);
+
+ EXPECT_EQ(list.Next(&nodes[1]), &nodes[0]);
+ EXPECT_EQ(list.Prev(&nodes[1]), &nodes[2]);
+
+ EXPECT_EQ(list.Next(&nodes[2]), &nodes[1]);
+ EXPECT_EQ(list.Prev(&nodes[2]), nullptr);
+
+ EXPECT_EQ(list.PopBack(), &nodes[0]);
+ EXPECT_EQ(list.PopFront(), &nodes[2]);
+ EXPECT_EQ(list.PopFront(), &nodes[1]);
+ EXPECT_TRUE(list.Empty());
+}
+
+TEST(IList, TwoLists) {
+ IList<Node, &Node::node1> list1;
+ IList<Node, &Node::node2, Parent> list2;
+ Parent nodes[3];
+
+ list1.PushBack(&nodes[2]);
+ list1.PushBack(&nodes[1]);
+ list1.PushBack(&nodes[0]);
+
+ list2.PushFront(&nodes[1]);
+
+ EXPECT_EQ(list1.Size(), (size_t)3);
+ EXPECT_TRUE(list1.Queued(&nodes[0]));
+ EXPECT_TRUE(list1.Queued(&nodes[1]));
+ EXPECT_TRUE(list1.Queued(&nodes[2]));
+
+ EXPECT_EQ(list2.Size(), (size_t)1);
+ EXPECT_FALSE(list2.Queued(&nodes[0]));
+ EXPECT_TRUE(list2.Queued(&nodes[1]));
+ EXPECT_FALSE(list2.Queued(&nodes[2]));
+
+ EXPECT_EQ(list1.Next(&nodes[1]), &nodes[0]);
+ EXPECT_EQ(list1.Prev(&nodes[1]), &nodes[2]);
+
+ EXPECT_EQ(list2.Next(&nodes[1]), nullptr);
+ EXPECT_EQ(list2.Prev(&nodes[1]), nullptr);
+
+ list1.Remove(&nodes[1]);
+ EXPECT_EQ(list1.Size(), (size_t)2);
+ EXPECT_FALSE(list1.Queued(&nodes[1]));
+ EXPECT_EQ(list2.Size(), (size_t)1);
+ EXPECT_TRUE(list2.Queued(&nodes[1]));
+
+ EXPECT_EQ(list1.PopBack(), &nodes[0]);
+ EXPECT_EQ(list1.PopBack(), &nodes[2]);
+ EXPECT_EQ(list1.Size(), (size_t)0);
+
+ EXPECT_EQ(list2.PopBack(), &nodes[1]);
+ EXPECT_EQ(list2.Size(), (size_t)0);
+}
+
+} // namespace __tsan
namespace __tsan {
TEST(Mman, Internal) {
- char *p = (char*)internal_alloc(MBlockScopedBuf, 10);
+ char *p = (char *)Alloc(10);
EXPECT_NE(p, (char*)0);
- char *p2 = (char*)internal_alloc(MBlockScopedBuf, 20);
+ char *p2 = (char *)Alloc(20);
EXPECT_NE(p2, (char*)0);
EXPECT_NE(p2, p);
for (int i = 0; i < 10; i++) {
for (int i = 0; i < 20; i++) {
((char*)p2)[i] = 42;
}
- internal_free(p);
- internal_free(p2);
+ Free(p);
+ Free(p2);
}
TEST(Mman, User) {
namespace __tsan {
-TEST(Shadow, FastState) {
- Shadow s(FastState(11, 22));
- EXPECT_EQ(s.tid(), (u64)11);
- EXPECT_EQ(s.epoch(), (u64)22);
- EXPECT_EQ(s.GetIgnoreBit(), false);
- EXPECT_EQ(s.GetFreedAndReset(), false);
- EXPECT_EQ(s.GetHistorySize(), 0);
- EXPECT_EQ(s.addr0(), (u64)0);
- EXPECT_EQ(s.size(), (u64)1);
- EXPECT_EQ(s.IsWrite(), true);
-
- s.IncrementEpoch();
- EXPECT_EQ(s.epoch(), (u64)23);
- s.IncrementEpoch();
- EXPECT_EQ(s.epoch(), (u64)24);
-
- s.SetIgnoreBit();
- EXPECT_EQ(s.GetIgnoreBit(), true);
- s.ClearIgnoreBit();
- EXPECT_EQ(s.GetIgnoreBit(), false);
-
- for (int i = 0; i < 8; i++) {
- s.SetHistorySize(i);
- EXPECT_EQ(s.GetHistorySize(), i);
- }
- s.SetHistorySize(2);
- s.ClearHistorySize();
- EXPECT_EQ(s.GetHistorySize(), 0);
+struct Region {
+ uptr start;
+ uptr end;
+};
+
+void CheckShadow(const Shadow *s, Sid sid, Epoch epoch, uptr addr, uptr size,
+ AccessType typ) {
+ uptr addr1 = 0;
+ uptr size1 = 0;
+ AccessType typ1 = 0;
+ s->GetAccess(&addr1, &size1, &typ1);
+ CHECK_EQ(s->sid(), sid);
+ CHECK_EQ(s->epoch(), epoch);
+ CHECK_EQ(addr1, addr);
+ CHECK_EQ(size1, size);
+ CHECK_EQ(typ1, typ);
+}
+
+TEST(Shadow, Shadow) {
+ Sid sid = static_cast<Sid>(11);
+ Epoch epoch = static_cast<Epoch>(22);
+ FastState fs;
+ fs.SetSid(sid);
+ fs.SetEpoch(epoch);
+ CHECK_EQ(fs.sid(), sid);
+ CHECK_EQ(fs.epoch(), epoch);
+ CHECK_EQ(fs.GetIgnoreBit(), false);
+ fs.SetIgnoreBit();
+ CHECK_EQ(fs.GetIgnoreBit(), true);
+ fs.ClearIgnoreBit();
+ CHECK_EQ(fs.GetIgnoreBit(), false);
+
+ Shadow s0(fs, 1, 2, kAccessWrite);
+ CheckShadow(&s0, sid, epoch, 1, 2, kAccessWrite);
+ Shadow s1(fs, 2, 3, kAccessRead);
+ CheckShadow(&s1, sid, epoch, 2, 3, kAccessRead);
+ Shadow s2(fs, 0xfffff8 + 4, 1, kAccessWrite | kAccessAtomic);
+ CheckShadow(&s2, sid, epoch, 4, 1, kAccessWrite | kAccessAtomic);
+ Shadow s3(fs, 0xfffff8 + 0, 8, kAccessRead | kAccessAtomic);
+ CheckShadow(&s3, sid, epoch, 0, 8, kAccessRead | kAccessAtomic);
+
+ CHECK(!s0.IsBothReadsOrAtomic(kAccessRead | kAccessAtomic));
+ CHECK(!s1.IsBothReadsOrAtomic(kAccessAtomic));
+ CHECK(!s1.IsBothReadsOrAtomic(kAccessWrite));
+ CHECK(s1.IsBothReadsOrAtomic(kAccessRead));
+ CHECK(s2.IsBothReadsOrAtomic(kAccessAtomic));
+ CHECK(!s2.IsBothReadsOrAtomic(kAccessWrite));
+ CHECK(!s2.IsBothReadsOrAtomic(kAccessRead));
+ CHECK(s3.IsBothReadsOrAtomic(kAccessAtomic));
+ CHECK(!s3.IsBothReadsOrAtomic(kAccessWrite));
+ CHECK(s3.IsBothReadsOrAtomic(kAccessRead));
+
+ CHECK(!s0.IsRWWeakerOrEqual(kAccessRead | kAccessAtomic));
+ CHECK(s1.IsRWWeakerOrEqual(kAccessWrite));
+ CHECK(s1.IsRWWeakerOrEqual(kAccessRead));
+ CHECK(!s1.IsRWWeakerOrEqual(kAccessWrite | kAccessAtomic));
+
+ CHECK(!s2.IsRWWeakerOrEqual(kAccessRead | kAccessAtomic));
+ CHECK(s2.IsRWWeakerOrEqual(kAccessWrite | kAccessAtomic));
+ CHECK(s2.IsRWWeakerOrEqual(kAccessRead));
+ CHECK(s2.IsRWWeakerOrEqual(kAccessWrite));
+
+ CHECK(s3.IsRWWeakerOrEqual(kAccessRead | kAccessAtomic));
+ CHECK(s3.IsRWWeakerOrEqual(kAccessWrite | kAccessAtomic));
+ CHECK(s3.IsRWWeakerOrEqual(kAccessRead));
+ CHECK(s3.IsRWWeakerOrEqual(kAccessWrite));
+
+ Shadow sro(Shadow::kRodata);
+ CheckShadow(&sro, static_cast<Sid>(0), kEpochZero, 0, 0, kAccessRead);
}
TEST(Shadow, Mapping) {
TEST(Shadow, Celling) {
u64 aligned_data[4];
char *data = (char*)aligned_data;
- CHECK_EQ((uptr)data % kShadowSize, 0);
- uptr s0 = MemToShadow((uptr)&data[0]);
- CHECK_EQ(s0 % kShadowSize, 0);
+ CHECK(IsAligned(reinterpret_cast<uptr>(data), kShadowSize));
+ RawShadow *s0 = MemToShadow((uptr)&data[0]);
+ CHECK(IsAligned(reinterpret_cast<uptr>(s0), kShadowSize));
for (unsigned i = 1; i < kShadowCell; i++)
CHECK_EQ(s0, MemToShadow((uptr)&data[i]));
for (unsigned i = kShadowCell; i < 2*kShadowCell; i++)
- CHECK_EQ(s0 + kShadowSize*kShadowCnt, MemToShadow((uptr)&data[i]));
+ CHECK_EQ(s0 + kShadowCnt, MemToShadow((uptr)&data[i]));
for (unsigned i = 2*kShadowCell; i < 3*kShadowCell; i++)
- CHECK_EQ(s0 + 2*kShadowSize*kShadowCnt, MemToShadow((uptr)&data[i]));
+ CHECK_EQ(s0 + 2 * kShadowCnt, MemToShadow((uptr)&data[i]));
+}
+
+// Detect is the Mapping has kBroken field.
+template <uptr>
+struct Has {
+ typedef bool Result;
+};
+
+template <typename Mapping>
+bool broken(...) {
+ return false;
+}
+
+template <typename Mapping>
+bool broken(uptr what, typename Has<Mapping::kBroken>::Result = false) {
+ return Mapping::kBroken & what;
+}
+
+static int CompareRegion(const void *region_a, const void *region_b) {
+ uptr start_a = ((const struct Region *)region_a)->start;
+ uptr start_b = ((const struct Region *)region_b)->start;
+
+ if (start_a < start_b) {
+ return -1;
+ } else if (start_a > start_b) {
+ return 1;
+ } else {
+ return 0;
+ }
+}
+
+template <typename Mapping>
+static void AddMetaRegion(struct Region *shadows, int *num_regions, uptr start,
+ uptr end) {
+ // If the app region is not empty, add its meta to the array.
+ if (start != end) {
+ shadows[*num_regions].start = (uptr)MemToMetaImpl::Apply<Mapping>(start);
+ shadows[*num_regions].end = (uptr)MemToMetaImpl::Apply<Mapping>(end - 1);
+ *num_regions = (*num_regions) + 1;
+ }
}
+struct MappingTest {
+ template <typename Mapping>
+ static void Apply() {
+ // Easy (but ugly) way to print the mapping name.
+ Printf("%s\n", __PRETTY_FUNCTION__);
+ TestRegion<Mapping>(Mapping::kLoAppMemBeg, Mapping::kLoAppMemEnd);
+ TestRegion<Mapping>(Mapping::kMidAppMemBeg, Mapping::kMidAppMemEnd);
+ TestRegion<Mapping>(Mapping::kHiAppMemBeg, Mapping::kHiAppMemEnd);
+ TestRegion<Mapping>(Mapping::kHeapMemBeg, Mapping::kHeapMemEnd);
+
+ TestDisjointMetas<Mapping>();
+
+ // Not tested: the ordering of regions (low app vs. shadow vs. mid app
+ // etc.). That is enforced at runtime by CheckAndProtect.
+ }
+
+ template <typename Mapping>
+ static void TestRegion(uptr beg, uptr end) {
+ if (beg == end)
+ return;
+ Printf("checking region [0x%zx-0x%zx)\n", beg, end);
+ uptr prev = 0;
+ for (uptr p0 = beg; p0 <= end; p0 += (end - beg) / 256) {
+ for (int x = -(int)kShadowCell; x <= (int)kShadowCell; x += kShadowCell) {
+ const uptr p = RoundDown(p0 + x, kShadowCell);
+ if (p < beg || p >= end)
+ continue;
+ const uptr s = MemToShadowImpl::Apply<Mapping>(p);
+ u32 *const m = MemToMetaImpl::Apply<Mapping>(p);
+ const uptr r = ShadowToMemImpl::Apply<Mapping>(s);
+ Printf(" addr=0x%zx: shadow=0x%zx meta=%p reverse=0x%zx\n", p, s, m,
+ r);
+ CHECK(IsAppMemImpl::Apply<Mapping>(p));
+ if (!broken<Mapping>(kBrokenMapping))
+ CHECK(IsShadowMemImpl::Apply<Mapping>(s));
+ CHECK(IsMetaMemImpl::Apply<Mapping>(reinterpret_cast<uptr>(m)));
+ CHECK_EQ(p, RestoreAddrImpl::Apply<Mapping>(CompressAddr(p)));
+ if (!broken<Mapping>(kBrokenReverseMapping))
+ CHECK_EQ(p, r);
+ if (prev && !broken<Mapping>(kBrokenLinearity)) {
+ // Ensure that shadow and meta mappings are linear within a single
+ // user range. Lots of code that processes memory ranges assumes it.
+ const uptr prev_s = MemToShadowImpl::Apply<Mapping>(prev);
+ u32 *const prev_m = MemToMetaImpl::Apply<Mapping>(prev);
+ CHECK_EQ(s - prev_s, (p - prev) * kShadowMultiplier);
+ CHECK_EQ(m - prev_m, (p - prev) / kMetaShadowCell);
+ }
+ prev = p;
+ }
+ }
+ }
+
+ template <typename Mapping>
+ static void TestDisjointMetas() {
+ // Checks that the meta for each app region does not overlap with
+ // the meta for other app regions. For example, the meta for a high
+ // app pointer shouldn't be aliased to the meta of a mid app pointer.
+ // Notice that this is important even though there does not exist a
+ // MetaToMem function.
+ // (If a MetaToMem function did exist, we could simply
+ // check in the TestRegion function that it inverts MemToMeta.)
+ //
+ // We don't try to be clever by allowing the non-PIE (low app)
+ // and PIE (mid and high app) meta regions to overlap.
+ struct Region metas[4];
+ int num_regions = 0;
+ AddMetaRegion<Mapping>(metas, &num_regions, Mapping::kLoAppMemBeg,
+ Mapping::kLoAppMemEnd);
+ AddMetaRegion<Mapping>(metas, &num_regions, Mapping::kMidAppMemBeg,
+ Mapping::kMidAppMemEnd);
+ AddMetaRegion<Mapping>(metas, &num_regions, Mapping::kHiAppMemBeg,
+ Mapping::kHiAppMemEnd);
+ AddMetaRegion<Mapping>(metas, &num_regions, Mapping::kHeapMemBeg,
+ Mapping::kHeapMemEnd);
+
+ // It is not required that the low app shadow is below the mid app
+ // shadow etc., hence we sort the shadows.
+ qsort(metas, num_regions, sizeof(struct Region), CompareRegion);
+
+ for (int i = 0; i < num_regions; i++)
+ Printf("[0x%lu, 0x%lu]\n", metas[i].start, metas[i].end);
+
+ if (!broken<Mapping>(kBrokenAliasedMetas))
+ for (int i = 1; i < num_regions; i++)
+ CHECK(metas[i - 1].end <= metas[i].start);
+ }
+};
+
+TEST(Shadow, AllMappings) { ForEachMapping<MappingTest>(); }
+
} // namespace __tsan
template <typename StackTraceTy>
static void TestStackTrace(StackTraceTy *trace) {
- ThreadState thr(0, 0, 0, 0, 0, 0, 0, 0, 0);
- uptr stack[128];
- thr.shadow_stack = &stack[0];
- thr.shadow_stack_pos = &stack[0];
- thr.shadow_stack_end = &stack[128];
+ ThreadState thr(kMainTid);
ObtainCurrentStack(&thr, 0, trace);
EXPECT_EQ(0U, trace->size);
template<typename StackTraceTy>
static void TestTrim(StackTraceTy *trace) {
- ThreadState thr(0, 0, 0, 0, 0, 0, 0, 0, 0);
- const uptr kShadowStackSize = 2 * kStackTraceMax;
- uptr stack[kShadowStackSize];
- thr.shadow_stack = &stack[0];
- thr.shadow_stack_pos = &stack[0];
- thr.shadow_stack_end = &stack[kShadowStackSize];
+ ThreadState thr(kMainTid);
- for (uptr i = 0; i < kShadowStackSize; ++i)
+ for (uptr i = 0; i < 2 * kStackTraceMax; ++i)
*thr.shadow_stack_pos++ = 100 + i;
ObtainCurrentStack(&thr, 0, trace);
TEST(MetaMap, Basic) {
ThreadState *thr = cur_thread();
+ SlotLocker locker(thr);
MetaMap *m = &ctx->metamap;
u64 block[1] = {}; // fake malloc block
m->AllocBlock(thr, 0, (uptr)&block[0], 1 * sizeof(u64));
MBlock *mb = m->GetBlock((uptr)&block[0]);
- EXPECT_NE(mb, (MBlock*)0);
- EXPECT_EQ(mb->siz, 1 * sizeof(u64));
- EXPECT_EQ(mb->tid, thr->tid);
- uptr sz = m->FreeBlock(thr->proc(), (uptr)&block[0]);
- EXPECT_EQ(sz, 1 * sizeof(u64));
+ CHECK_NE(mb, (MBlock *)0);
+ CHECK_EQ(mb->siz, 1 * sizeof(u64));
+ CHECK_EQ(mb->tid, thr->tid);
+ uptr sz = m->FreeBlock(thr->proc(), (uptr)&block[0], true);
+ CHECK_EQ(sz, 1 * sizeof(u64));
mb = m->GetBlock((uptr)&block[0]);
- EXPECT_EQ(mb, (MBlock*)0);
+ CHECK_EQ(mb, (MBlock *)0);
}
TEST(MetaMap, FreeRange) {
ThreadState *thr = cur_thread();
+ SlotLocker locker(thr);
MetaMap *m = &ctx->metamap;
u64 block[4] = {}; // fake malloc block
m->AllocBlock(thr, 0, (uptr)&block[0], 1 * sizeof(u64));
m->AllocBlock(thr, 0, (uptr)&block[1], 3 * sizeof(u64));
MBlock *mb1 = m->GetBlock((uptr)&block[0]);
- EXPECT_EQ(mb1->siz, 1 * sizeof(u64));
+ CHECK_EQ(mb1->siz, 1 * sizeof(u64));
MBlock *mb2 = m->GetBlock((uptr)&block[1]);
- EXPECT_EQ(mb2->siz, 3 * sizeof(u64));
- m->FreeRange(thr->proc(), (uptr)&block[0], 4 * sizeof(u64));
+ CHECK_EQ(mb2->siz, 3 * sizeof(u64));
+ m->FreeRange(thr->proc(), (uptr)&block[0], 4 * sizeof(u64), true);
mb1 = m->GetBlock((uptr)&block[0]);
- EXPECT_EQ(mb1, (MBlock*)0);
+ CHECK_EQ(mb1, (MBlock *)0);
mb2 = m->GetBlock((uptr)&block[1]);
- EXPECT_EQ(mb2, (MBlock*)0);
+ CHECK_EQ(mb2, (MBlock *)0);
}
-TEST(MetaMap, Sync) NO_THREAD_SAFETY_ANALYSIS {
- // EXPECT can call memset/etc. Disable interceptors to prevent
+TEST(MetaMap, Sync) {
+ // CHECK can call memset/etc. Disable interceptors to prevent
// them from detecting that we exit runtime with mutexes held.
ScopedIgnoreInterceptors ignore;
ThreadState *thr = cur_thread();
+ SlotLocker locker(thr);
MetaMap *m = &ctx->metamap;
u64 block[4] = {}; // fake malloc block
m->AllocBlock(thr, 0, (uptr)&block[0], 4 * sizeof(u64));
- SyncVar *s1 = m->GetIfExistsAndLock((uptr)&block[0], true);
- EXPECT_EQ(s1, (SyncVar*)0);
- s1 = m->GetOrCreateAndLock(thr, 0, (uptr)&block[0], true);
- EXPECT_NE(s1, (SyncVar*)0);
- EXPECT_EQ(s1->addr, (uptr)&block[0]);
- s1->mtx.Unlock();
- SyncVar *s2 = m->GetOrCreateAndLock(thr, 0, (uptr)&block[1], false);
- EXPECT_NE(s2, (SyncVar*)0);
- EXPECT_EQ(s2->addr, (uptr)&block[1]);
- s2->mtx.ReadUnlock();
- m->FreeBlock(thr->proc(), (uptr)&block[0]);
- s1 = m->GetIfExistsAndLock((uptr)&block[0], true);
- EXPECT_EQ(s1, (SyncVar*)0);
- s2 = m->GetIfExistsAndLock((uptr)&block[1], true);
- EXPECT_EQ(s2, (SyncVar*)0);
+ SyncVar *s1 = m->GetSyncIfExists((uptr)&block[0]);
+ CHECK_EQ(s1, (SyncVar *)0);
+ s1 = m->GetSyncOrCreate(thr, 0, (uptr)&block[0], false);
+ CHECK_NE(s1, (SyncVar *)0);
+ CHECK_EQ(s1->addr, (uptr)&block[0]);
+ SyncVar *s2 = m->GetSyncOrCreate(thr, 0, (uptr)&block[1], false);
+ CHECK_NE(s2, (SyncVar *)0);
+ CHECK_EQ(s2->addr, (uptr)&block[1]);
+ m->FreeBlock(thr->proc(), (uptr)&block[0], true);
+ s1 = m->GetSyncIfExists((uptr)&block[0]);
+ CHECK_EQ(s1, (SyncVar *)0);
+ s2 = m->GetSyncIfExists((uptr)&block[1]);
+ CHECK_EQ(s2, (SyncVar *)0);
m->OnProcIdle(thr->proc());
}
-TEST(MetaMap, MoveMemory) NO_THREAD_SAFETY_ANALYSIS {
+TEST(MetaMap, MoveMemory) {
ScopedIgnoreInterceptors ignore;
ThreadState *thr = cur_thread();
+ SlotLocker locker(thr);
MetaMap *m = &ctx->metamap;
u64 block1[4] = {}; // fake malloc block
u64 block2[4] = {}; // fake malloc block
m->AllocBlock(thr, 0, (uptr)&block1[0], 3 * sizeof(u64));
m->AllocBlock(thr, 0, (uptr)&block1[3], 1 * sizeof(u64));
- SyncVar *s1 = m->GetOrCreateAndLock(thr, 0, (uptr)&block1[0], true);
- s1->mtx.Unlock();
- SyncVar *s2 = m->GetOrCreateAndLock(thr, 0, (uptr)&block1[1], true);
- s2->mtx.Unlock();
+ SyncVar *s1 = m->GetSyncOrCreate(thr, 0, (uptr)&block1[0], false);
+ SyncVar *s2 = m->GetSyncOrCreate(thr, 0, (uptr)&block1[1], false);
m->MoveMemory((uptr)&block1[0], (uptr)&block2[0], 4 * sizeof(u64));
MBlock *mb1 = m->GetBlock((uptr)&block1[0]);
- EXPECT_EQ(mb1, (MBlock*)0);
+ CHECK_EQ(mb1, (MBlock *)0);
MBlock *mb2 = m->GetBlock((uptr)&block1[3]);
- EXPECT_EQ(mb2, (MBlock*)0);
+ CHECK_EQ(mb2, (MBlock *)0);
mb1 = m->GetBlock((uptr)&block2[0]);
- EXPECT_NE(mb1, (MBlock*)0);
- EXPECT_EQ(mb1->siz, 3 * sizeof(u64));
+ CHECK_NE(mb1, (MBlock *)0);
+ CHECK_EQ(mb1->siz, 3 * sizeof(u64));
mb2 = m->GetBlock((uptr)&block2[3]);
- EXPECT_NE(mb2, (MBlock*)0);
- EXPECT_EQ(mb2->siz, 1 * sizeof(u64));
- s1 = m->GetIfExistsAndLock((uptr)&block1[0], true);
- EXPECT_EQ(s1, (SyncVar*)0);
- s2 = m->GetIfExistsAndLock((uptr)&block1[1], true);
- EXPECT_EQ(s2, (SyncVar*)0);
- s1 = m->GetIfExistsAndLock((uptr)&block2[0], true);
- EXPECT_NE(s1, (SyncVar*)0);
- EXPECT_EQ(s1->addr, (uptr)&block2[0]);
- s1->mtx.Unlock();
- s2 = m->GetIfExistsAndLock((uptr)&block2[1], true);
- EXPECT_NE(s2, (SyncVar*)0);
- EXPECT_EQ(s2->addr, (uptr)&block2[1]);
- s2->mtx.Unlock();
- m->FreeRange(thr->proc(), (uptr)&block2[0], 4 * sizeof(u64));
+ CHECK_NE(mb2, (MBlock *)0);
+ CHECK_EQ(mb2->siz, 1 * sizeof(u64));
+ s1 = m->GetSyncIfExists((uptr)&block1[0]);
+ CHECK_EQ(s1, (SyncVar *)0);
+ s2 = m->GetSyncIfExists((uptr)&block1[1]);
+ CHECK_EQ(s2, (SyncVar *)0);
+ s1 = m->GetSyncIfExists((uptr)&block2[0]);
+ CHECK_NE(s1, (SyncVar *)0);
+ CHECK_EQ(s1->addr, (uptr)&block2[0]);
+ s2 = m->GetSyncIfExists((uptr)&block2[1]);
+ CHECK_NE(s2, (SyncVar *)0);
+ CHECK_EQ(s2->addr, (uptr)&block2[1]);
+ m->FreeRange(thr->proc(), (uptr)&block2[0], 4 * sizeof(u64), true);
}
-TEST(MetaMap, ResetSync) NO_THREAD_SAFETY_ANALYSIS {
+TEST(MetaMap, ResetSync) {
ScopedIgnoreInterceptors ignore;
ThreadState *thr = cur_thread();
+ SlotLocker locker(thr);
MetaMap *m = &ctx->metamap;
u64 block[1] = {}; // fake malloc block
m->AllocBlock(thr, 0, (uptr)&block[0], 1 * sizeof(u64));
- SyncVar *s = m->GetOrCreateAndLock(thr, 0, (uptr)&block[0], true);
- s->Reset(thr->proc());
- s->mtx.Unlock();
- uptr sz = m->FreeBlock(thr->proc(), (uptr)&block[0]);
- EXPECT_EQ(sz, 1 * sizeof(u64));
+ SyncVar *s = m->GetSyncOrCreate(thr, 0, (uptr)&block[0], false);
+ s->Reset();
+ uptr sz = m->FreeBlock(thr->proc(), (uptr)&block[0], true);
+ CHECK_EQ(sz, 1 * sizeof(u64));
}
} // namespace __tsan
--- /dev/null
+//===-- tsan_trace_test.cpp -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#include "tsan_trace.h"
+
+#include <pthread.h>
+
+#include "gtest/gtest.h"
+#include "tsan_rtl.h"
+
+#if !defined(__x86_64__)
+// These tests are currently crashing on ppc64:
+// https://reviews.llvm.org/D110546#3025422
+// due to the way we create thread contexts
+// There must be some difference in thread initialization
+// between normal execution and unit tests.
+# define TRACE_TEST(SUITE, NAME) TEST(SUITE, DISABLED_##NAME)
+#else
+# define TRACE_TEST(SUITE, NAME) TEST(SUITE, NAME)
+#endif
+
+namespace __tsan {
+
+// We need to run all trace tests in a new thread,
+// so that the thread trace is empty initially.
+template <uptr N>
+struct ThreadArray {
+ ThreadArray() {
+ for (auto *&thr : threads) {
+ thr = static_cast<ThreadState *>(
+ MmapOrDie(sizeof(ThreadState), "ThreadState"));
+ Tid tid = ThreadCreate(cur_thread(), 0, 0, true);
+ Processor *proc = ProcCreate();
+ ProcWire(proc, thr);
+ ThreadStart(thr, tid, 0, ThreadType::Fiber);
+ }
+ }
+
+ ~ThreadArray() {
+ for (uptr i = 0; i < N; i++) {
+ if (threads[i])
+ Finish(i);
+ }
+ }
+
+ void Finish(uptr i) {
+ auto *thr = threads[i];
+ threads[i] = nullptr;
+ Processor *proc = thr->proc();
+ ThreadFinish(thr);
+ ProcUnwire(proc, thr);
+ ProcDestroy(proc);
+ UnmapOrDie(thr, sizeof(ThreadState));
+ }
+
+ ThreadState *threads[N];
+ ThreadState *operator[](uptr i) { return threads[i]; }
+ ThreadState *operator->() { return threads[0]; }
+ operator ThreadState *() { return threads[0]; }
+};
+
+TRACE_TEST(Trace, RestoreAccess) {
+ // A basic test with some function entry/exit events,
+ // some mutex lock/unlock events and some other distracting
+ // memory events.
+ ThreadArray<1> thr;
+ TraceFunc(thr, 0x1000);
+ TraceFunc(thr, 0x1001);
+ TraceMutexLock(thr, EventType::kLock, 0x4000, 0x5000, 0x6000);
+ TraceMutexLock(thr, EventType::kLock, 0x4001, 0x5001, 0x6001);
+ TraceMutexUnlock(thr, 0x5000);
+ TraceFunc(thr);
+ CHECK(TryTraceMemoryAccess(thr, 0x2001, 0x3001, 8, kAccessRead));
+ TraceMutexLock(thr, EventType::kRLock, 0x4002, 0x5002, 0x6002);
+ TraceFunc(thr, 0x1002);
+ CHECK(TryTraceMemoryAccess(thr, 0x2000, 0x3000, 8, kAccessRead));
+ // This is the access we want to find.
+ // The previous one is equivalent, but RestoreStack must prefer
+ // the last of the matchig accesses.
+ CHECK(TryTraceMemoryAccess(thr, 0x2002, 0x3000, 8, kAccessRead));
+ Lock slot_lock(&ctx->slots[static_cast<uptr>(thr->fast_state.sid())].mtx);
+ ThreadRegistryLock lock1(&ctx->thread_registry);
+ Lock lock2(&ctx->slot_mtx);
+ Tid tid = kInvalidTid;
+ VarSizeStackTrace stk;
+ MutexSet mset;
+ uptr tag = kExternalTagNone;
+ bool res = RestoreStack(EventType::kAccessExt, thr->fast_state.sid(),
+ thr->fast_state.epoch(), 0x3000, 8, kAccessRead, &tid,
+ &stk, &mset, &tag);
+ CHECK(res);
+ CHECK_EQ(tid, thr->tid);
+ CHECK_EQ(stk.size, 3);
+ CHECK_EQ(stk.trace[0], 0x1000);
+ CHECK_EQ(stk.trace[1], 0x1002);
+ CHECK_EQ(stk.trace[2], 0x2002);
+ CHECK_EQ(mset.Size(), 2);
+ CHECK_EQ(mset.Get(0).addr, 0x5001);
+ CHECK_EQ(mset.Get(0).stack_id, 0x6001);
+ CHECK_EQ(mset.Get(0).write, true);
+ CHECK_EQ(mset.Get(1).addr, 0x5002);
+ CHECK_EQ(mset.Get(1).stack_id, 0x6002);
+ CHECK_EQ(mset.Get(1).write, false);
+ CHECK_EQ(tag, kExternalTagNone);
+}
+
+TRACE_TEST(Trace, MemoryAccessSize) {
+ // Test tracing and matching of accesses of different sizes.
+ struct Params {
+ uptr access_size, offset, size;
+ bool res;
+ };
+ Params tests[] = {
+ {1, 0, 1, true}, {4, 0, 2, true},
+ {4, 2, 2, true}, {8, 3, 1, true},
+ {2, 1, 1, true}, {1, 1, 1, false},
+ {8, 5, 4, false}, {4, static_cast<uptr>(-1l), 4, false},
+ };
+ for (auto params : tests) {
+ for (int type = 0; type < 3; type++) {
+ ThreadArray<1> thr;
+ Printf("access_size=%zu, offset=%zu, size=%zu, res=%d, type=%d\n",
+ params.access_size, params.offset, params.size, params.res, type);
+ TraceFunc(thr, 0x1000);
+ switch (type) {
+ case 0:
+ // This should emit compressed event.
+ CHECK(TryTraceMemoryAccess(thr, 0x2000, 0x3000, params.access_size,
+ kAccessRead));
+ break;
+ case 1:
+ // This should emit full event.
+ CHECK(TryTraceMemoryAccess(thr, 0x2000000, 0x3000, params.access_size,
+ kAccessRead));
+ break;
+ case 2:
+ TraceMemoryAccessRange(thr, 0x2000000, 0x3000, params.access_size,
+ kAccessRead);
+ break;
+ }
+ Lock slot_lock(&ctx->slots[static_cast<uptr>(thr->fast_state.sid())].mtx);
+ ThreadRegistryLock lock1(&ctx->thread_registry);
+ Lock lock2(&ctx->slot_mtx);
+ Tid tid = kInvalidTid;
+ VarSizeStackTrace stk;
+ MutexSet mset;
+ uptr tag = kExternalTagNone;
+ bool res =
+ RestoreStack(EventType::kAccessExt, thr->fast_state.sid(),
+ thr->fast_state.epoch(), 0x3000 + params.offset,
+ params.size, kAccessRead, &tid, &stk, &mset, &tag);
+ CHECK_EQ(res, params.res);
+ if (params.res) {
+ CHECK_EQ(stk.size, 2);
+ CHECK_EQ(stk.trace[0], 0x1000);
+ CHECK_EQ(stk.trace[1], type ? 0x2000000 : 0x2000);
+ }
+ }
+ }
+}
+
+TRACE_TEST(Trace, RestoreMutexLock) {
+ // Check of restoration of a mutex lock event.
+ ThreadArray<1> thr;
+ TraceFunc(thr, 0x1000);
+ TraceMutexLock(thr, EventType::kLock, 0x4000, 0x5000, 0x6000);
+ TraceMutexLock(thr, EventType::kRLock, 0x4001, 0x5001, 0x6001);
+ TraceMutexLock(thr, EventType::kRLock, 0x4002, 0x5001, 0x6002);
+ Lock slot_lock(&ctx->slots[static_cast<uptr>(thr->fast_state.sid())].mtx);
+ ThreadRegistryLock lock1(&ctx->thread_registry);
+ Lock lock2(&ctx->slot_mtx);
+ Tid tid = kInvalidTid;
+ VarSizeStackTrace stk;
+ MutexSet mset;
+ uptr tag = kExternalTagNone;
+ bool res = RestoreStack(EventType::kLock, thr->fast_state.sid(),
+ thr->fast_state.epoch(), 0x5001, 0, 0, &tid, &stk,
+ &mset, &tag);
+ CHECK(res);
+ CHECK_EQ(stk.size, 2);
+ CHECK_EQ(stk.trace[0], 0x1000);
+ CHECK_EQ(stk.trace[1], 0x4002);
+ CHECK_EQ(mset.Size(), 2);
+ CHECK_EQ(mset.Get(0).addr, 0x5000);
+ CHECK_EQ(mset.Get(0).stack_id, 0x6000);
+ CHECK_EQ(mset.Get(0).write, true);
+ CHECK_EQ(mset.Get(1).addr, 0x5001);
+ CHECK_EQ(mset.Get(1).stack_id, 0x6001);
+ CHECK_EQ(mset.Get(1).write, false);
+}
+
+TRACE_TEST(Trace, MultiPart) {
+ // Check replay of a trace with multiple parts.
+ ThreadArray<1> thr;
+ FuncEntry(thr, 0x1000);
+ FuncEntry(thr, 0x2000);
+ MutexPreLock(thr, 0x4000, 0x5000, 0);
+ MutexPostLock(thr, 0x4000, 0x5000, 0);
+ MutexPreLock(thr, 0x4000, 0x5000, 0);
+ MutexPostLock(thr, 0x4000, 0x5000, 0);
+ const uptr kEvents = 3 * sizeof(TracePart) / sizeof(Event);
+ for (uptr i = 0; i < kEvents; i++) {
+ FuncEntry(thr, 0x3000);
+ MutexPreLock(thr, 0x4002, 0x5002, 0);
+ MutexPostLock(thr, 0x4002, 0x5002, 0);
+ MutexUnlock(thr, 0x4003, 0x5002, 0);
+ FuncExit(thr);
+ }
+ FuncEntry(thr, 0x4000);
+ TraceMutexLock(thr, EventType::kRLock, 0x4001, 0x5001, 0x6001);
+ CHECK(TryTraceMemoryAccess(thr, 0x2002, 0x3000, 8, kAccessRead));
+ Lock slot_lock(&ctx->slots[static_cast<uptr>(thr->fast_state.sid())].mtx);
+ ThreadRegistryLock lock1(&ctx->thread_registry);
+ Lock lock2(&ctx->slot_mtx);
+ Tid tid = kInvalidTid;
+ VarSizeStackTrace stk;
+ MutexSet mset;
+ uptr tag = kExternalTagNone;
+ bool res = RestoreStack(EventType::kAccessExt, thr->fast_state.sid(),
+ thr->fast_state.epoch(), 0x3000, 8, kAccessRead, &tid,
+ &stk, &mset, &tag);
+ CHECK(res);
+ CHECK_EQ(tid, thr->tid);
+ CHECK_EQ(stk.size, 4);
+ CHECK_EQ(stk.trace[0], 0x1000);
+ CHECK_EQ(stk.trace[1], 0x2000);
+ CHECK_EQ(stk.trace[2], 0x4000);
+ CHECK_EQ(stk.trace[3], 0x2002);
+ CHECK_EQ(mset.Size(), 2);
+ CHECK_EQ(mset.Get(0).addr, 0x5000);
+ CHECK_EQ(mset.Get(0).write, true);
+ CHECK_EQ(mset.Get(0).count, 2);
+ CHECK_EQ(mset.Get(1).addr, 0x5001);
+ CHECK_EQ(mset.Get(1).write, false);
+ CHECK_EQ(mset.Get(1).count, 1);
+}
+
+TRACE_TEST(Trace, DeepSwitch) {
+ ThreadArray<1> thr;
+ for (int i = 0; i < 2000; i++) {
+ FuncEntry(thr, 0x1000);
+ const uptr kEvents = sizeof(TracePart) / sizeof(Event);
+ for (uptr i = 0; i < kEvents; i++) {
+ TraceMutexLock(thr, EventType::kLock, 0x4000, 0x5000, 0x6000);
+ TraceMutexUnlock(thr, 0x5000);
+ }
+ }
+}
+
+void CheckTraceState(uptr count, uptr finished, uptr excess, uptr recycle) {
+ Lock l(&ctx->slot_mtx);
+ Printf("CheckTraceState(%zu/%zu, %zu/%zu, %zu/%zu, %zu/%zu)\n",
+ ctx->trace_part_total_allocated, count,
+ ctx->trace_part_recycle_finished, finished,
+ ctx->trace_part_finished_excess, excess,
+ ctx->trace_part_recycle.Size(), recycle);
+ CHECK_EQ(ctx->trace_part_total_allocated, count);
+ CHECK_EQ(ctx->trace_part_recycle_finished, finished);
+ CHECK_EQ(ctx->trace_part_finished_excess, excess);
+ CHECK_EQ(ctx->trace_part_recycle.Size(), recycle);
+}
+
+TRACE_TEST(TraceAlloc, SingleThread) {
+ TraceResetForTesting();
+ auto check_thread = [&](ThreadState *thr, uptr size, uptr count,
+ uptr finished, uptr excess, uptr recycle) {
+ CHECK_EQ(thr->tctx->trace.parts.Size(), size);
+ CheckTraceState(count, finished, excess, recycle);
+ };
+ ThreadArray<2> threads;
+ check_thread(threads[0], 0, 0, 0, 0, 0);
+ TraceSwitchPartImpl(threads[0]);
+ check_thread(threads[0], 1, 1, 0, 0, 0);
+ TraceSwitchPartImpl(threads[0]);
+ check_thread(threads[0], 2, 2, 0, 0, 0);
+ TraceSwitchPartImpl(threads[0]);
+ check_thread(threads[0], 3, 3, 0, 0, 1);
+ TraceSwitchPartImpl(threads[0]);
+ check_thread(threads[0], 3, 3, 0, 0, 1);
+ threads.Finish(0);
+ CheckTraceState(3, 3, 0, 3);
+ threads.Finish(1);
+ CheckTraceState(3, 3, 0, 3);
+}
+
+TRACE_TEST(TraceAlloc, FinishedThreadReuse) {
+ TraceResetForTesting();
+ constexpr uptr Hi = Trace::kFinishedThreadHi;
+ constexpr uptr kThreads = 4 * Hi;
+ ThreadArray<kThreads> threads;
+ for (uptr i = 0; i < kThreads; i++) {
+ Printf("thread %zu\n", i);
+ TraceSwitchPartImpl(threads[i]);
+ if (i <= Hi)
+ CheckTraceState(i + 1, i, 0, i);
+ else if (i <= 2 * Hi)
+ CheckTraceState(Hi + 1, Hi, i - Hi, Hi);
+ else
+ CheckTraceState(Hi + 1, Hi, Hi, Hi);
+ threads.Finish(i);
+ if (i < Hi)
+ CheckTraceState(i + 1, i + 1, 0, i + 1);
+ else if (i < 2 * Hi)
+ CheckTraceState(Hi + 1, Hi + 1, i - Hi + 1, Hi + 1);
+ else
+ CheckTraceState(Hi + 1, Hi + 1, Hi + 1, Hi + 1);
+ }
+}
+
+TRACE_TEST(TraceAlloc, FinishedThreadReuse2) {
+ TraceResetForTesting();
+ // constexpr uptr Lo = Trace::kFinishedThreadLo;
+ // constexpr uptr Hi = Trace::kFinishedThreadHi;
+ constexpr uptr Min = Trace::kMinParts;
+ constexpr uptr kThreads = 10;
+ constexpr uptr kParts = 2 * Min;
+ ThreadArray<kThreads> threads;
+ for (uptr i = 0; i < kThreads; i++) {
+ Printf("thread %zu\n", i);
+ for (uptr j = 0; j < kParts; j++) TraceSwitchPartImpl(threads[i]);
+ if (i == 0)
+ CheckTraceState(Min, 0, 0, 1);
+ else
+ CheckTraceState(2 * Min, 0, Min, Min + 1);
+ threads.Finish(i);
+ if (i == 0)
+ CheckTraceState(Min, Min, 0, Min);
+ else
+ CheckTraceState(2 * Min, 2 * Min, Min, 2 * Min);
+ }
+}
+
+} // namespace __tsan
//===----------------------------------------------------------------------===//
#include "gtest/gtest.h"
-namespace __sanitizer {
-bool ReexecDisabled() {
- return true;
-}
-}
-
int main(int argc, char **argv) {
testing::GTEST_FLAG(death_test_style) = "threadsafe";
testing::InitGoogleTest(&argc, argv);
--- /dev/null
+//===-- tsan_clock_test.cpp -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#include "tsan_vector_clock.h"
+
+#include "gtest/gtest.h"
+#include "tsan_rtl.h"
+
+namespace __tsan {
+
+TEST(VectorClock, GetSet) {
+ // Compiler won't ensure alignment on stack.
+ VectorClock *vc = New<VectorClock>();
+ for (uptr i = 0; i < kThreadSlotCount; i++)
+ ASSERT_EQ(vc->Get(static_cast<Sid>(i)), kEpochZero);
+ for (uptr i = 0; i < kThreadSlotCount; i++)
+ vc->Set(static_cast<Sid>(i), static_cast<Epoch>(i));
+ for (uptr i = 0; i < kThreadSlotCount; i++)
+ ASSERT_EQ(vc->Get(static_cast<Sid>(i)), static_cast<Epoch>(i));
+ vc->Reset();
+ for (uptr i = 0; i < kThreadSlotCount; i++)
+ ASSERT_EQ(vc->Get(static_cast<Sid>(i)), kEpochZero);
+ DestroyAndFree(vc);
+}
+
+TEST(VectorClock, VectorOps) {
+ VectorClock *vc1 = New<VectorClock>();
+ VectorClock *vc2 = nullptr;
+ VectorClock *vc3 = nullptr;
+
+ vc1->Acquire(vc2);
+ for (uptr i = 0; i < kThreadSlotCount; i++)
+ ASSERT_EQ(vc1->Get(static_cast<Sid>(i)), kEpochZero);
+ vc1->Release(&vc2);
+ EXPECT_NE(vc2, nullptr);
+ vc1->Acquire(vc2);
+ for (uptr i = 0; i < kThreadSlotCount; i++)
+ ASSERT_EQ(vc1->Get(static_cast<Sid>(i)), kEpochZero);
+
+ for (uptr i = 0; i < kThreadSlotCount; i++) {
+ vc1->Set(static_cast<Sid>(i), static_cast<Epoch>(i));
+ vc2->Set(static_cast<Sid>(i), static_cast<Epoch>(kThreadSlotCount - i));
+ }
+ vc1->Acquire(vc2);
+ for (uptr i = 0; i < kThreadSlotCount; i++) {
+ ASSERT_EQ(vc1->Get(static_cast<Sid>(i)),
+ static_cast<Epoch>(i < kThreadSlotCount / 2 ? kThreadSlotCount - i
+ : i));
+ ASSERT_EQ(vc2->Get(static_cast<Sid>(i)),
+ static_cast<Epoch>(kThreadSlotCount - i));
+ }
+ vc2->ReleaseStore(&vc3);
+ for (uptr i = 0; i < kThreadSlotCount; i++) {
+ ASSERT_EQ(vc3->Get(static_cast<Sid>(i)),
+ static_cast<Epoch>(kThreadSlotCount - i));
+ ASSERT_EQ(vc2->Get(static_cast<Sid>(i)),
+ static_cast<Epoch>(kThreadSlotCount - i));
+ }
+
+ vc1->Reset();
+ vc2->Reset();
+ for (uptr i = 0; i < kThreadSlotCount; i++) {
+ vc1->Set(static_cast<Sid>(i), static_cast<Epoch>(i));
+ vc2->Set(static_cast<Sid>(i), static_cast<Epoch>(kThreadSlotCount - i));
+ }
+ vc1->ReleaseAcquire(&vc2);
+ for (uptr i = 0; i < kThreadSlotCount; i++) {
+ Epoch expect =
+ static_cast<Epoch>(i < kThreadSlotCount / 2 ? kThreadSlotCount - i : i);
+ ASSERT_EQ(vc1->Get(static_cast<Sid>(i)), expect);
+ ASSERT_EQ(vc2->Get(static_cast<Sid>(i)), expect);
+ }
+
+ vc1->Reset();
+ vc2->Reset();
+ for (uptr i = 0; i < kThreadSlotCount; i++) {
+ vc1->Set(static_cast<Sid>(i), static_cast<Epoch>(i));
+ vc2->Set(static_cast<Sid>(i), static_cast<Epoch>(kThreadSlotCount - i));
+ }
+ vc1->ReleaseStoreAcquire(&vc2);
+ for (uptr i = 0; i < kThreadSlotCount; i++) {
+ ASSERT_EQ(vc1->Get(static_cast<Sid>(i)),
+ static_cast<Epoch>(i < kThreadSlotCount / 2 ? kThreadSlotCount - i
+ : i));
+ ASSERT_EQ(vc2->Get(static_cast<Sid>(i)), static_cast<Epoch>(i));
+ }
+
+ DestroyAndFree(vc1);
+ DestroyAndFree(vc2);
+ DestroyAndFree(vc3);
+}
+
+} // namespace __tsan
append_rtti_flag(OFF UBSAN_CFLAGS)
append_list_if(SANITIZER_CAN_USE_CXXABI -DUBSAN_CAN_USE_CXXABI UBSAN_CFLAGS)
+# Too many existing bugs, needs cleanup.
+append_list_if(COMPILER_RT_HAS_WNO_FORMAT -Wno-format UBSAN_CFLAGS)
+
set(UBSAN_STANDALONE_CFLAGS ${SANITIZER_COMMON_CFLAGS})
append_rtti_flag(OFF UBSAN_STANDALONE_CFLAGS)
append_list_if(SANITIZER_CAN_USE_CXXABI -DUBSAN_CAN_USE_CXXABI UBSAN_STANDALONE_CFLAGS)
append_rtti_flag(ON UBSAN_CXXFLAGS)
append_list_if(SANITIZER_CAN_USE_CXXABI -DUBSAN_CAN_USE_CXXABI UBSAN_CXXFLAGS)
+# Silence warnings in system headers with MSVC.
+if(NOT CLANG_CL)
+ append_list_if(COMPILER_RT_HAS_EXTERNAL_FLAG "/experimental:external /external:W0 /external:anglebrackets" UBSAN_CXXFLAGS)
+endif()
+
set(UBSAN_LINK_FLAGS ${SANITIZER_COMMON_LINK_FLAGS})
-set(UBSAN_DYNAMIC_LIBS ${SANITIZER_CXX_ABI_LIBRARIES} ${SANITIZER_COMMON_LINK_LIBS})
+set(UBSAN_DYNAMIC_LIBS
+ ${COMPILER_RT_UNWINDER_LINK_LIBS}
+ ${SANITIZER_CXX_ABI_LIBRARIES}
+ ${SANITIZER_COMMON_LINK_LIBS})
append_list_if(COMPILER_RT_HAS_LIBDL dl UBSAN_DYNAMIC_LIBS)
append_list_if(COMPILER_RT_HAS_LIBLOG log UBSAN_DYNAMIC_LIBS)
if(COMPILER_RT_HAS_UBSAN)
# Initializer of standalone UBSan runtime.
add_compiler_rt_object_libraries(RTUbsan_standalone
- OS ${SANITIZER_COMMON_SUPPORTED_OS}
+ OS ${UBSAN_SUPPORTED_OS}
ARCHS ${UBSAN_SUPPORTED_ARCH}
SOURCES ${UBSAN_STANDALONE_SOURCES}
ADDITIONAL_HEADERS ${UBSAN_HEADERS}
add_compiler_rt_runtime(clang_rt.ubsan
SHARED
- OS ${SANITIZER_COMMON_SUPPORTED_OS}
+ OS ${UBSAN_SUPPORTED_OS}
ARCHS ${UBSAN_SUPPORTED_ARCH}
OBJECT_LIBS RTUbsan
RTUbsan_standalone
LINK_FLAGS ${WEAK_SYMBOL_LINK_FLAGS}
PARENT_TARGET ubsan)
- add_compiler_rt_runtime(clang_rt.ubsan
- STATIC
- OS ${SANITIZER_COMMON_SUPPORTED_OS}
- ARCHS ${UBSAN_SUPPORTED_ARCH}
- OBJECT_LIBS RTUbsan
- RTUbsan_standalone
- RTSanitizerCommonNoHooks
- RTSanitizerCommonLibcNoHooks
- RTSanitizerCommonCoverage
- RTSanitizerCommonSymbolizerNoHooks
- RTInterception
- LINK_FLAGS ${WEAK_SYMBOL_LINK_FLAGS}
- PARENT_TARGET ubsan)
+ if (NOT APPLE)
+ add_compiler_rt_runtime(clang_rt.ubsan
+ STATIC
+ OS ${UBSAN_SUPPORTED_OS}
+ ARCHS ${UBSAN_SUPPORTED_ARCH}
+ OBJECT_LIBS RTUbsan
+ RTUbsan_standalone
+ RTSanitizerCommonNoHooks
+ RTSanitizerCommonLibcNoHooks
+ RTSanitizerCommonCoverage
+ RTSanitizerCommonSymbolizerNoHooks
+ RTInterception
+ LINK_FLAGS ${WEAK_SYMBOL_LINK_FLAGS}
+ PARENT_TARGET ubsan)
+ endif()
endif()
else()
add_compiler_rt_object_libraries(UbsanWeakInterception
${SANITIZER_COMMON_SUPPORTED_OS}
ARCHS ${UBSAN_SUPPORTED_ARCH}
- SOURCES ubsan_win_weak_interception.cpp
+ SOURCES
+ ubsan_win_weak_interception.cpp
CFLAGS ${UBSAN_CFLAGS} -DSANITIZER_DYNAMIC
DEFS ${UBSAN_COMMON_DEFINITIONS})
add_compiler_rt_object_libraries(UbsanDllThunk
${SANITIZER_COMMON_SUPPORTED_OS}
ARCHS ${UBSAN_SUPPORTED_ARCH}
- SOURCES ubsan_win_dll_thunk.cpp
+ SOURCES
+ ubsan_win_dll_thunk.cpp
CFLAGS ${UBSAN_CFLAGS} -DSANITIZER_DLL_THUNK
DEFS ${UBSAN_COMMON_DEFINITIONS})
add_compiler_rt_object_libraries(UbsanDynamicRuntimeThunk
${SANITIZER_COMMON_SUPPORTED_OS}
ARCHS ${UBSAN_SUPPORTED_ARCH}
- SOURCES ubsan_win_dynamic_runtime_thunk.cpp
+ SOURCES
+ ubsan_win_dynamic_runtime_thunk.cpp
CFLAGS ${UBSAN_CFLAGS} ${DYNAMIC_RUNTIME_THUNK_CFLAGS}
DEFS ${UBSAN_COMMON_DEFINITIONS})
endif()
add_compiler_rt_runtime(clang_rt.ubsan_standalone
STATIC
ARCHS ${UBSAN_SUPPORTED_ARCH}
- SOURCES ubsan_init_standalone_preinit.cpp
+ SOURCES
+ ubsan_init_standalone_preinit.cpp
ADDITIONAL_HEADERS ${UBSAN_HEADERS}
OBJECT_LIBS RTSanitizerCommon
RTSanitizerCommonLibc
// Windows.
// TODO(yln): This is a temporary workaround. GetStackTrace functions will be
// removed in the future.
-void ubsan_GetStackTrace(BufferedStackTrace *stack, uptr max_depth,
- uptr pc, uptr bp, void *context, bool fast) {
+void ubsan_GetStackTrace(BufferedStackTrace *stack, uptr max_depth, uptr pc,
+ uptr bp, void *context, bool request_fast) {
uptr top = 0;
uptr bottom = 0;
- if (StackTrace::WillUseFastUnwind(fast)) {
- GetThreadStackTopAndBottom(false, &top, &bottom);
- stack->Unwind(max_depth, pc, bp, nullptr, top, bottom, true);
- } else
- stack->Unwind(max_depth, pc, bp, context, 0, 0, false);
+ GetThreadStackTopAndBottom(false, &top, &bottom);
+ bool fast = StackTrace::WillUseFastUnwind(request_fast);
+ stack->Unwind(max_depth, pc, bp, context, top, bottom, fast);
}
static void MaybePrintStackTrace(uptr pc, uptr bp) {
return;
}
case Location::LK_Memory:
- Buffer->append("%p", Loc.getMemoryLocation());
+ Buffer->append("%p", reinterpret_cast<void *>(Loc.getMemoryLocation()));
return;
case Location::LK_Symbolized: {
const AddressInfo &Info = Loc.getSymbolizedStack()->info;
RenderModuleLocation(Buffer, Info.module, Info.module_offset,
Info.module_arch, common_flags()->strip_path_prefix);
else
- Buffer->append("%p", Info.address);
+ Buffer->append("%p", reinterpret_cast<void *>(Info.address));
return;
}
case Location::LK_Null:
Buffer.append("\n");
// Emit highlights.
- Buffer.append(Decor.Highlight());
+ Buffer.append("%s", Decor.Highlight());
Range *InRange = upperBound(Min, Ranges, NumRanges);
for (uptr P = Min; P != Max; ++P) {
char Pad = ' ', Byte = ' ';
Buffer.clear();
}
- Buffer.append(Decor.Bold());
+ Buffer.append("%s", Decor.Bold());
RenderLocation(&Buffer, Loc);
Buffer.append(":");
TCK_DynamicOperation
};
-const char *TypeCheckKinds[] = {
+extern const char *const TypeCheckKinds[] = {
"load of", "store to", "reference binding to", "member access within",
"member call on", "constructor call on", "downcast of", "downcast of",
"upcast of", "cast to virtual base of", "_Nonnull binding to",
using namespace __ubsan;
namespace __ubsan {
- extern const char *TypeCheckKinds[];
+ extern const char *const TypeCheckKinds[];
}
// Returns true if UBSan has printed an error report.
ValueHandle fnRTTI);
}
-#endif // UBSAN_HANDLERS_H
+#endif // UBSAN_HANDLERS_CXX_H
#include "ubsan_platform.h"
#if CAN_SANITIZE_UB
-#include "ubsan_diag.h"
-#include "ubsan_init.h"
-#include "ubsan_flags.h"
#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_interface_internal.h"
#include "sanitizer_common/sanitizer_libc.h"
#include "sanitizer_common/sanitizer_mutex.h"
#include "sanitizer_common/sanitizer_symbolizer.h"
+#include "ubsan_diag.h"
+#include "ubsan_flags.h"
+#include "ubsan_init.h"
using namespace __ubsan;
#include "sanitizer_common/sanitizer_common.h"
#include "sanitizer_common/sanitizer_ptrauth.h"
+#include <stdint.h>
// The following are intended to be binary compatible with the definitions
// given in the Itanium ABI. We make no attempt to be ODR-compatible with
namespace std {
class type_info {
public:
+ typedef const char *__type_name_t;
virtual ~type_info();
const char *__type_name;
+
+ __type_name_t name() const {
+#if defined(__APPLE__) && defined(__LP64__) && !defined(__x86_64__)
+ uintptr_t __non_unique_rtti_bit =
+ (1ULL << ((__CHAR_BIT__ * sizeof(__type_name_t)) - 1));
+ return (__type_name_t)((uintptr_t)__type_name & ~__non_unique_rtti_bit);
+#else
+ return __type_name;
+#endif
+ }
};
}
static bool isDerivedFromAtOffset(const abi::__class_type_info *Derived,
const abi::__class_type_info *Base,
sptr Offset) {
- if (Derived->__type_name == Base->__type_name ||
+ if (Derived->name() == Base->name() ||
__ubsan::checkTypeInfoEquality(Derived, Base))
return Offset == 0;
const abi::__class_type_info *ObjectType = findBaseAtOffset(
static_cast<const abi::__class_type_info*>(Vtable->TypeInfo),
-Vtable->Offset);
- return DynamicTypeInfo(Vtable->TypeInfo->__type_name, -Vtable->Offset,
- ObjectType ? ObjectType->__type_name : "<unknown>");
+ return DynamicTypeInfo(Vtable->TypeInfo->name(), -Vtable->Offset,
+ ObjectType ? ObjectType->name() : "<unknown>");
}
bool __ubsan::checkTypeInfoEquality(const void *TypeInfo1,
const void *TypeInfo2) {
auto TI1 = static_cast<const std::type_info *>(TypeInfo1);
auto TI2 = static_cast<const std::type_info *>(TypeInfo2);
- return SANITIZER_NON_UNIQUE_TYPEINFO && TI1->__type_name[0] != '*' &&
- TI2->__type_name[0] != '*' &&
- !internal_strcmp(TI1->__type_name, TI2->__type_name);
+ return SANITIZER_NON_UNIQUE_TYPEINFO && TI1->name()[0] != '*' &&
+ TI2->name()[0] != '*' && !internal_strcmp(TI1->name(), TI2->name());
}
#endif // CAN_SANITIZE_UB && !SANITIZER_WINDOWS
#include "sanitizer_common/sanitizer_libc.h"
#include "sanitizer_common/sanitizer_mutex.h"
-// TODO(dliew): Prefer '__APPLE__' here over 'SANITIZER_MAC', as the latter is
-// unclear. rdar://58124919 tracks using a more obviously portable guard.
-#if defined(__APPLE__)
+#if SANITIZER_APPLE
#include <dlfcn.h>
#endif
typedef const char *(*ObjCGetClassNameTy)(void *);
const char *__ubsan::getObjCClassName(ValueHandle Pointer) {
-#if defined(__APPLE__)
+#if SANITIZER_APPLE
// We need to query the ObjC runtime for some information, but do not want
// to introduce a static dependency from the ubsan runtime onto ObjC. Try to
// grab a handle to the ObjC runtime used by the process.
# Standalone minimal UBSan runtimes.
add_compiler_rt_runtime(clang_rt.ubsan_minimal
STATIC
- OS ${SANITIZER_COMMON_SUPPORTED_OS}
+ OS ${UBSAN_SUPPORTED_OS}
ARCHS ${UBSAN_SUPPORTED_ARCH}
OBJECT_LIBS RTUbsan_minimal
CFLAGS ${UBSAN_CFLAGS}
add_compiler_rt_runtime(clang_rt.ubsan_minimal
SHARED
- OS ${SANITIZER_COMMON_SUPPORTED_OS}
+ OS ${UBSAN_SUPPORTED_OS}
ARCHS ${UBSAN_SUPPORTED_ARCH}
OBJECT_LIBS RTUbsan_minimal
CFLAGS ${UBSAN_CFLAGS}
// that "too many errors" has already been reported.
static __sanitizer::atomic_uint32_t caller_pcs_sz;
-__attribute__((noinline)) static bool report_this_error(void *caller_p) {
- uintptr_t caller = reinterpret_cast<uintptr_t>(caller_p);
- if (caller == 0) return false;
+__attribute__((noinline)) static bool report_this_error(uintptr_t caller) {
+ if (caller == 0)
+ return false;
while (true) {
unsigned sz = __sanitizer::atomic_load_relaxed(&caller_pcs_sz);
if (sz > kMaxCallerPcs) return false; // early exit
}
}
+__attribute__((noinline)) static void decorate_msg(char *buf,
+ uintptr_t caller) {
+ // print the address by nibbles
+ for (unsigned shift = sizeof(uintptr_t) * 8; shift;) {
+ shift -= 4;
+ unsigned nibble = (caller >> shift) & 0xf;
+ *(buf++) = nibble < 10 ? nibble + '0' : nibble - 10 + 'a';
+ }
+ // finish the message
+ buf[0] = '\n';
+ buf[1] = '\0';
+}
+
#if defined(__ANDROID__)
extern "C" __attribute__((weak)) void android_set_abort_message(const char *);
static void abort_with_message(const char *msg) {
#define INTERFACE extern "C" __attribute__((visibility("default")))
-// FIXME: add caller pc to the error message (possibly as "ubsan: error-type
-// @1234ABCD").
+// How many chars we need to reserve to print an address.
+constexpr unsigned kAddrBuf = SANITIZER_WORDSIZE / 4;
+#define MSG_TMPL(msg) "ubsan: " msg " by 0x"
+#define MSG_TMPL_END(buf, msg) (buf + sizeof(MSG_TMPL(msg)) - 1)
+// Reserve an additional byte for '\n'.
+#define MSG_BUF_LEN(msg) (sizeof(MSG_TMPL(msg)) + kAddrBuf + 1)
+
#define HANDLER_RECOVER(name, msg) \
INTERFACE void __ubsan_handle_##name##_minimal() { \
- if (!report_this_error(__builtin_return_address(0))) return; \
- message("ubsan: " msg "\n"); \
+ uintptr_t caller = GET_CALLER_PC(); \
+ if (!report_this_error(caller)) return; \
+ char msg_buf[MSG_BUF_LEN(msg)] = MSG_TMPL(msg); \
+ decorate_msg(MSG_TMPL_END(msg_buf, msg), caller); \
+ message(msg_buf); \
}
#define HANDLER_NORECOVER(name, msg) \
INTERFACE void __ubsan_handle_##name##_minimal_abort() { \
- message("ubsan: " msg "\n"); \
- abort_with_message("ubsan: " msg); \
+ char msg_buf[MSG_BUF_LEN(msg)] = MSG_TMPL(msg); \
+ decorate_msg(MSG_TMPL_END(msg_buf, msg), GET_CALLER_PC()); \
+ message(msg_buf); \
+ abort_with_message(msg_buf); \
}
#define HANDLER(name, msg) \
xray_trampoline_powerpc64_asm.S
)
+set(hexagon_SOURCES
+ xray_hexagon.cpp
+ xray_trampoline_hexagon.S
+ )
+
set(XRAY_IMPL_HEADERS
xray_allocator.h
xray_basic_flags.h
${x86_64_SOURCES}
${arm_SOURCES}
${armhf_SOURCES}
+ ${hexagon_SOURCES}
${mips_SOURCES}
${mipsel_SOURCES}
${mips64_SOURCES}
include_directories(..)
include_directories(../../include)
-set(XRAY_CFLAGS ${COMPILER_RT_COMMON_CFLAGS})
+set(XRAY_CFLAGS
+ ${COMPILER_RT_COMMON_CFLAGS}
+ ${COMPILER_RT_CXX_CFLAGS})
set(XRAY_COMMON_DEFINITIONS XRAY_HAS_EXCEPTIONS=1)
+# Too many existing bugs, needs cleanup.
+append_list_if(COMPILER_RT_HAS_WNO_FORMAT -Wno-format XRAY_CFLAGS)
+
# We don't need RTTI in XRay, so turn that off.
append_rtti_flag(OFF XRAY_CFLAGS)
+set(XRAY_LINK_FLAGS ${COMPILER_RT_COMMON_LINK_FLAGS})
+set(XRAY_LINK_LIBS
+ ${COMPILER_RT_UNWINDER_LINK_LIBS}
+ ${COMPILER_RT_CXX_LINK_LIBS})
+
append_list_if(
COMPILER_RT_HAS_XRAY_COMPILER_FLAG XRAY_SUPPORTED=1 XRAY_COMMON_DEFINITIONS)
append_list_if(
endif()
if (APPLE)
- set(XRAY_LINK_LIBS ${SANITIZER_COMMON_LINK_LIBS})
add_asm_sources(XRAY_ASM_SOURCES xray_trampoline_x86_64.S)
add_weak_symbols("sanitizer_common" WEAK_SYMBOL_LINK_FLAGS)
RTSanitizerCommonLibc
CFLAGS ${XRAY_CFLAGS}
DEFS ${XRAY_COMMON_DEFINITIONS}
- LINK_FLAGS ${SANITIZER_COMMON_LINK_FLAGS} ${WEAK_SYMBOL_LINK_FLAGS}
+ LINK_FLAGS ${XRAY_LINK_FLAGS} ${WEAK_SYMBOL_LINK_FLAGS}
LINK_LIBS ${XRAY_LINK_LIBS}
PARENT_TARGET xray)
add_compiler_rt_runtime(clang_rt.xray-fdr
OBJECT_LIBS RTXrayFDR
CFLAGS ${XRAY_CFLAGS}
DEFS ${XRAY_COMMON_DEFINITIONS}
- LINK_FLAGS ${SANITIZER_COMMON_LINK_FLAGS} ${WEAK_SYMBOL_LINK_FLAGS}
+ LINK_FLAGS ${XRAY_LINK_FLAGS} ${WEAK_SYMBOL_LINK_FLAGS}
LINK_LIBS ${XRAY_LINK_LIBS}
PARENT_TARGET xray)
add_compiler_rt_runtime(clang_rt.xray-basic
OBJECT_LIBS RTXrayBASIC
CFLAGS ${XRAY_CFLAGS}
DEFS ${XRAY_COMMON_DEFINITIONS}
- LINK_FLAGS ${SANITIZER_COMMON_LINK_FLAGS} ${WEAK_SYMBOL_LINK_FLAGS}
+ LINK_FLAGS ${XRAY_LINK_FLAGS} ${WEAK_SYMBOL_LINK_FLAGS}
LINK_LIBS ${XRAY_LINK_LIBS}
PARENT_TARGET xray)
add_compiler_rt_runtime(clang_rt.xray-profiling
OBJECT_LIBS RTXrayPROFILING
CFLAGS ${XRAY_CFLAGS}
DEFS ${XRAY_COMMON_DEFINITIONS}
- LINK_FLAGS ${SANITIZER_COMMON_LINK_FLAGS} ${WEAK_SYMBOL_LINK_FLAGS}
+ LINK_FLAGS ${XRAY_LINK_FLAGS} ${WEAK_SYMBOL_LINK_FLAGS}
LINK_LIBS ${XRAY_LINK_LIBS}
PARENT_TARGET xray)
else() # not Apple
STATIC
ARCHS ${arch}
CFLAGS ${XRAY_CFLAGS}
+ LINK_FLAGS ${XRAY_LINK_FLAGS}
+ LINK_LIBS ${XRAY_LINK_LIBS}
DEFS ${XRAY_COMMON_DEFINITIONS}
OBJECT_LIBS ${XRAY_COMMON_RUNTIME_OBJECT_LIBS} RTXray
PARENT_TARGET xray)
STATIC
ARCHS ${arch}
CFLAGS ${XRAY_CFLAGS}
+ LINK_FLAGS ${XRAY_LINK_FLAGS}
+ LINK_LIBS ${XRAY_LINK_LIBS}
DEFS ${XRAY_COMMON_DEFINITIONS}
OBJECT_LIBS RTXrayFDR
PARENT_TARGET xray)
STATIC
ARCHS ${arch}
CFLAGS ${XRAY_CFLAGS}
+ LINK_FLAGS ${XRAY_LINK_FLAGS}
+ LINK_LIBS ${XRAY_LINK_LIBS}
DEFS ${XRAY_COMMON_DEFINITIONS}
OBJECT_LIBS RTXrayBASIC
PARENT_TARGET xray)
STATIC
ARCHS ${arch}
CFLAGS ${XRAY_CFLAGS}
+ LINK_FLAGS ${XRAY_LINK_FLAGS}
+ LINK_LIBS ${XRAY_LINK_LIBS}
DEFS ${XRAY_COMMON_DEFINITIONS}
OBJECT_LIBS RTXrayPROFILING
PARENT_TARGET xray)
set(XRAY_UNITTEST_LINK_FLAGS
${COMPILER_RT_UNITTEST_LINK_FLAGS}
${CMAKE_THREAD_LIBS_INIT}
- )
+ ${COMPILER_RT_UNWINDER_LINK_LIBS}
+ ${COMPILER_RT_CXX_LINK_LIBS})
if (NOT APPLE)
# Needed by LLVMSupport.
LLVM_ENABLE_TERMINFO
-l${COMPILER_RT_TERMINFO_LIB} XRAY_UNITTEST_LINK_FLAGS)
+ # We add the library directories one at a time in our CFLAGS.
+ foreach (DIR ${LLVM_LIBRARY_DIR})
+ list(APPEND XRAY_UNITTEST_LINK_FLAGS -L${DIR})
+ endforeach()
+
if (COMPILER_RT_STANDALONE_BUILD)
- append_list_if(COMPILER_RT_HAS_LLVMXRAY ${LLVM_XRAY_LDFLAGS} XRAY_UNITTEST_LINK_FLAGS)
- append_list_if(COMPILER_RT_HAS_LLVMXRAY ${LLVM_XRAY_LIBLIST} XRAY_UNITTEST_LINK_FLAGS)
- append_list_if(COMPILER_RT_HAS_LLVMTESTINGSUPPORT
- ${LLVM_TESTINGSUPPORT_LDFLAGS} XRAY_UNITTEST_LINK_FLAGS)
- append_list_if(COMPILER_RT_HAS_LLVMTESTINGSUPPORT
- ${LLVM_TESTINGSUPPORT_LIBLIST} XRAY_UNITTEST_LINK_FLAGS)
+ if (COMPILER_RT_HAS_LLVMXRAY OR COMPILER_RT_HAS_LLVMTESTINGSUPPORT)
+ if (LLVM_LINK_LLVM_DYLIB)
+ list(APPEND XRAY_UNITTEST_LINK_FLAGS -lLLVM)
+ endif()
+ else()
+ if (COMPILER_RT_HAS_LLVMXRAY)
+ list(APPEND XRAY_UNITTEST_LINK_FLAGS -lLLVMXRay)
+ endif()
+ if (COMPILER_RT_HAS_TESTINGSUPPORT)
+ list(APPEND XRAY_UNITTEST_LINK_FLAGS -lLLVMTestingSupport)
+ endif()
+ list(APPEND XRAY_UNITTEST_LINK_FLAGS -lLLVMSupport -lLLVMDemangle)
+ endif()
else()
- # We add the library directories one at a time in our CFLAGS.
- foreach (DIR ${LLVM_LIBRARY_DIR})
- list(APPEND XRAY_UNITTEST_LINK_FLAGS -L${DIR})
- endforeach()
-
# We also add the actual libraries to link as dependencies.
list(APPEND XRAY_UNITTEST_LINK_FLAGS -lLLVMXRay -lLLVMSupport -lLLVMDemangle -lLLVMTestingSupport)
endif()
append_list_if(COMPILER_RT_HAS_LIBEXECINFO -lexecinfo XRAY_UNITTEST_LINK_FLAGS)
endif()
-foreach(lib ${SANITIZER_TEST_CXX_LIBRARIES})
- list(APPEND XRAY_UNITTEST_LINK_FLAGS -l${lib})
-endforeach()
-
macro(add_xray_unittest testname)
cmake_parse_arguments(TEST "" "" "SOURCES;HEADERS" ${ARGN})
if(UNIX AND NOT APPLE)
${XRAY_HEADERS} ${XRAY_ALL_SOURCE_FILES_ABS_PATHS}
"test_helpers.h"
RUNTIME "${XRAY_RUNTIME_LIBS}"
- DEPS gtest xray llvm-xray LLVMXRay LLVMTestingSupport
+ DEPS llvm_gtest xray llvm-xray LLVMXRay LLVMTestingSupport
CFLAGS ${XRAY_UNITTEST_CFLAGS}
LINK_FLAGS ${TARGET_LINK_FLAGS} ${XRAY_UNITTEST_LINK_FLAGS}
)
// We use a different allocator here to make sure that we're able to transfer
// data into a FunctionCallTrie which uses a different allocator. This
- // reflects the inteded usage scenario for when we're collecting profiles that
- // aggregate across threads.
+ // reflects the intended usage scenario for when we're collecting profiles
+ // that aggregate across threads.
auto B = FunctionCallTrie::InitAllocators();
FunctionCallTrie Merged(B);
int ErrNo = 0;
if (UNLIKELY(internal_iserror(B, &ErrNo))) {
if (Verbosity())
- Report(
- "XRay Profiling: Failed to allocate memory of size %d; Error = %d.\n",
- RoundedSize, B);
+ Report("XRay Profiling: Failed to allocate memory of size %zu; Error = "
+ "%zu\n",
+ RoundedSize, B);
return nullptr;
}
#endif
int ErrNo = 0;
if (UNLIKELY(internal_iserror(B, &ErrNo))) {
if (Verbosity())
- Report(
- "XRay Profiling: Failed to allocate memory of size %d; Error = %d.\n",
- RoundedSize, B);
+ Report("XRay Profiling: Failed to allocate memory of size %zu; Error = "
+ "%zu\n",
+ RoundedSize, B);
return nullptr;
}
#endif
BackingStore = allocateBuffer(MaxMemory);
if (BackingStore == nullptr) {
if (Verbosity())
- Report("XRay Profiling: Failed to allocate memory for allocator.\n");
+ Report("XRay Profiling: Failed to allocate memory for allocator\n");
return nullptr;
}
AlignedNextBlock = BackingStore = nullptr;
if (Verbosity())
Report("XRay Profiling: Cannot obtain enough memory from "
- "preallocated region.\n");
+ "preallocated region\n");
return nullptr;
}
//
//===----------------------------------------------------------------------===//
//
-// This file is a part of XRay, a dynamic runtime instruementation system.
+// This file is a part of XRay, a dynamic runtime instrumentation system.
//
// XRay Basic Mode runtime flags.
//===----------------------------------------------------------------------===//
#include <fcntl.h>
#include <pthread.h>
#include <sys/stat.h>
-#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_MAC
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_APPLE
#include <sys/syscall.h>
#endif
#include <sys/types.h>
if (TLD.ShadowStack)
InternalFree(TLD.ShadowStack);
if (Verbosity())
- Report("Cleaned up log for TID: %d\n", GetTid());
+ Report("Cleaned up log for TID: %llu\n", GetTid());
});
if (TLD.LogWriter == nullptr || TLD.BufferOffset == 0) {
if (Verbosity())
- Report("Skipping buffer for TID: %d; Offset = %llu\n", GetTid(),
+ Report("Skipping buffer for TID: %llu; Offset = %zu\n", GetTid(),
TLD.BufferOffset);
return;
}
//
//===----------------------------------------------------------------------===//
//
-// This file is a part of XRay, a dynamic runtime instruementation system.
+// This file is a part of XRay, a dynamic runtime instrumentation system.
//
// Defines the interface for a buffer queue implementation.
//
//
//===----------------------------------------------------------------------===//
//
-// This file is a part of XRay, a dynamic runtime instruementation system.
+// This file is a part of XRay, a dynamic runtime instrumentation system.
//
// XRay runtime flags.
//===----------------------------------------------------------------------===//
--- /dev/null
+//===-- xray_hexagon.cpp --------------------------------------*- C++ ---*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of XRay, a dynamic runtime instrumentation system.
+//
+// Implementation of hexagon-specific routines (32-bit).
+//
+//===----------------------------------------------------------------------===//
+#include "sanitizer_common/sanitizer_common.h"
+#include "xray_defs.h"
+#include "xray_interface_internal.h"
+#include <assert.h>
+#include <atomic>
+
+namespace __xray {
+
+// The machine codes for some instructions used in runtime patching.
+enum PatchOpcodes : uint32_t {
+ PO_JUMPI_14 = 0x5800c00a, // jump #0x014 (PC + 0x014)
+ PO_CALLR_R6 = 0x50a6c000, // indirect call: callr r6
+ PO_TFR_IMM = 0x78000000, // transfer immed
+ // ICLASS 0x7 - S2-type A-type
+ PO_IMMEXT = 0x00000000, // constant extender
+};
+
+enum PacketWordParseBits : uint32_t {
+ PP_DUPLEX = 0x00 << 14,
+ PP_NOT_END = 0x01 << 14,
+ PP_PACKET_END = 0x03 << 14,
+};
+
+enum RegNum : uint32_t {
+ RN_R6 = 0x6,
+ RN_R7 = 0x7,
+};
+
+inline static uint32_t
+encodeExtendedTransferImmediate(uint32_t Imm, RegNum DestReg,
+ bool PacketEnd = false) XRAY_NEVER_INSTRUMENT {
+ static const uint32_t REG_MASK = 0x1f;
+ assert((DestReg & (~REG_MASK)) == 0);
+ // The constant-extended register transfer encodes the 6 least
+ // significant bits of the effective constant:
+ Imm = Imm & 0x03f;
+ const PacketWordParseBits ParseBits = PacketEnd ? PP_PACKET_END : PP_NOT_END;
+
+ return PO_TFR_IMM | ParseBits | (Imm << 5) | (DestReg & REG_MASK);
+}
+
+inline static uint32_t
+encodeConstantExtender(uint32_t Imm) XRAY_NEVER_INSTRUMENT {
+ // Bits Name Description
+ // ----- ------- ------------------------------------------
+ // 31:28 ICLASS Instruction class = 0000
+ // 27:16 high High 12 bits of 26-bit constant extension
+ // 15:14 Parse Parse bits
+ // 13:0 low Low 14 bits of 26-bit constant extension
+ static const uint32_t IMM_MASK_LOW = 0x03fff;
+ static const uint32_t IMM_MASK_HIGH = 0x00fff << 14;
+
+ // The extender encodes the 26 most significant bits of the effective
+ // constant:
+ Imm = Imm >> 6;
+
+ const uint32_t high = (Imm & IMM_MASK_HIGH) << 16;
+ const uint32_t low = Imm & IMM_MASK_LOW;
+
+ return PO_IMMEXT | high | PP_NOT_END | low;
+}
+
+static void WriteInstFlushCache(void *Addr, uint32_t NewInstruction) {
+ asm volatile("icinva(%[inst_addr])\n\t"
+ "isync\n\t"
+ "memw(%[inst_addr]) = %[new_inst]\n\t"
+ "dccleaninva(%[inst_addr])\n\t"
+ "syncht\n\t"
+ :
+ : [ inst_addr ] "r"(Addr), [ new_inst ] "r"(NewInstruction)
+ : "memory");
+}
+
+inline static bool patchSled(const bool Enable, const uint32_t FuncId,
+ const XRaySledEntry &Sled,
+ void (*TracingHook)()) XRAY_NEVER_INSTRUMENT {
+ // When |Enable| == true,
+ // We replace the following compile-time stub (sled):
+ //
+ // .L_xray_sled_N:
+ // <xray_sled_base>:
+ // { jump .Ltmp0 }
+ // { nop
+ // nop
+ // nop
+ // nop }
+ // .Ltmp0:
+
+ // With the following runtime patch:
+ //
+ // xray_sled_n (32-bit):
+ //
+ // <xray_sled_n>:
+ // { immext(#...) // upper 26-bits of func id
+ // r7 = ##... // lower 6-bits of func id
+ // immext(#...) // upper 26-bits of trampoline
+ // r6 = ##... } // lower 6 bits of trampoline
+ // { callr r6 }
+ //
+ // When |Enable|==false, we set back the first instruction in the sled to be
+ // { jump .Ltmp0 }
+
+ uint32_t *FirstAddress = reinterpret_cast<uint32_t *>(Sled.address());
+ if (Enable) {
+ uint32_t *CurAddress = FirstAddress + 1;
+ *CurAddress = encodeExtendedTransferImmediate(FuncId, RN_R7);
+ CurAddress++;
+ *CurAddress = encodeConstantExtender(reinterpret_cast<uint32_t>(TracingHook));
+ CurAddress++;
+ *CurAddress =
+ encodeExtendedTransferImmediate(reinterpret_cast<uint32_t>(TracingHook), RN_R6, true);
+ CurAddress++;
+
+ *CurAddress = uint32_t(PO_CALLR_R6);
+
+ WriteInstFlushCache(FirstAddress, uint32_t(encodeConstantExtender(FuncId)));
+ } else {
+ WriteInstFlushCache(FirstAddress, uint32_t(PatchOpcodes::PO_JUMPI_14));
+ }
+ return true;
+}
+
+bool patchFunctionEntry(const bool Enable, const uint32_t FuncId,
+ const XRaySledEntry &Sled,
+ void (*Trampoline)()) XRAY_NEVER_INSTRUMENT {
+ return patchSled(Enable, FuncId, Sled, Trampoline);
+}
+
+bool patchFunctionExit(const bool Enable, const uint32_t FuncId,
+ const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT {
+ return patchSled(Enable, FuncId, Sled, __xray_FunctionExit);
+}
+
+bool patchFunctionTailExit(const bool Enable, const uint32_t FuncId,
+ const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT {
+ return patchSled(Enable, FuncId, Sled, __xray_FunctionExit);
+}
+
+bool patchCustomEvent(const bool Enable, const uint32_t FuncId,
+ const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT {
+ // FIXME: Implement in hexagon?
+ return false;
+}
+
+bool patchTypedEvent(const bool Enable, const uint32_t FuncId,
+ const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT {
+ // FIXME: Implement in hexagon?
+ return false;
+}
+
+} // namespace __xray
+
+extern "C" void __xray_ArgLoggerEntry() XRAY_NEVER_INSTRUMENT {
+ // FIXME: this will have to be implemented in the trampoline assembly file
+}
extern const XRayFunctionSledIndex __start_xray_fn_idx[] __attribute__((weak));
extern const XRayFunctionSledIndex __stop_xray_fn_idx[] __attribute__((weak));
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
// HACK: This is a temporary workaround to make XRay build on
// Darwin, but it will probably not work at runtime.
const XRaySledEntry __start_xray_instr_map[] = {};
#include "xray_interface_internal.h"
-#include <cstdint>
+#include <cinttypes>
#include <cstdio>
#include <errno.h>
#include <limits>
static const int16_t cSledLength = 64;
#elif defined(__powerpc64__)
static const int16_t cSledLength = 8;
+#elif defined(__hexagon__)
+static const int16_t cSledLength = 20;
#else
#error "Unsupported CPU Architecture"
#endif /* CPU architecture */
Success = patchTypedEvent(Enable, FuncId, Sled);
break;
default:
- Report("Unsupported sled kind '%d' @%04x\n", Sled.Address, int(Sled.Kind));
+ Report("Unsupported sled kind '%" PRIu64 "' @%04x\n", Sled.Address,
+ int(Sled.Kind));
return false;
}
return Success;
? flags()->xray_page_size_override
: GetPageSizeCached();
if ((PageSize == 0) || ((PageSize & (PageSize - 1)) != 0)) {
- Report("System page size is not a power of two: %lld\n", PageSize);
+ Report("System page size is not a power of two: %zu\n", PageSize);
return XRayPatchingStatus::FAILED;
}
? flags()->xray_page_size_override
: GetPageSizeCached();
if ((PageSize == 0) || ((PageSize & (PageSize - 1)) != 0)) {
- Report("Provided page size is not a power of two: %lld\n", PageSize);
+ Report("Provided page size is not a power of two: %zu\n", PageSize);
return XRayPatchingStatus::FAILED;
}
- // Here we compute the minumum sled and maximum sled associated with a
+ // Here we compute the minimum sled and maximum sled associated with a
// particular function ID.
auto SledRange = InstrMap.SledsIndex ? InstrMap.SledsIndex[FuncId - 1]
: findFunctionSleds(FuncId, InstrMap);
unsigned char Version;
unsigned char Padding[13]; // Need 32 bytes
uint64_t function() const {
- if (Version < 2)
- return Function;
// The target address is relative to the location of the Function variable.
return reinterpret_cast<uint64_t>(&Function) + Function;
}
uint64_t address() const {
- if (Version < 2)
- return Address;
// The target address is relative to the location of the Address variable.
return reinterpret_cast<uint64_t>(&Address) + Address;
}
unsigned char Version;
unsigned char Padding[5]; // Need 16 bytes
uint32_t function() const {
- if (Version < 2)
- return Function;
// The target address is relative to the location of the Function variable.
return reinterpret_cast<uint32_t>(&Function) + Function;
}
uint32_t address() const {
- if (Version < 2)
- return Address;
// The target address is relative to the location of the Address variable.
return reinterpret_cast<uint32_t>(&Address) + Address;
}
#include <cstdint>
#include <mutex>
+#ifdef __linux__
#include <sys/platform/ppc.h>
+#elif defined(__FreeBSD__)
+#include <sys/types.h>
+#include <sys/sysctl.h>
+
+#define __ppc_get_timebase __builtin_ppc_get_timebase
+
+uint64_t __ppc_get_timebase_freq (void)
+{
+ uint64_t tb_freq = 0;
+ size_t length = sizeof(tb_freq);
+ sysctlbyname("kern.timecounter.tc.timebase.frequency", &tb_freq, &length, nullptr, 0);
+ return tb_freq;
+}
+#endif
#include "xray_defs.h"
return XRayLogInitStatus::XRAY_LOG_UNINITIALIZED;
}
- // If we've succeded, set the global pointer to the initialised storage.
+ // If we've succeeded, set the global pointer to the initialised storage.
BQ = reinterpret_cast<BufferQueue *>(&BufferQueueStorage);
} else {
BQ->finalize();
--- /dev/null
+//===-- xray_trampoline_hexagon.s -------------------------------*- ASM -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of XRay, a dynamic runtime instrumentation system.
+//
+// This implements the hexagon-specific assembler for the trampolines.
+//
+//===----------------------------------------------------------------------===//
+
+#include "../builtins/assembly.h"
+#include "../sanitizer_common/sanitizer_asm.h"
+
+.macro SAVE_REGISTERS
+memw(sp+#0)=r0
+memw(sp+#4)=r1
+memw(sp+#8)=r2
+memw(sp+#12)=r3
+memw(sp+#16)=r4
+.endm
+.macro RESTORE_REGISTERS
+r0=memw(sp+#0)
+r1=memw(sp+#4)
+r2=memw(sp+#8)
+r3=memw(sp+#12)
+r4=memw(sp+#16)
+.endm
+
+.macro CALL_PATCHED_FUNC entry_type
+ // if (xray::XRayPatchedFunctionE != NULL)
+ // xray::XRayPatchedFunctionE(FuncType);
+
+ r8 = #ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)
+
+ // The patched sled puts the function type
+ // into r6. Move it into r0 to pass it to
+ // the patched function.
+ { r0 = r6
+ r1 = \entry_type
+ p0 = !cmp.eq(r8, #0)
+ if (p0) callr r8 }
+.endm
+
+ .text
+ .globl ASM_SYMBOL(__xray_FunctionEntry)
+ ASM_HIDDEN(__xray_FunctionEntry)
+ ASM_TYPE_FUNCTION(__xray_FunctionEntry)
+# LLVM-MCA-BEGIN __xray_FunctionEntry
+ASM_SYMBOL(__xray_FunctionEntry):
+ CFI_STARTPROC
+ SAVE_REGISTERS
+
+ CALL_PATCHED_FUNC #0 // XRayEntryType::ENTRY
+.Ltmp0:
+ RESTORE_REGISTERS
+ // return
+# LLVM-MCA-END
+ ASM_SIZE(__xray_FunctionEntry)
+ CFI_ENDPROC
+
+
+ .globl ASM_SYMBOL(__xray_FunctionExit)
+ ASM_HIDDEN(__xray_FunctionExit)
+ ASM_TYPE_FUNCTION(__xray_FunctionExit)
+# LLVM-MCA-BEGIN __xray_FunctionExit
+ASM_SYMBOL(__xray_FunctionExit):
+ CFI_STARTPROC
+ SAVE_REGISTERS
+
+ CALL_PATCHED_FUNC #1 // XRayEntryType::EXIT
+.Ltmp1:
+ RESTORE_REGISTERS
+ // return
+ jumpr r31
+# LLVM-MCA-END
+ ASM_SIZE(__xray_FunctionExit)
+ CFI_ENDPROC
+
+
+ .globl ASM_SYMBOL(__xray_FunctionTailExit)
+ ASM_HIDDEN(__xray_FunctionTailExit)
+ ASM_TYPE_FUNCTION(__xray_FunctionTailExit)
+# LLVM-MCA-BEGIN __xray_FunctionTailExit
+ASM_SYMBOL(__xray_FunctionTailExit):
+ CFI_STARTPROC
+ SAVE_REGISTERS
+
+ CALL_PATCHED_FUNC #2 // XRayEntryType::TAIL
+.Ltmp2:
+ RESTORE_REGISTERS
+ // return
+ jumpr r31
+# LLVM-MCA-END
+ ASM_SIZE(__xray_FunctionTailExit)
+ CFI_ENDPROC
#include "xray_x86_64.inc"
#elif defined(__powerpc64__)
#include "xray_powerpc64.inc"
-#elif defined(__arm__) || defined(__aarch64__) || defined(__mips__)
+#elif defined(__arm__) || defined(__aarch64__) || defined(__mips__) || \
+ defined(__hexagon__)
// Emulated TSC.
// There is no instruction like RDTSCP in user mode on ARM. ARM's CP15 does
// not have a constant frequency like TSC on x86(_64), it may go faster
#include "xray_defs.h"
#include "xray_interface_internal.h"
-#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_MAC
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_APPLE
#include <sys/types.h>
#include <sys/sysctl.h>
#elif SANITIZER_FUCHSIA
}
return TSCFrequency == -1 ? 0 : static_cast<uint64_t>(TSCFrequency);
}
-#elif SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_MAC
+#elif SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_APPLE
uint64_t getTSCFrequency() XRAY_NEVER_INSTRUMENT {
long long TSCFrequency = -1;
size_t tscfreqsz = sizeof(TSCFrequency);
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
if (internal_sysctlbyname("machdep.tsc.frequency", &TSCFrequency,
&tscfreqsz, NULL, 0) != -1) {
int64_t TrampolineOffset = reinterpret_cast<int64_t>(Trampoline) -
(static_cast<int64_t>(Address) + 11);
if (TrampolineOffset < MinOffset || TrampolineOffset > MaxOffset) {
- Report("XRay Entry trampoline (%p) too far from sled (%p)\n", Trampoline,
+ Report("XRay Entry trampoline (%p) too far from sled (%p)\n",
+ reinterpret_cast<void *>(Trampoline),
reinterpret_cast<void *>(Address));
return false;
}
(static_cast<int64_t>(Address) + 11);
if (TrampolineOffset < MinOffset || TrampolineOffset > MaxOffset) {
Report("XRay Exit trampoline (%p) too far from sled (%p)\n",
- __xray_FunctionExit, reinterpret_cast<void *>(Address));
+ reinterpret_cast<void *>(__xray_FunctionExit),
+ reinterpret_cast<void *>(Address));
return false;
}
if (Enable) {
(static_cast<int64_t>(Address) + 11);
if (TrampolineOffset < MinOffset || TrampolineOffset > MaxOffset) {
Report("XRay Tail Exit trampoline (%p) too far from sled (%p)\n",
- __xray_FunctionTailExit, reinterpret_cast<void *>(Address));
+ reinterpret_cast<void *>(__xray_FunctionTailExit),
+ reinterpret_cast<void *>(Address));
return false;
}
if (Enable) {
# inefficient handling of large mmap'd regions (terabytes) by the kernel.
lit_config.parallelism_groups["shadow-memory"] = 3
- # The test config gets pickled and sent to multiprocessing workers, and that
- # only works for code if it is stored at the top level of some module.
- # Therefore, we have to put the code in a .py file, add it to path, and import
- # it to store it in the config.
- import site
- site.addsitedir(os.path.dirname(__file__))
- import lit_unittest_cfg_utils
- config.darwin_sanitizer_parallelism_group_func = \
- lit_unittest_cfg_utils.darwin_sanitizer_parallelism_group_func
+ # Disable libmalloc nano allocator due to crashes running on macOS 12.0.
+ # rdar://80086125
+ config.environment['MallocNanoZone'] = '0'
+
+ # We crash when we set DYLD_INSERT_LIBRARIES for unit tests, so interceptors
+ # don't work.
+ config.environment['ASAN_OPTIONS'] = 'verify_interceptors=0'
+ config.environment['TSAN_OPTIONS'] = 'verify_interceptors=0'
@LIT_SITE_CFG_IN_HEADER@
# Generic config options for all compiler-rt unit tests.
-config.target_triple = "@TARGET_TRIPLE@"
+config.target_triple = "@LLVM_TARGET_TRIPLE@"
config.llvm_src_root = "@LLVM_MAIN_SRC_DIR@"
config.llvm_obj_root = "@LLVM_BINARY_DIR@"
-config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
+config.llvm_tools_dir = lit_config.substitute("@LLVM_TOOLS_DIR@")
config.compiler_rt_src_root = "@COMPILER_RT_SOURCE_DIR@"
-config.compiler_rt_libdir = "@COMPILER_RT_RESOLVED_LIBRARY_OUTPUT_DIR@"
-config.llvm_build_mode = "@LLVM_BUILD_MODE@"
+config.compiler_rt_libdir = lit_config.substitute("@COMPILER_RT_RESOLVED_LIBRARY_OUTPUT_DIR@")
+config.enable_per_target_runtime_dir = @LLVM_ENABLE_PER_TARGET_RUNTIME_DIR_PYBOOL@
+config.llvm_build_mode = lit_config.substitute("@LLVM_BUILD_MODE@")
config.host_arch = "@HOST_ARCH@"
config.host_os = "@HOST_OS@"
config.llvm_lib_dir = "@LLVM_LIBRARY_DIR@"
config.gwp_asan = @COMPILER_RT_HAS_GWP_ASAN_PYBOOL@
config.emulator = "@COMPILER_RT_EMULATOR@"
-# LLVM tools dir and build mode can be passed in lit parameters,
-# so try to apply substitution.
-try:
- config.llvm_tools_dir = config.llvm_tools_dir % lit_config.params
- config.compiler_rt_libdir = config.compiler_rt_libdir % lit_config.params
- config.llvm_build_mode = config.llvm_build_mode % lit_config.params
-except KeyError as e:
- key, = e.args
- lit_config.fatal("unable to find %r parameter, use '--param=%s=VALUE'" % (key, key))
-
# Setup attributes common for all compiler-rt unit tests.
lit_config.load_config(config, "@COMPILER_RT_SOURCE_DIR@/unittests/lit.common.unit.cfg.py")
#===------------------------------------------------------------------------===#
BEGIN {
- # harcode the script name
+ # hardcode the script name
script_name = "generate_netbsd_ioctls.awk"
outputinc = "../lib/sanitizer_common/sanitizer_interceptors_ioctl_netbsd.inc"
}
pcmd("#undef _")
- pcmd("} // NOLINT")
+ pcmd("}")
pcmd("")
pcmd("static bool ioctl_initialized = false;")
pcmd("")
#===------------------------------------------------------------------------===#
BEGIN {
- # harcode the script name
+ # hardcode the script name
script_name = "generate_netbsd_syscalls.awk"
outputh = "../include/sanitizer/netbsd_syscall_hooks.h"
outputinc = "../lib/sanitizer_common/sanitizer_syscalls_netbsd.inc"
<p><b>builtins</b> is known to work on the following platforms:</p>
<ul>
<li>Machine Architectures: i386, X86-64, SPARC64, ARM, PowerPC, PowerPC 64.</li>
- <li>OS: AuroraUX, DragonFlyBSD, FreeBSD, NetBSD, Linux, Darwin.</li>
+ <li>OS: DragonFlyBSD, FreeBSD, NetBSD, OpenBSD, Linux, Darwin.</li>
</ul>
<p>Most sanitizer runtimes are supported only on Linux x86-64. See tool-specific
<p>Generally, you need to build LLVM/Clang in order to build compiler-rt. You
can build it either together with llvm and clang, or separately.
- <p>To build it together, simply add compiler-rt to the -DLLVM_ENABLE_PROJECTS= option to
+ <p>To build it together, simply add compiler-rt to the -DLLVM_ENABLE_RUNTIMES= option to
cmake.
<p>To build it separately, first
command in either LLVM/Clang/compiler-rt or standalone
compiler-rt build tree.</p>
- <p>compiler-rt doesn't have its own mailing list, if you have questions please
- email the <a
- href="https://lists.llvm.org/mailman/listinfo/llvm-dev">llvm-dev</a> mailing
- list. Commits to the compiler-rt SVN module are automatically sent to the
- <a
+ <p>If you have questions, please ask on the <a href="https://discourse.llvm.org/c/runtimes/64">
+ Discourse forums</a> under the Runtime category. Commits to compiler-rt are automatically
+ sent to the <a
href="https://lists.llvm.org/mailman/listinfo/llvm-commits">llvm-commits</a>
mailing list.</p>
</div>
<div class="submenu">
<label>Quick Links</label>
- <a href="http://lists.llvm.org/mailman/listinfo/llvm-dev">llvm-dev</a>
+ <a href="https://discourse.llvm.org">LLVM Forum</a>
<a href="http://lists.llvm.org/mailman/listinfo/llvm-commits">llvm-commits</a>
<a href="http://llvm.org/bugs/">Bug Reports</a>
<a href="https://github.com/llvm/llvm-project/tree/main/compiler-rt/">Browse Sources</a>
projects / openbsd / commitdiff